You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by jd...@apache.org on 2016/07/25 17:15:06 UTC
[01/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Repository: incubator-kudu
Updated Branches:
refs/heads/master 88a1cf0db -> 81b2d9a7c
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
new file mode 100644
index 0000000..ec929d7
--- /dev/null
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.spark.kudu
+
+import java.sql.Timestamp
+import java.text.SimpleDateFormat
+import java.util.TimeZone
+
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.functions._
+import org.junit.Assert._
+import org.junit.runner.RunWith
+import org.kududb.client.CreateTableOptions
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{BeforeAndAfter, FunSuite}
+
+import scala.collection.JavaConverters._
+import scala.collection.immutable.IndexedSeq
+
+@RunWith(classOf[JUnitRunner])
+class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter {
+
+ test("timestamp conversion") {
+ val epoch = new Timestamp(0)
+ assertEquals(0, KuduRelation.timestampToMicros(epoch))
+ assertEquals(epoch, KuduRelation.microsToTimestamp(0))
+
+ val t1 = new Timestamp(0)
+ t1.setNanos(123456000)
+ assertEquals(123456, KuduRelation.timestampToMicros(t1))
+ assertEquals(t1, KuduRelation.microsToTimestamp(123456))
+
+ val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")
+ iso8601.setTimeZone(TimeZone.getTimeZone("UTC"))
+
+ val t3 = new Timestamp(iso8601.parse("1923-12-01T00:44:36.876").getTime)
+ t3.setNanos(876544000)
+ assertEquals(-1454368523123456L, KuduRelation.timestampToMicros(t3))
+ assertEquals(t3, KuduRelation.microsToTimestamp(-1454368523123456L))
+ }
+
+ val rowCount = 10
+ var sqlContext : SQLContext = _
+ var rows : IndexedSeq[(Int, Int, String)] = _
+ var kuduOptions : Map[String, String] = _
+
+ before {
+ val rowCount = 10
+ rows = insertRows(rowCount)
+
+ sqlContext = new SQLContext(sc)
+
+ kuduOptions = Map(
+ "kudu.table" -> tableName,
+ "kudu.master" -> miniCluster.getMasterAddresses)
+
+ sqlContext.read.options(kuduOptions).kudu.registerTempTable(tableName)
+ }
+
+ test("table creation") {
+ if(kuduContext.tableExists("testcreatetable")) {
+ kuduContext.deleteTable("testcreatetable")
+ }
+
+ val df = sqlContext.read.options(kuduOptions).kudu
+
+ kuduContext.createTable("testcreatetable", df.schema, Seq("key"),
+ new CreateTableOptions().setRangePartitionColumns(List("key").asJava)
+ .setNumReplicas(1))
+
+ // now use new options to refer to the new table name
+ val newOptions: Map[String, String] = Map(
+ "kudu.table" -> "testcreatetable",
+ "kudu.master" -> miniCluster.getMasterAddresses)
+
+ df.write.options(newOptions).mode("append").kudu
+
+ val checkDf = sqlContext.read.options(newOptions).kudu
+
+ assert(checkDf.schema === df.schema)
+
+ assertTrue(kuduContext.tableExists("testcreatetable"))
+ assert(checkDf.count == 10)
+ kuduContext.deleteTable("testcreatetable")
+
+ assertFalse(kuduContext.tableExists("testcreatetable"))
+ }
+
+ test("insertion") {
+ val df = sqlContext.read.options( kuduOptions).kudu
+ val changedDF = df.limit(1).withColumn("key", df("key").plus(100)).withColumn("c2_s", lit("abc"))
+ changedDF.show
+ changedDF.write.options(kuduOptions).mode("append").kudu
+
+ val newDF = sqlContext.read.options(kuduOptions).kudu
+ newDF.show
+ val collected = newDF.filter("key = 100").collect()
+ assertEquals("abc", collected(0).getAs[String]("c2_s"))
+
+ deleteRow(100)
+ }
+
+ test("insertion multiple") {
+ val df = sqlContext.read.options(kuduOptions).kudu
+ val changedDF = df.limit(2).withColumn("key", df("key").plus(100)).withColumn("c2_s", lit("abc"))
+ changedDF.show
+ changedDF.write.options(kuduOptions).mode("append").kudu
+
+ val newDF = sqlContext.read.options(kuduOptions).kudu
+ newDF.show
+ val collected = newDF.filter("key = 100").collect()
+ assertEquals("abc", collected(0).getAs[String]("c2_s"))
+
+ val collectedTwo = newDF.filter("key = 101").collect()
+ assertEquals("abc", collectedTwo(0).getAs[String]("c2_s"))
+
+ deleteRow(100)
+ deleteRow(101)
+ }
+
+ test("update row") {
+ val df = sqlContext.read.options(kuduOptions).kudu
+ val baseDF = df.limit(1) // filter down to just the first row
+ baseDF.show
+ // change the c2 string to abc and update
+ val changedDF = baseDF.withColumn("c2_s", lit("abc"))
+ changedDF.show
+ changedDF.write.options(kuduOptions).mode("overwrite").kudu
+
+ //read the data back
+ val newDF = sqlContext.read.options(kuduOptions).kudu
+ newDF.show
+ val collected = newDF.filter("key = 0").collect()
+ assertEquals("abc", collected(0).getAs[String]("c2_s"))
+
+ //rewrite the original value
+ baseDF.withColumn("c2_s", lit("0")).write.options(kuduOptions)
+ .mode("overwrite").kudu
+ }
+
+ test("out of order selection") {
+ val df = sqlContext.read.options(kuduOptions).kudu.select( "c2_s", "c1_i", "key")
+ val collected = df.collect()
+ assert(collected(0).getString(0).equals("0"))
+
+ }
+
+ test("table scan") {
+ val results = sqlContext.sql(s"SELECT * FROM $tableName").collectAsList()
+ assert(results.size() == rowCount)
+
+ assert(!results.get(0).isNullAt(2))
+ assert(results.get(1).isNullAt(2))
+ }
+
+ test("table scan with projection") {
+ assertEquals(10, sqlContext.sql(s"""SELECT key FROM $tableName""").count())
+ }
+
+ test("table scan with projection and predicate double") {
+ assertEquals(rows.count { case (key, i, s) => i != null && i > 5 },
+ sqlContext.sql(s"""SELECT key, c3_double FROM $tableName where c3_double > "5.0"""").count())
+ }
+
+ test("table scan with projection and predicate long") {
+ assertEquals(rows.count { case (key, i, s) => i != null && i > 5 },
+ sqlContext.sql(s"""SELECT key, c4_long FROM $tableName where c4_long > "5"""").count())
+
+ }
+ test("table scan with projection and predicate bool") {
+ assertEquals(rows.count { case (key, i, s) => i != null && i%2==0 },
+ sqlContext.sql(s"""SELECT key, c5_bool FROM $tableName where c5_bool = true""").count())
+
+ }
+ test("table scan with projection and predicate short") {
+ assertEquals(rows.count { case (key, i, s) => i != null && i > 5},
+ sqlContext.sql(s"""SELECT key, c6_short FROM $tableName where c6_short > 5""").count())
+
+ }
+ test("table scan with projection and predicate float") {
+ assertEquals(rows.count { case (key, i, s) => i != null && i > 5},
+ sqlContext.sql(s"""SELECT key, c7_float FROM $tableName where c7_float > 5""").count())
+
+ }
+
+ test("table scan with projection and predicate ") {
+ assertEquals(rows.count { case (key, i, s) => s != null && s > "5" },
+ sqlContext.sql(s"""SELECT key FROM $tableName where c2_s > "5"""").count())
+
+ assertEquals(rows.count { case (key, i, s) => s != null },
+ sqlContext.sql(s"""SELECT key, c2_s FROM $tableName where c2_s IS NOT NULL""").count())
+ }
+
+
+ test("Test basic SparkSQL") {
+ val results = sqlContext.sql("SELECT * FROM " + tableName).collectAsList()
+ assert(results.size() == rowCount)
+
+ assert(results.get(1).isNullAt(2))
+ assert(!results.get(0).isNullAt(2))
+ }
+
+ test("Test basic SparkSQL projection") {
+ val results = sqlContext.sql("SELECT key FROM " + tableName).collectAsList()
+ assert(results.size() == rowCount)
+ assert(results.get(0).size.equals(1))
+ assert(results.get(0).getInt(0).equals(0))
+ }
+
+ test("Test basic SparkSQL with predicate") {
+ val results = sqlContext.sql("SELECT key FROM " + tableName + " where key=1").collectAsList()
+ assert(results.size() == 1)
+ assert(results.get(0).size.equals(1))
+ assert(results.get(0).getInt(0).equals(1))
+
+ }
+
+ test("Test basic SparkSQL with two predicates") {
+ val results = sqlContext.sql("SELECT key FROM " + tableName + " where key=2 and c2_s='2'").collectAsList()
+ assert(results.size() == 1)
+ assert(results.get(0).size.equals(1))
+ assert(results.get(0).getInt(0).equals(2))
+ }
+
+ test("Test basic SparkSQL with two predicates negative") {
+ val results = sqlContext.sql("SELECT key FROM " + tableName + " where key=1 and c2_s='2'").collectAsList()
+ assert(results.size() == 0)
+ }
+
+ test("Test basic SparkSQL with two predicates including string") {
+ val results = sqlContext.sql("SELECT key FROM " + tableName + " where c2_s='2'").collectAsList()
+ assert(results.size() == 1)
+ assert(results.get(0).size.equals(1))
+ assert(results.get(0).getInt(0).equals(2))
+ }
+
+ test("Test basic SparkSQL with two predicates and projection") {
+ val results = sqlContext.sql("SELECT key, c2_s FROM " + tableName + " where c2_s='2'").collectAsList()
+ assert(results.size() == 1)
+ assert(results.get(0).size.equals(2))
+ assert(results.get(0).getInt(0).equals(2))
+ assert(results.get(0).getString(1).equals("2"))
+ }
+
+ test("Test basic SparkSQL with two predicates greater than") {
+ val results = sqlContext.sql("SELECT key, c2_s FROM " + tableName + " where c2_s>='2'").collectAsList()
+ assert(results.size() == 4)
+ assert(results.get(0).size.equals(2))
+ assert(results.get(0).getInt(0).equals(2))
+ assert(results.get(0).getString(1).equals("2"))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
new file mode 100644
index 0000000..fc2dff5
--- /dev/null
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.spark.kudu
+
+import org.junit.runner.RunWith
+import org.scalatest.FunSuite
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class KuduContextTest extends FunSuite with TestContext {
+ test("Test basic kuduRDD") {
+ val rowCount = 10
+
+ insertRows(rowCount)
+
+ val scanRdd = kuduContext.kuduRDD(sc, "test", Seq("key"))
+
+ val scanList = scanRdd.map(r => r.getInt(0)).collect()
+ assert(scanList.length == rowCount)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
new file mode 100644
index 0000000..1664401
--- /dev/null
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.spark.kudu
+
+import java.util.Date
+
+import com.google.common.collect.ImmutableList
+import org.apache.spark.{SparkConf, SparkContext}
+import org.kududb.ColumnSchema.ColumnSchemaBuilder
+import org.kududb.client.KuduClient.KuduClientBuilder
+import org.kududb.client.MiniKuduCluster.MiniKuduClusterBuilder
+import org.kududb.client.{CreateTableOptions, KuduClient, KuduTable, MiniKuduCluster}
+import org.kududb.{Schema, Type}
+import org.scalatest.{BeforeAndAfterAll, Suite}
+
+import scala.collection.JavaConverters._
+import scala.collection.immutable.IndexedSeq
+
+trait TestContext extends BeforeAndAfterAll { self: Suite =>
+
+ var sc: SparkContext = null
+ var miniCluster: MiniKuduCluster = null
+ var kuduClient: KuduClient = null
+ var table: KuduTable = null
+ var kuduContext: KuduContext = null
+
+ val tableName = "test"
+
+ lazy val schema: Schema = {
+ val columns = ImmutableList.of(
+ new ColumnSchemaBuilder("key", Type.INT32).key(true).build(),
+ new ColumnSchemaBuilder("c1_i", Type.INT32).build(),
+ new ColumnSchemaBuilder("c2_s", Type.STRING).nullable(true).build(),
+ new ColumnSchemaBuilder("c3_double", Type.DOUBLE).build(),
+ new ColumnSchemaBuilder("c4_long", Type.INT64).build(),
+ new ColumnSchemaBuilder("c5_bool", Type.BOOL).build(),
+ new ColumnSchemaBuilder("c6_short", Type.INT16).build(),
+ new ColumnSchemaBuilder("c7_float", Type.FLOAT).build())
+ new Schema(columns)
+ }
+
+ val appID = new Date().toString + math.floor(math.random * 10E4).toLong.toString
+
+ val conf = new SparkConf().
+ setMaster("local[*]").
+ setAppName("test").
+ set("spark.ui.enabled", "false").
+ set("spark.app.id", appID)
+
+ override def beforeAll() {
+ miniCluster = new MiniKuduClusterBuilder()
+ .numMasters(1)
+ .numTservers(1)
+ .build()
+ val envMap = Map[String,String](("Xmx", "512m"))
+
+ sc = new SparkContext(conf)
+
+ kuduClient = new KuduClientBuilder(miniCluster.getMasterAddresses).build()
+ assert(miniCluster.waitForTabletServers(1))
+
+ kuduContext = new KuduContext(miniCluster.getMasterAddresses)
+
+ val tableOptions = new CreateTableOptions().setRangePartitionColumns(List("key").asJava)
+ .setNumReplicas(1)
+ table = kuduClient.createTable(tableName, schema, tableOptions)
+ }
+
+ override def afterAll() {
+ if (kuduClient != null) kuduClient.shutdown()
+ if (miniCluster != null) miniCluster.shutdown()
+ if (sc != null) sc.stop()
+ }
+
+ def deleteRow(key: Int): Unit = {
+ val kuduSession = kuduClient.newSession()
+ val delete = table.newDelete()
+ delete.getRow.addInt(0, key)
+ kuduSession.apply(delete)
+ }
+
+ def insertRows(rowCount: Integer): IndexedSeq[(Int, Int, String)] = {
+ val kuduSession = kuduClient.newSession()
+
+ val rows = Range(0, rowCount).map { i =>
+ val insert = table.newInsert
+ val row = insert.getRow
+ row.addInt(0, i)
+ row.addInt(1, i)
+ row.addDouble(3, i.toDouble)
+ row.addLong(4, i.toLong)
+ row.addBoolean(5, i%2==1)
+ row.addShort(6, i.toShort)
+ row.addFloat(7, i.toFloat)
+
+ // Sprinkling some nulls so that queries see them.
+ val s = if (i % 2 == 0) {
+ row.addString(2, i.toString)
+ i.toString
+ } else {
+ row.setNull(2)
+ null
+ }
+
+ kuduSession.apply(insert)
+ (i, i, s)
+ }
+ rows
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/DefaultSourceTest.scala
deleted file mode 100644
index ec929d7..0000000
--- a/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/DefaultSourceTest.scala
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.spark.kudu
-
-import java.sql.Timestamp
-import java.text.SimpleDateFormat
-import java.util.TimeZone
-
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.functions._
-import org.junit.Assert._
-import org.junit.runner.RunWith
-import org.kududb.client.CreateTableOptions
-import org.scalatest.junit.JUnitRunner
-import org.scalatest.{BeforeAndAfter, FunSuite}
-
-import scala.collection.JavaConverters._
-import scala.collection.immutable.IndexedSeq
-
-@RunWith(classOf[JUnitRunner])
-class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter {
-
- test("timestamp conversion") {
- val epoch = new Timestamp(0)
- assertEquals(0, KuduRelation.timestampToMicros(epoch))
- assertEquals(epoch, KuduRelation.microsToTimestamp(0))
-
- val t1 = new Timestamp(0)
- t1.setNanos(123456000)
- assertEquals(123456, KuduRelation.timestampToMicros(t1))
- assertEquals(t1, KuduRelation.microsToTimestamp(123456))
-
- val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")
- iso8601.setTimeZone(TimeZone.getTimeZone("UTC"))
-
- val t3 = new Timestamp(iso8601.parse("1923-12-01T00:44:36.876").getTime)
- t3.setNanos(876544000)
- assertEquals(-1454368523123456L, KuduRelation.timestampToMicros(t3))
- assertEquals(t3, KuduRelation.microsToTimestamp(-1454368523123456L))
- }
-
- val rowCount = 10
- var sqlContext : SQLContext = _
- var rows : IndexedSeq[(Int, Int, String)] = _
- var kuduOptions : Map[String, String] = _
-
- before {
- val rowCount = 10
- rows = insertRows(rowCount)
-
- sqlContext = new SQLContext(sc)
-
- kuduOptions = Map(
- "kudu.table" -> tableName,
- "kudu.master" -> miniCluster.getMasterAddresses)
-
- sqlContext.read.options(kuduOptions).kudu.registerTempTable(tableName)
- }
-
- test("table creation") {
- if(kuduContext.tableExists("testcreatetable")) {
- kuduContext.deleteTable("testcreatetable")
- }
-
- val df = sqlContext.read.options(kuduOptions).kudu
-
- kuduContext.createTable("testcreatetable", df.schema, Seq("key"),
- new CreateTableOptions().setRangePartitionColumns(List("key").asJava)
- .setNumReplicas(1))
-
- // now use new options to refer to the new table name
- val newOptions: Map[String, String] = Map(
- "kudu.table" -> "testcreatetable",
- "kudu.master" -> miniCluster.getMasterAddresses)
-
- df.write.options(newOptions).mode("append").kudu
-
- val checkDf = sqlContext.read.options(newOptions).kudu
-
- assert(checkDf.schema === df.schema)
-
- assertTrue(kuduContext.tableExists("testcreatetable"))
- assert(checkDf.count == 10)
- kuduContext.deleteTable("testcreatetable")
-
- assertFalse(kuduContext.tableExists("testcreatetable"))
- }
-
- test("insertion") {
- val df = sqlContext.read.options( kuduOptions).kudu
- val changedDF = df.limit(1).withColumn("key", df("key").plus(100)).withColumn("c2_s", lit("abc"))
- changedDF.show
- changedDF.write.options(kuduOptions).mode("append").kudu
-
- val newDF = sqlContext.read.options(kuduOptions).kudu
- newDF.show
- val collected = newDF.filter("key = 100").collect()
- assertEquals("abc", collected(0).getAs[String]("c2_s"))
-
- deleteRow(100)
- }
-
- test("insertion multiple") {
- val df = sqlContext.read.options(kuduOptions).kudu
- val changedDF = df.limit(2).withColumn("key", df("key").plus(100)).withColumn("c2_s", lit("abc"))
- changedDF.show
- changedDF.write.options(kuduOptions).mode("append").kudu
-
- val newDF = sqlContext.read.options(kuduOptions).kudu
- newDF.show
- val collected = newDF.filter("key = 100").collect()
- assertEquals("abc", collected(0).getAs[String]("c2_s"))
-
- val collectedTwo = newDF.filter("key = 101").collect()
- assertEquals("abc", collectedTwo(0).getAs[String]("c2_s"))
-
- deleteRow(100)
- deleteRow(101)
- }
-
- test("update row") {
- val df = sqlContext.read.options(kuduOptions).kudu
- val baseDF = df.limit(1) // filter down to just the first row
- baseDF.show
- // change the c2 string to abc and update
- val changedDF = baseDF.withColumn("c2_s", lit("abc"))
- changedDF.show
- changedDF.write.options(kuduOptions).mode("overwrite").kudu
-
- //read the data back
- val newDF = sqlContext.read.options(kuduOptions).kudu
- newDF.show
- val collected = newDF.filter("key = 0").collect()
- assertEquals("abc", collected(0).getAs[String]("c2_s"))
-
- //rewrite the original value
- baseDF.withColumn("c2_s", lit("0")).write.options(kuduOptions)
- .mode("overwrite").kudu
- }
-
- test("out of order selection") {
- val df = sqlContext.read.options(kuduOptions).kudu.select( "c2_s", "c1_i", "key")
- val collected = df.collect()
- assert(collected(0).getString(0).equals("0"))
-
- }
-
- test("table scan") {
- val results = sqlContext.sql(s"SELECT * FROM $tableName").collectAsList()
- assert(results.size() == rowCount)
-
- assert(!results.get(0).isNullAt(2))
- assert(results.get(1).isNullAt(2))
- }
-
- test("table scan with projection") {
- assertEquals(10, sqlContext.sql(s"""SELECT key FROM $tableName""").count())
- }
-
- test("table scan with projection and predicate double") {
- assertEquals(rows.count { case (key, i, s) => i != null && i > 5 },
- sqlContext.sql(s"""SELECT key, c3_double FROM $tableName where c3_double > "5.0"""").count())
- }
-
- test("table scan with projection and predicate long") {
- assertEquals(rows.count { case (key, i, s) => i != null && i > 5 },
- sqlContext.sql(s"""SELECT key, c4_long FROM $tableName where c4_long > "5"""").count())
-
- }
- test("table scan with projection and predicate bool") {
- assertEquals(rows.count { case (key, i, s) => i != null && i%2==0 },
- sqlContext.sql(s"""SELECT key, c5_bool FROM $tableName where c5_bool = true""").count())
-
- }
- test("table scan with projection and predicate short") {
- assertEquals(rows.count { case (key, i, s) => i != null && i > 5},
- sqlContext.sql(s"""SELECT key, c6_short FROM $tableName where c6_short > 5""").count())
-
- }
- test("table scan with projection and predicate float") {
- assertEquals(rows.count { case (key, i, s) => i != null && i > 5},
- sqlContext.sql(s"""SELECT key, c7_float FROM $tableName where c7_float > 5""").count())
-
- }
-
- test("table scan with projection and predicate ") {
- assertEquals(rows.count { case (key, i, s) => s != null && s > "5" },
- sqlContext.sql(s"""SELECT key FROM $tableName where c2_s > "5"""").count())
-
- assertEquals(rows.count { case (key, i, s) => s != null },
- sqlContext.sql(s"""SELECT key, c2_s FROM $tableName where c2_s IS NOT NULL""").count())
- }
-
-
- test("Test basic SparkSQL") {
- val results = sqlContext.sql("SELECT * FROM " + tableName).collectAsList()
- assert(results.size() == rowCount)
-
- assert(results.get(1).isNullAt(2))
- assert(!results.get(0).isNullAt(2))
- }
-
- test("Test basic SparkSQL projection") {
- val results = sqlContext.sql("SELECT key FROM " + tableName).collectAsList()
- assert(results.size() == rowCount)
- assert(results.get(0).size.equals(1))
- assert(results.get(0).getInt(0).equals(0))
- }
-
- test("Test basic SparkSQL with predicate") {
- val results = sqlContext.sql("SELECT key FROM " + tableName + " where key=1").collectAsList()
- assert(results.size() == 1)
- assert(results.get(0).size.equals(1))
- assert(results.get(0).getInt(0).equals(1))
-
- }
-
- test("Test basic SparkSQL with two predicates") {
- val results = sqlContext.sql("SELECT key FROM " + tableName + " where key=2 and c2_s='2'").collectAsList()
- assert(results.size() == 1)
- assert(results.get(0).size.equals(1))
- assert(results.get(0).getInt(0).equals(2))
- }
-
- test("Test basic SparkSQL with two predicates negative") {
- val results = sqlContext.sql("SELECT key FROM " + tableName + " where key=1 and c2_s='2'").collectAsList()
- assert(results.size() == 0)
- }
-
- test("Test basic SparkSQL with two predicates including string") {
- val results = sqlContext.sql("SELECT key FROM " + tableName + " where c2_s='2'").collectAsList()
- assert(results.size() == 1)
- assert(results.get(0).size.equals(1))
- assert(results.get(0).getInt(0).equals(2))
- }
-
- test("Test basic SparkSQL with two predicates and projection") {
- val results = sqlContext.sql("SELECT key, c2_s FROM " + tableName + " where c2_s='2'").collectAsList()
- assert(results.size() == 1)
- assert(results.get(0).size.equals(2))
- assert(results.get(0).getInt(0).equals(2))
- assert(results.get(0).getString(1).equals("2"))
- }
-
- test("Test basic SparkSQL with two predicates greater than") {
- val results = sqlContext.sql("SELECT key, c2_s FROM " + tableName + " where c2_s>='2'").collectAsList()
- assert(results.size() == 4)
- assert(results.get(0).size.equals(2))
- assert(results.get(0).getInt(0).equals(2))
- assert(results.get(0).getString(1).equals("2"))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/KuduContextTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/KuduContextTest.scala b/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/KuduContextTest.scala
deleted file mode 100644
index fc2dff5..0000000
--- a/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/KuduContextTest.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.spark.kudu
-
-import org.junit.runner.RunWith
-import org.scalatest.FunSuite
-import org.scalatest.junit.JUnitRunner
-
-@RunWith(classOf[JUnitRunner])
-class KuduContextTest extends FunSuite with TestContext {
- test("Test basic kuduRDD") {
- val rowCount = 10
-
- insertRows(rowCount)
-
- val scanRdd = kuduContext.kuduRDD(sc, "test", Seq("key"))
-
- val scanList = scanRdd.map(r => r.getInt(0)).collect()
- assert(scanList.length == rowCount)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/TestContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/TestContext.scala b/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/TestContext.scala
deleted file mode 100644
index 1664401..0000000
--- a/java/kudu-spark/src/test/scala/org/kududb/spark/kudu/TestContext.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.spark.kudu
-
-import java.util.Date
-
-import com.google.common.collect.ImmutableList
-import org.apache.spark.{SparkConf, SparkContext}
-import org.kududb.ColumnSchema.ColumnSchemaBuilder
-import org.kududb.client.KuduClient.KuduClientBuilder
-import org.kududb.client.MiniKuduCluster.MiniKuduClusterBuilder
-import org.kududb.client.{CreateTableOptions, KuduClient, KuduTable, MiniKuduCluster}
-import org.kududb.{Schema, Type}
-import org.scalatest.{BeforeAndAfterAll, Suite}
-
-import scala.collection.JavaConverters._
-import scala.collection.immutable.IndexedSeq
-
-trait TestContext extends BeforeAndAfterAll { self: Suite =>
-
- var sc: SparkContext = null
- var miniCluster: MiniKuduCluster = null
- var kuduClient: KuduClient = null
- var table: KuduTable = null
- var kuduContext: KuduContext = null
-
- val tableName = "test"
-
- lazy val schema: Schema = {
- val columns = ImmutableList.of(
- new ColumnSchemaBuilder("key", Type.INT32).key(true).build(),
- new ColumnSchemaBuilder("c1_i", Type.INT32).build(),
- new ColumnSchemaBuilder("c2_s", Type.STRING).nullable(true).build(),
- new ColumnSchemaBuilder("c3_double", Type.DOUBLE).build(),
- new ColumnSchemaBuilder("c4_long", Type.INT64).build(),
- new ColumnSchemaBuilder("c5_bool", Type.BOOL).build(),
- new ColumnSchemaBuilder("c6_short", Type.INT16).build(),
- new ColumnSchemaBuilder("c7_float", Type.FLOAT).build())
- new Schema(columns)
- }
-
- val appID = new Date().toString + math.floor(math.random * 10E4).toLong.toString
-
- val conf = new SparkConf().
- setMaster("local[*]").
- setAppName("test").
- set("spark.ui.enabled", "false").
- set("spark.app.id", appID)
-
- override def beforeAll() {
- miniCluster = new MiniKuduClusterBuilder()
- .numMasters(1)
- .numTservers(1)
- .build()
- val envMap = Map[String,String](("Xmx", "512m"))
-
- sc = new SparkContext(conf)
-
- kuduClient = new KuduClientBuilder(miniCluster.getMasterAddresses).build()
- assert(miniCluster.waitForTabletServers(1))
-
- kuduContext = new KuduContext(miniCluster.getMasterAddresses)
-
- val tableOptions = new CreateTableOptions().setRangePartitionColumns(List("key").asJava)
- .setNumReplicas(1)
- table = kuduClient.createTable(tableName, schema, tableOptions)
- }
-
- override def afterAll() {
- if (kuduClient != null) kuduClient.shutdown()
- if (miniCluster != null) miniCluster.shutdown()
- if (sc != null) sc.stop()
- }
-
- def deleteRow(key: Int): Unit = {
- val kuduSession = kuduClient.newSession()
- val delete = table.newDelete()
- delete.getRow.addInt(0, key)
- kuduSession.apply(delete)
- }
-
- def insertRows(rowCount: Integer): IndexedSeq[(Int, Int, String)] = {
- val kuduSession = kuduClient.newSession()
-
- val rows = Range(0, rowCount).map { i =>
- val insert = table.newInsert
- val row = insert.getRow
- row.addInt(0, i)
- row.addInt(1, i)
- row.addDouble(3, i.toDouble)
- row.addLong(4, i.toLong)
- row.addBoolean(5, i%2==1)
- row.addShort(6, i.toShort)
- row.addFloat(7, i.toFloat)
-
- // Sprinkling some nulls so that queries see them.
- val s = if (i % 2 == 0) {
- row.addString(2, i.toString)
- i.toString
- } else {
- row.setNull(2)
- null
- }
-
- kuduSession.apply(insert)
- (i, i, s)
- }
- rows
- }
-}
\ No newline at end of file
[34/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 2)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
index 392bed9..2b733c7 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java
index fce7ddc..2e82586 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.util;
+package org.apache.kudu.util;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java
index 051107c..f095f6c 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.util;
+package org.apache.kudu.util;
import com.google.common.primitives.UnsignedLongs;
import com.sangupta.murmur.Murmur2;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java
index 5c003ae..cc88a3f 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.util;
+package org.apache.kudu.util;
import com.google.common.net.HostAndPort;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-csd/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-csd/pom.xml b/java/kudu-csd/pom.xml
index dca8963..edfd24c 100644
--- a/java/kudu-csd/pom.xml
+++ b/java/kudu-csd/pom.xml
@@ -17,7 +17,7 @@ limitations under the License.
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-flume-sink/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/pom.xml b/java/kudu-flume-sink/pom.xml
index 3ef2a1a..890449e 100644
--- a/java/kudu-flume-sink/pom.xml
+++ b/java/kudu-flume-sink/pom.xml
@@ -14,7 +14,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>kudu-parent</artifactId>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<version>0.10.0-SNAPSHOT</version>
</parent>
<artifactId>kudu-flume-sink</artifactId>
@@ -49,7 +49,7 @@
<dependencies>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${project.version}</version>
</dependency>
@@ -76,7 +76,7 @@
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java
index 7166300..95f63b1 100644
--- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java
@@ -16,15 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.kududb.flume.sink;
+package org.apache.kudu.flume.sink;
import org.apache.flume.Event;
import org.apache.flume.conf.Configurable;
import org.apache.flume.conf.ConfigurableComponent;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.Operation;
import java.util.List;
@@ -47,7 +47,7 @@ public interface KuduEventProducer extends Configurable, ConfigurableComponent {
/**
* Get the operations that should be written out to Kudu as a result of this
* event. This list is written to Kudu using the Kudu client API.
- * @return List of {@link org.kududb.client.Operation} which
+ * @return List of {@link org.apache.kudu.client.Operation} which
* are written as such to Kudu
*/
List<Operation> getOperations();
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
index 080cda2..3066323 100644
--- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.kududb.flume.sink;
+package org.apache.kudu.flume.sink;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -30,15 +30,15 @@ import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
import org.apache.flume.instrumentation.SinkCounter;
import org.apache.flume.sink.AbstractSink;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduSession;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
-import org.kududb.client.OperationResponse;
-import org.kududb.client.SessionConfiguration;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.AsyncKuduClient;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.Operation;
+import org.apache.kudu.client.OperationResponse;
+import org.apache.kudu.client.SessionConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,13 +52,13 @@ import java.util.List;
* <table cellpadding=3 cellspacing=0 border=1>
* <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr>
* <tr><td>channel</td><td></td><td>Yes</td><td>The name of the Flume channel to read from.</td></tr>
- * <tr><td>type</td><td></td><td>Yes</td><td>Component name. Must be {@code org.kududb.flume.sink.KuduSink}</td></tr>
+ * <tr><td>type</td><td></td><td>Yes</td><td>Component name. Must be {@code org.apache.kudu.flume.sink.KuduSink}</td></tr>
* <tr><td>masterAddresses</td><td></td><td>Yes</td><td>Comma-separated list of "host:port" pairs of the Kudu master servers. The port is optional.</td></tr>
* <tr><td>tableName</td><td></td><td>Yes</td><td>The name of the Kudu table to write to.</td></tr>
* <tr><td>batchSize</td><td>100</td><td>No</td><td>The maximum number of events the sink will attempt to take from the channel per transaction.</td></tr>
* <tr><td>ignoreDuplicateRows</td><td>true</td><td>No</td><td>Whether to ignore errors indicating that we attempted to insert duplicate rows into Kudu.</td></tr>
* <tr><td>timeoutMillis</td><td>10000</td><td>No</td><td>Timeout period for Kudu write operations, in milliseconds.</td></tr>
- * <tr><td>producer</td><td>{@link org.kududb.flume.sink.SimpleKuduEventProducer}</td><td>No</td><td>The fully qualified class name of the {@link KuduEventProducer} the sink should use.</td></tr>
+ * <tr><td>producer</td><td>{@link org.apache.kudu.flume.sink.SimpleKuduEventProducer}</td><td>No</td><td>The fully qualified class name of the {@link KuduEventProducer} the sink should use.</td></tr>
* <tr><td>producer.*</td><td></td><td>(Varies by event producer)</td><td>Configuration properties to pass to the event producer implementation.</td></tr>
* </table>
*
@@ -79,7 +79,7 @@ public class KuduSink extends AbstractSink implements Configurable {
private static final Long DEFAULT_TIMEOUT_MILLIS =
AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
private static final String DEFAULT_KUDU_EVENT_PRODUCER =
- "org.kududb.flume.sink.SimpleKuduEventProducer";
+ "org.apache.kudu.flume.sink.SimpleKuduEventProducer";
private static final boolean DEFAULT_IGNORE_DUPLICATE_ROWS = true;
private String masterAddresses;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java
index 6486137..7da815e 100644
--- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java
@@ -17,10 +17,10 @@
* under the License.
*/
-package org.kududb.flume.sink;
+package org.apache.kudu.flume.sink;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Constants used for configuration of KuduSink
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java
index b5be054..9eb07c4 100644
--- a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java
@@ -17,16 +17,16 @@
* under the License.
*/
-package org.kududb.flume.sink;
+package org.apache.kudu.flume.sink;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.FlumeException;
import org.apache.flume.conf.ComponentConfiguration;
-import org.kududb.client.Insert;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
-import org.kududb.client.PartialRow;
+import org.apache.kudu.client.Insert;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.Operation;
+import org.apache.kudu.client.PartialRow;
import java.util.Collections;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
index d732b71..7c426cb 100644
--- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
+++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.kududb.flume.sink;
+package org.apache.kudu.flume.sink;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
@@ -34,12 +34,12 @@ import org.apache.flume.conf.Configurables;
import org.apache.flume.event.EventBuilder;
import org.junit.Assert;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.client.CreateTableOptions;
-import org.kududb.client.KuduTable;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.BaseKuduTest;
+import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.KuduTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/pom.xml b/java/kudu-mapreduce/pom.xml
index 02293c7..da4c024 100644
--- a/java/kudu-mapreduce/pom.xml
+++ b/java/kudu-mapreduce/pom.xml
@@ -21,7 +21,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
</parent>
@@ -31,12 +31,12 @@
<dependencies>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java
index 05c18f2..385752e 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.AsyncKuduClient;
import org.apache.hadoop.conf.Configuration;
-import org.kududb.client.KuduClient;
+import org.apache.kudu.client.KuduClient;
/**
* Utility class that manages common configurations to all MR jobs. For example,
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
index 57593db..3dbfa04 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
index 25235cb..0772833 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import com.google.common.base.Objects;
import com.google.common.base.Splitter;
@@ -44,20 +44,20 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.net.DNS;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
-import org.kududb.client.Bytes;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduPredicate;
-import org.kududb.client.KuduScanner;
-import org.kududb.client.KuduTable;
-import org.kududb.client.LocatedTablet;
-import org.kududb.client.RowResult;
-import org.kududb.client.RowResultIterator;
-import org.kududb.client.KuduScanToken;
+import org.apache.kudu.Common;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.AsyncKuduClient;
+import org.apache.kudu.client.Bytes;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduPredicate;
+import org.apache.kudu.client.KuduScanner;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.LocatedTablet;
+import org.apache.kudu.client.RowResult;
+import org.apache.kudu.client.RowResultIterator;
+import org.apache.kudu.client.KuduScanToken;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
index 0b919d9..f0e5b62 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -38,13 +38,13 @@ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.hadoop.util.StringUtils;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
-import org.kududb.client.ColumnRangePredicate;
-import org.kududb.client.KuduPredicate;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.AsyncKuduClient;
+import org.apache.kudu.client.ColumnRangePredicate;
+import org.apache.kudu.client.KuduPredicate;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.Operation;
/**
* Utility class to setup MR jobs that use Kudu as an input and/or output.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java
index 8af750b..c84c3d5 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java
index e80b73f..69b076d 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.*;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.*;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java
index 7cf3ada..e319fe4 100644
--- a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Operation;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.Operation;
import org.apache.hadoop.mapreduce.Reducer;
@InterfaceAudience.Public
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
index 1e2cb41..0db2c9a 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
index 3d04043..2f4b05e 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
@@ -25,9 +25,9 @@ import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.client.KuduPredicate;
-import org.kududb.client.RowResult;
+import org.apache.kudu.client.BaseKuduTest;
+import org.apache.kudu.client.KuduPredicate;
+import org.apache.kudu.client.RowResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
index ff4d81a..818f33f 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import org.kududb.Schema;
-import org.kududb.client.*;
+import org.apache.kudu.Schema;
+import org.apache.kudu.client.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
index 86452ed..57193b3 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
@@ -14,9 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
-import org.kududb.client.*;
+import org.apache.kudu.client.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
index dff2400..d3f1dbf 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
@@ -14,9 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
-import org.kududb.client.*;
+import org.apache.kudu.client.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
index 3801a0c..acf155f 100644
--- a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce;
+package org.apache.kudu.mapreduce;
import org.apache.commons.logging.LogFactory;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-spark/pom.xml b/java/kudu-spark/pom.xml
index 9909d93..2ac92a9 100644
--- a/java/kudu-spark/pom.xml
+++ b/java/kudu-spark/pom.xml
@@ -15,7 +15,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
</parent>
@@ -57,19 +57,19 @@
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>interface-annotations</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index 95145d5..138eff1 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.kududb.spark.kudu
+package org.apache.kudu.spark.kudu
import java.sql.Timestamp
@@ -23,11 +23,11 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types._
import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
-import org.kududb.Type
-import org.kududb.annotations.InterfaceStability
-import org.kududb.client._
-import org.kududb.client.KuduPredicate.ComparisonOp
-import org.kududb.client.SessionConfiguration.FlushMode
+import org.apache.kudu.Type
+import org.apache.kudu.annotations.InterfaceStability
+import org.apache.kudu.client._
+import org.apache.kudu.client.KuduPredicate.ComparisonOp
+import org.apache.kudu.client.SessionConfiguration.FlushMode
import org.apache.spark.sql.SaveMode._
import scala.collection.JavaConverters._
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
index 167ee13..3a42365 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.kududb.spark.kudu
+package org.apache.kudu.spark.kudu
import java.util
import org.apache.hadoop.util.ShutdownHookManager
@@ -23,10 +23,10 @@ import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spark.sql.types.{StructType, DataType, DataTypes}
-import org.kududb.{ColumnSchema, Schema, Type}
-import org.kududb.annotations.InterfaceStability
-import org.kududb.client.SessionConfiguration.FlushMode
-import org.kududb.client._
+import org.apache.kudu.{ColumnSchema, Schema, Type}
+import org.apache.kudu.annotations.InterfaceStability
+import org.apache.kudu.client.SessionConfiguration.FlushMode
+import org.apache.kudu.client._
import scala.collection.mutable
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
index 5395d5a..b3b69ec 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.spark.kudu
+package org.apache.kudu.spark.kudu
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
import org.apache.spark.{Partition, SparkContext, TaskContext}
-import org.kududb.client._
-import org.kududb.{Type, client}
+import org.apache.kudu.client._
+import org.apache.kudu.{Type, client}
import scala.collection.JavaConverters._
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
index 4203e31..ffc45ed 100755
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.spark
+package org.apache.kudu.spark
import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter}
@@ -25,7 +25,7 @@ package object kudu {
* the DataFrameReader.
*/
implicit class KuduDataFrameReader(reader: DataFrameReader) {
- def kudu: DataFrame = reader.format("org.kududb.spark.kudu").load
+ def kudu: DataFrame = reader.format("org.apache.kudu.spark.kudu").load
}
/**
@@ -33,6 +33,6 @@ package object kudu {
* the DataFileWriter
*/
implicit class KuduDataFrameWriter(writer: DataFrameWriter) {
- def kudu = writer.format("org.kududb.spark.kudu").save
+ def kudu = writer.format("org.apache.kudu.spark.kudu").save
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/resources/log4j.properties b/java/kudu-spark/src/test/resources/log4j.properties
index 4c1b502..535996c 100644
--- a/java/kudu-spark/src/test/resources/log4j.properties
+++ b/java/kudu-spark/src/test/resources/log4j.properties
@@ -20,4 +20,4 @@ log4j.appender.out = org.apache.log4j.ConsoleAppender
log4j.appender.out.layout = org.apache.log4j.PatternLayout
log4j.appender.out.layout.ConversionPattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
-log4j.logger.org.kududb = INFO
\ No newline at end of file
+log4j.logger.org.apache.kudu = INFO
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
index ec929d7..8e5d68e 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.spark.kudu
+package org.apache.kudu.spark.kudu
import java.sql.Timestamp
import java.text.SimpleDateFormat
@@ -24,7 +24,7 @@ import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.functions._
import org.junit.Assert._
import org.junit.runner.RunWith
-import org.kududb.client.CreateTableOptions
+import org.apache.kudu.client.CreateTableOptions
import org.scalatest.junit.JUnitRunner
import org.scalatest.{BeforeAndAfter, FunSuite}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
index fc2dff5..f020919 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.spark.kudu
+package org.apache.kudu.spark.kudu
import org.junit.runner.RunWith
import org.scalatest.FunSuite
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
index 1664401..f4af644 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
@@ -14,17 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.spark.kudu
+package org.apache.kudu.spark.kudu
import java.util.Date
import com.google.common.collect.ImmutableList
import org.apache.spark.{SparkConf, SparkContext}
-import org.kududb.ColumnSchema.ColumnSchemaBuilder
-import org.kududb.client.KuduClient.KuduClientBuilder
-import org.kududb.client.MiniKuduCluster.MiniKuduClusterBuilder
-import org.kududb.client.{CreateTableOptions, KuduClient, KuduTable, MiniKuduCluster}
-import org.kududb.{Schema, Type}
+import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
+import org.apache.kudu.client.KuduClient.KuduClientBuilder
+import org.apache.kudu.client.MiniKuduCluster.MiniKuduClusterBuilder
+import org.apache.kudu.client.{CreateTableOptions, KuduClient, KuduTable, MiniKuduCluster}
+import org.apache.kudu.{Schema, Type}
import org.scalatest.{BeforeAndAfterAll, Suite}
import scala.collection.JavaConverters._
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 17e165e..1ac44a9 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -22,7 +22,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
<packaging>pom</packaging>
@@ -208,23 +208,23 @@
<maxmemory>2048m</maxmemory>
<notimestamp>true</notimestamp>
<doclet>
- org.kududb.annotations.tools.IncludePublicAnnotationsStandardDoclet
+ org.apache.kudu.annotations.tools.IncludePublicAnnotationsStandardDoclet
</doclet>
<docletArtifact>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>interface-annotations</artifactId>
<version>${project.version}</version>
</docletArtifact>
<name>User API</name>
<description>The Kudu Application Programmer's API</description>
<excludePackageNames>
- com.google:org.kududb.cfile:org.kududb.consensus:org.kududb.log:org.kududb.master:org.kududb.rpc:org.kududb.server:org.kududb.tablet:org.kududb.tserver
+ com.google:org.apache.kudu.cfile:org.apache.kudu.consensus:org.apache.kudu.log:org.apache.kudu.master:org.apache.kudu.rpc:org.apache.kudu.server:org.apache.kudu.tablet:org.apache.kudu.tserver
</excludePackageNames>
<includeDependencySources>false</includeDependencySources>
<dependencySourceIncludes>
<dependencySourceInclude>
- org.kududb:interface-annotations
+ org.apache.kudu:interface-annotations
</dependencySourceInclude>
</dependencySourceIncludes>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/cfile/cfile.proto
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile.proto b/src/kudu/cfile/cfile.proto
index d9a0776..34794b0 100644
--- a/src/kudu/cfile/cfile.proto
+++ b/src/kudu/cfile/cfile.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.cfile;
-option java_package = "org.kududb.cfile";
+option java_package = "org.apache.kudu.cfile";
import "kudu/common/common.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/client/client.proto
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.proto b/src/kudu/client/client.proto
index 8a2bdd0..fcff805 100644
--- a/src/kudu/client/client.proto
+++ b/src/kudu/client/client.proto
@@ -19,7 +19,7 @@
package kudu.client;
-option java_package = "org.kududb.client";
+option java_package = "org.apache.kudu.client";
import "kudu/common/common.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/common/common.proto
----------------------------------------------------------------------
diff --git a/src/kudu/common/common.proto b/src/kudu/common/common.proto
index a3505bb..e5ce03c 100644
--- a/src/kudu/common/common.proto
+++ b/src/kudu/common/common.proto
@@ -25,7 +25,7 @@
// etc, as appropriate.
package kudu;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
// If you add a new type keep in mind to add it to the end
// or update AddMapping() functions like the one in key_encoder.cc
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/common/wire_protocol.proto
----------------------------------------------------------------------
diff --git a/src/kudu/common/wire_protocol.proto b/src/kudu/common/wire_protocol.proto
index af957cf..387439d 100644
--- a/src/kudu/common/wire_protocol.proto
+++ b/src/kudu/common/wire_protocol.proto
@@ -22,7 +22,7 @@
// place such as common/common.proto or within cfile/, server/, etc.
package kudu;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
import "kudu/common/common.proto";
import "kudu/consensus/metadata.proto";
@@ -32,7 +32,7 @@ import "kudu/consensus/metadata.proto";
// should have this (or a more complex error result) as an optional field
// in its response.
//
-// This maps to kudu::Status in C++ and org.kududb.Status in Java.
+// This maps to kudu::Status in C++ and org.apache.kudu.Status in Java.
message AppStatusPB {
enum ErrorCode {
UNKNOWN_ERROR = 999;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/consensus/consensus.proto
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/consensus.proto b/src/kudu/consensus/consensus.proto
index d8fb1d3..0091080 100644
--- a/src/kudu/consensus/consensus.proto
+++ b/src/kudu/consensus/consensus.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.consensus;
-option java_package = "org.kududb.consensus";
+option java_package = "org.apache.kudu.consensus";
import "kudu/common/common.proto";
import "kudu/common/wire_protocol.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/consensus/log.proto
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/log.proto b/src/kudu/consensus/log.proto
index 20c38d6..5a21bcb 100644
--- a/src/kudu/consensus/log.proto
+++ b/src/kudu/consensus/log.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.log;
-option java_package = "org.kududb.log";
+option java_package = "org.apache.kudu.log";
import "kudu/common/common.proto";
import "kudu/consensus/consensus.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/consensus/metadata.proto
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/metadata.proto b/src/kudu/consensus/metadata.proto
index 7df9d27..a326039 100644
--- a/src/kudu/consensus/metadata.proto
+++ b/src/kudu/consensus/metadata.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.consensus;
-option java_package = "org.kududb.consensus";
+option java_package = "org.apache.kudu.consensus";
import "kudu/common/common.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/consensus/opid.proto
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/opid.proto b/src/kudu/consensus/opid.proto
index 4433533..b4eedc2 100644
--- a/src/kudu/consensus/opid.proto
+++ b/src/kudu/consensus/opid.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.consensus;
-option java_package = "org.kududb.consensus";
+option java_package = "org.apache.kudu.consensus";
// An id for a generic state machine operation. Composed of the leaders' term
// plus the index of the operation in that term, e.g., the <index>th operation
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/fs/fs.proto
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs.proto b/src/kudu/fs/fs.proto
index bf1cfca..4c88507 100644
--- a/src/kudu/fs/fs.proto
+++ b/src/kudu/fs/fs.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
// ============================================================================
// Local file system metadata
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/master/master.proto
----------------------------------------------------------------------
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index 402ab2b..a5292e1 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.master;
-option java_package = "org.kududb.master";
+option java_package = "org.apache.kudu.master";
import "kudu/common/common.proto";
import "kudu/common/wire_protocol.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/rpc/rpc_header.proto
----------------------------------------------------------------------
diff --git a/src/kudu/rpc/rpc_header.proto b/src/kudu/rpc/rpc_header.proto
index f394fbe..0292c74 100644
--- a/src/kudu/rpc/rpc_header.proto
+++ b/src/kudu/rpc/rpc_header.proto
@@ -19,7 +19,7 @@ option optimize_for = SPEED;
package kudu.rpc;
-option java_package = "org.kududb.rpc";
+option java_package = "org.apache.kudu.rpc";
import "google/protobuf/descriptor.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/rpc/rpc_introspection.proto
----------------------------------------------------------------------
diff --git a/src/kudu/rpc/rpc_introspection.proto b/src/kudu/rpc/rpc_introspection.proto
index 5a223e6..7718569 100644
--- a/src/kudu/rpc/rpc_introspection.proto
+++ b/src/kudu/rpc/rpc_introspection.proto
@@ -20,7 +20,7 @@
package kudu.rpc;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
import "kudu/rpc/rpc_header.proto";
@@ -86,4 +86,4 @@ message DumpRpczStoreRequestPB {
}
message DumpRpczStoreResponsePB {
repeated RpczMethodPB methods = 1;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/server/server_base.proto
----------------------------------------------------------------------
diff --git a/src/kudu/server/server_base.proto b/src/kudu/server/server_base.proto
index cf69a2d..6ad9f48 100644
--- a/src/kudu/server/server_base.proto
+++ b/src/kudu/server/server_base.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.server;
-option java_package = "org.kududb.server";
+option java_package = "org.apache.kudu.server";
import "kudu/common/common.proto";
import "kudu/common/wire_protocol.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/tablet/metadata.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/metadata.proto b/src/kudu/tablet/metadata.proto
index 8e71d94..eb80611 100644
--- a/src/kudu/tablet/metadata.proto
+++ b/src/kudu/tablet/metadata.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.tablet;
-option java_package = "org.kududb.tablet";
+option java_package = "org.apache.kudu.tablet";
import "kudu/common/common.proto";
import "kudu/consensus/opid.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/tablet/tablet.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet.proto b/src/kudu/tablet/tablet.proto
index 18ebe50..468add3 100644
--- a/src/kudu/tablet/tablet.proto
+++ b/src/kudu/tablet/tablet.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.tablet;
-option java_package = "org.kududb.tablet";
+option java_package = "org.apache.kudu.tablet";
import "kudu/common/common.proto";
import "kudu/common/wire_protocol.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/tserver/remote_bootstrap.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/remote_bootstrap.proto b/src/kudu/tserver/remote_bootstrap.proto
index 9fa6522..342765d 100644
--- a/src/kudu/tserver/remote_bootstrap.proto
+++ b/src/kudu/tserver/remote_bootstrap.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.tserver;
-option java_package = "org.kududb.tserver";
+option java_package = "org.apache.kudu.tserver";
import "kudu/common/wire_protocol.proto";
import "kudu/consensus/metadata.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/tserver/tserver.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tserver.proto b/src/kudu/tserver/tserver.proto
index e44c1bc..5964817 100644
--- a/src/kudu/tserver/tserver.proto
+++ b/src/kudu/tserver/tserver.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.tserver;
-option java_package = "org.kududb.tserver";
+option java_package = "org.apache.kudu.tserver";
import "kudu/common/common.proto";
import "kudu/common/wire_protocol.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/tserver/tserver_admin.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tserver_admin.proto b/src/kudu/tserver/tserver_admin.proto
index e971af8..dcbcbde 100644
--- a/src/kudu/tserver/tserver_admin.proto
+++ b/src/kudu/tserver/tserver_admin.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.tserver;
-option java_package = "org.kududb.tserver";
+option java_package = "org.apache.kudu.tserver";
import "kudu/common/common.proto";
import "kudu/consensus/metadata.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/tserver/tserver_service.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tserver_service.proto b/src/kudu/tserver/tserver_service.proto
index 0897cae..665a073 100644
--- a/src/kudu/tserver/tserver_service.proto
+++ b/src/kudu/tserver/tserver_service.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu.tserver;
-option java_package = "org.kududb.tserver";
+option java_package = "org.apache.kudu.tserver";
import "kudu/rpc/rpc_header.proto";
import "kudu/tserver/tserver.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/util/histogram.proto
----------------------------------------------------------------------
diff --git a/src/kudu/util/histogram.proto b/src/kudu/util/histogram.proto
index f54bc93..c7c846e 100644
--- a/src/kudu/util/histogram.proto
+++ b/src/kudu/util/histogram.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
// Captures the state of an Histogram.
message HistogramSnapshotPB {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/util/maintenance_manager.proto
----------------------------------------------------------------------
diff --git a/src/kudu/util/maintenance_manager.proto b/src/kudu/util/maintenance_manager.proto
index 04b4519..79c67e9 100644
--- a/src/kudu/util/maintenance_manager.proto
+++ b/src/kudu/util/maintenance_manager.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
// Used to present the maintenance manager's internal state.
message MaintenanceManagerStatusPB {
@@ -45,4 +45,4 @@ message MaintenanceManagerStatusPB {
// This list isn't in order of anything. Can contain the same operation mutiple times.
repeated CompletedOpPB completed_operations = 3;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/util/pb_util.proto
----------------------------------------------------------------------
diff --git a/src/kudu/util/pb_util.proto b/src/kudu/util/pb_util.proto
index b7265e2..71e8bdb 100644
--- a/src/kudu/util/pb_util.proto
+++ b/src/kudu/util/pb_util.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
import "google/protobuf/descriptor.proto";
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/src/kudu/util/version_info.proto
----------------------------------------------------------------------
diff --git a/src/kudu/util/version_info.proto b/src/kudu/util/version_info.proto
index b543d47..0135698 100644
--- a/src/kudu/util/version_info.proto
+++ b/src/kudu/util/version_info.proto
@@ -16,7 +16,7 @@
// under the License.
package kudu;
-option java_package = "org.kududb";
+option java_package = "org.apache.kudu";
// Information about the build environment, configuration, etc.
message VersionInfoPB {
[18/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Bytes.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Bytes.java b/java/kudu-client/src/main/java/org/kududb/client/Bytes.java
deleted file mode 100644
index 0e68e93..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Bytes.java
+++ /dev/null
@@ -1,1094 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import com.google.common.io.BaseEncoding;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Slice;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.util.CharsetUtil;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Comparator;
-
-/**
- * Helper functions to manipulate byte arrays.
- */
-@InterfaceAudience.Private
-public final class Bytes {
-
- // Two's complement reference: 2^n .
- // In this case, 2^64 (so as to emulate a unsigned long)
- // from http://stackoverflow.com/questions/10886962/interpret-a-negative-number-as-unsigned-with-
- // biginteger-java
- private static final BigInteger TWO_COMPL_REF = BigInteger.ONE.shiftLeft(64);
-
- private Bytes() { // Can't instantiate.
- }
-
- // -------------------------------- //
- // Byte array conversion utilities. //
- // -------------------------------- //
-
- /**
- * Reads a boolean from the beginning of the given array.
- * @param b The array to read from.
- * @return A boolean
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static boolean getBoolean(final byte[] b) {
- byte v = getByte(b, 0);
- return v == 1;
- }
-
- /**
- * Reads a boolean from an offset in the given array.
- * @param b The array to read from.
- * @param offset The offset into the array.
- * @return A boolean
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static boolean getBoolean(final byte[] b, final int offset) {
- byte v = getByte(b, offset);
- return v == 1;
- }
-
- /**
- * Reads a byte from the beginning of the given array.
- * @param b The array to read from.
- * @return A byte
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static byte getByte(final byte[] b) {
- return getByte(b, 0);
- }
-
- /**
- * Reads a byte from an offset in the given array.
- * @param b The array to read from.
- * @return A byte
- * @return
- */
- public static byte getByte(final byte[] b, final int offset) {
- return b[offset];
- }
-
- /**
- * Reads an unsigned byte from the beginning of the given array.
- * @param b The array to read from.
- * @return A positive byte
- */
- public static short getUnsignedByte(final byte[] b) {
- return getUnsignedByte(b, 0);
- }
-
- /**
- * Reads an unsigned byte from an offset in the given array.
- * @param b The array to read from.
- * @return A positive byte
- */
- public static short getUnsignedByte(final byte[] b, final int offset) {
- return (short) (b[offset] & 0x00FF);
- }
-
- /**
- * Writes an unsigned byte at the beginning of the given array.
- * @param b The array to write to.
- * @param n An unsigned byte.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedByte(final byte[] b, final short n) {
- setUnsignedByte(b, n, 0);
- }
-
- /**
- * Writes an unsigned byte at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n An unsigned byte.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedByte(final byte[] b, final short n,
- final int offset) {
- b[offset] = (byte) n;
- }
-
- /**
- * Creates a new byte array containing an unsigned byte.
- * @param n An unsigned byte.
- * @return A new byte array containing the given value.
- */
- public static byte[] fromUnsignedByte(final short n) {
- final byte[] b = new byte[1];
- setUnsignedByte(b, n);
- return b;
- }
-
- /**
- * Reads a little-endian 2-byte short from the beginning of the given array.
- * @param b The array to read from.
- * @return A short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static short getShort(final byte[] b) {
- return getShort(b, 0);
- }
-
- /**
- * Reads a little-endian 2-byte short from an offset in the given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return A short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static short getShort(final byte[] b, final int offset) {
- return (short) (b[offset] & 0xFF | b[offset + 1] << 8 );
- }
-
- /**
- * Reads a little-endian 2-byte unsigned short from the beginning of the
- * given array.
- * @param b The array to read from.
- * @return A positive short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static int getUnsignedShort(final byte[] b) {
- return getUnsignedShort(b, 0);
- }
-
- /**
- * Reads a little-endian 2-byte unsigned short from an offset in the
- * given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return A positive short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static int getUnsignedShort(final byte[] b, final int offset) {
- return getShort(b, offset) & 0x0000FFFF;
- }
-
- /**
- * Writes a little-endian 2-byte short at the beginning of the given array.
- * @param b The array to write to.
- * @param n A short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setShort(final byte[] b, final short n) {
- setShort(b, n, 0);
- }
-
- /**
- * Writes a little-endian 2-byte short at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n A short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setShort(final byte[] b, final short n,
- final int offset) {
- b[offset + 0] = (byte) (n >>> 0);
- b[offset + 1] = (byte) (n >>> 8);
- }
-
- /**
- * Writes a little-endian 2-byte unsigned short at the beginning of the given array.
- * @param b The array to write to.
- * @param n An unsigned short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedShort(final byte[] b, final int n) {
- setUnsignedShort(b, n, 0);
- }
-
- /**
- * Writes a little-endian 2-byte unsigned short at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n An unsigned short integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedShort(final byte[] b, final int n,
- final int offset) {
- b[offset + 0] = (byte) (n >>> 0);
- b[offset + 1] = (byte) (n >>> 8);
- }
-
- /**
- * Creates a new byte array containing a little-endian 2-byte short integer.
- * @param n A short integer.
- * @return A new byte array containing the given value.
- */
- public static byte[] fromShort(final short n) {
- final byte[] b = new byte[2];
- setShort(b, n);
- return b;
- }
-
- /**
- * Creates a new byte array containing a little-endian 2-byte unsigned short integer.
- * @param n An unsigned short integer.
- * @return A new byte array containing the given value.
- */
- public static byte[] fromUnsignedShort(final int n) {
- final byte[] b = new byte[2];
- setUnsignedShort(b, n);
- return b;
- }
-
- /**
- * Reads a little-endian 4-byte integer from the beginning of the given array.
- * @param b The array to read from.
- * @return An integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static int getInt(final byte[] b) {
- return getInt(b, 0);
- }
-
- /**
- * Reads a little-endian 4-byte integer from an offset in the given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return An integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static int getInt(final byte[] b, final int offset) {
- return (b[offset + 0] & 0xFF) << 0
- | (b[offset + 1] & 0xFF) << 8
- | (b[offset + 2] & 0xFF) << 16
- | (b[offset + 3] & 0xFF) << 24;
- }
-
- /**
- * Reads a little-endian 4-byte unsigned integer from the beginning of the
- * given array.
- * @param b The array to read from.
- * @return A positive integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static long getUnsignedInt(final byte[] b) {
- return getUnsignedInt(b, 0);
- }
-
- /**
- * Reads a little-endian 4-byte unsigned integer from an offset in the
- * given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return A positive integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static long getUnsignedInt(final byte[] b, final int offset) {
- return getInt(b, offset) & 0x00000000FFFFFFFFL;
- }
-
- /**
- * Writes a little-endian 4-byte int at the beginning of the given array.
- * @param b The array to write to.
- * @param n An integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setInt(final byte[] b, final int n) {
- setInt(b, n, 0);
- }
-
- /**
- * Writes a little-endian 4-byte int at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n An integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setInt(final byte[] b, final int n, final int offset) {
- b[offset + 0] = (byte) (n >>> 0);
- b[offset + 1] = (byte) (n >>> 8);
- b[offset + 2] = (byte) (n >>> 16);
- b[offset + 3] = (byte) (n >>> 24);
- }
-
- /**
- * Writes a little-endian 4-byte unsigned int at the beginning of the given array.
- * @param b The array to write to.
- * @param n An unsigned integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedInt(final byte[] b, final long n) {
- setUnsignedInt(b, n, 0);
- }
-
- /**
- * Writes a little-endian 4-byte unsigned int at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n An unsigned integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedInt(final byte[] b, final long n, final int offset) {
- b[offset + 0] = (byte) (n >>> 0);
- b[offset + 1] = (byte) (n >>> 8);
- b[offset + 2] = (byte) (n >>> 16);
- b[offset + 3] = (byte) (n >>> 24);
- }
-
- public static void putVarInt32(final ByteBuffer b, final int v) {
- int B = 128;
- if (v < (1<<7)) {
- b.put((byte)v);
- } else if (v < (1<<14)) {
- b.put((byte)(v | B));
- b.put((byte)((v>>7) | B));
- } else if (v < (1<<21)) {
- b.put((byte)(v | B));
- b.put((byte)((v>>7) | B));
- b.put((byte)(v>>14));
- } else if (v < (1<<28)) {
- b.put((byte)(v | B));
- b.put((byte)((v>>7) | B));
- b.put((byte)((v>>14) | B));
- b.put((byte)(v>>21));
- } else {
- b.put((byte)(v | B));
- b.put((byte)((v>>7) | B));
- b.put((byte)((v>>14) | B));
- b.put((byte)((v>>21) | B));
- b.put((byte)(v>>28));
- }
- }
-
- /**
- * Reads a 32-bit variable-length integer value as used in Protocol Buffers.
- * @param buf The buffer to read from.
- * @return The integer read.
- */
- static int readVarInt32(final ChannelBuffer buf) {
- int result = buf.readByte();
- if (result >= 0) {
- return result;
- }
- result &= 0x7F;
- result |= buf.readByte() << 7;
- if (result >= 0) {
- return result;
- }
- result &= 0x3FFF;
- result |= buf.readByte() << 14;
- if (result >= 0) {
- return result;
- }
- result &= 0x1FFFFF;
- result |= buf.readByte() << 21;
- if (result >= 0) {
- return result;
- }
- result &= 0x0FFFFFFF;
- final byte b = buf.readByte();
- result |= b << 28;
- if (b >= 0) {
- return result;
- }
- throw new IllegalArgumentException("Not a 32 bit varint: " + result
- + " (5th byte: " + b + ")");
- }
-
- public static byte[] fromBoolean(final boolean n) {
- final byte[] b = new byte[1];
- b[0] = (byte) (n ? 1 : 0);
- return b;
- }
-
- /**
- * Creates a new byte array containing a little-endian 4-byte integer.
- * @param n An integer.
- * @return A new byte array containing the given value.
- */
- public static byte[] fromInt(final int n) {
- final byte[] b = new byte[4];
- setInt(b, n);
- return b;
- }
-
- /**
- * Creates a new byte array containing a little-endian 4-byte unsigned integer.
- * @param n An unsigned integer.
- * @return A new byte array containing the given value.
- */
- public static byte[] fromUnsignedInt(final long n) {
- final byte[] b = new byte[4];
- setUnsignedInt(b, n);
- return b;
- }
-
- /**
- * Reads a little-endian 8-byte unsigned long from the beginning of the given array.
- * @param b The array to read from.
- * @return A long integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static BigInteger getUnsignedLong(final byte[] b) {
- return getUnsignedLong(b, 0);
- }
-
- /**
- * Reads a little-endian 8-byte unsigned long from an offset in the given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return A long integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static BigInteger getUnsignedLong(final byte[] b, final int offset) {
- long l = getLong(b, offset);
- BigInteger bi = new BigInteger(l+"");
- if (bi.compareTo(BigInteger.ZERO) < 0) {
- bi = bi.add(TWO_COMPL_REF);
- }
- return bi;
- }
-
- /**
- * Reads a little-endian 8-byte long from the beginning of the given array.
- * @param b The array to read from.
- * @return A long integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static long getLong(final byte[] b) {
- return getLong(b, 0);
- }
-
- /**
- * Reads a little-endian 8-byte long from an offset in the given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return A long integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static long getLong(final byte[] b, final int offset) {
- return (b[offset + 0] & 0xFFL) << 0
- | (b[offset + 1] & 0xFFL) << 8
- | (b[offset + 2] & 0xFFL) << 16
- | (b[offset + 3] & 0xFFL) << 24
- | (b[offset + 4] & 0xFFL) << 32
- | (b[offset + 5] & 0xFFL) << 40
- | (b[offset + 6] & 0xFFL) << 48
- | (b[offset + 7] & 0xFFL) << 56;
- }
-
- /**
- * Writes a little-endian 8-byte long at the beginning of the given array.
- * @param b The array to write to.
- * @param n A long integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setLong(final byte[] b, final long n) {
- setLong(b, n, 0);
- }
-
- /**
- * Writes a little-endian 8-byte long at an offset in the given array.
- * @param b The array to write to.
- * @param n A long integer.
- * @param offset The offset in the array to start writing at.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setLong(final byte[] b, final long n, final int offset) {
- b[offset + 0] = (byte) (n >>> 0);
- b[offset + 1] = (byte) (n >>> 8);
- b[offset + 2] = (byte) (n >>> 16);
- b[offset + 3] = (byte) (n >>> 24);
- b[offset + 4] = (byte) (n >>> 32);
- b[offset + 5] = (byte) (n >>> 40);
- b[offset + 6] = (byte) (n >>> 48);
- b[offset + 7] = (byte) (n >>> 56);
- }
-
- /**
- * Writes a little-endian 8-byte unsigned long at the beginning of the given array.
- * @param b The array to write to.
- * @param n An unsigned long integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedLong(final byte[] b, final BigInteger n) {
- setUnsignedLong(b, n, 0);
- }
-
- /**
- * Writes a little-endian 8-byte unsigned long at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n An unsigned long integer.
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setUnsignedLong(final byte[] b, final BigInteger n, final int offset) {
- setLong(b, n.longValue(), offset);
- }
-
- /**
- * Creates a new byte array containing a little-endian 8-byte long integer.
- * @param n A long integer.
- * @return A new byte array containing the given value.
- */
- public static byte[] fromLong(final long n) {
- final byte[] b = new byte[8];
- setLong(b, n);
- return b;
- }
-
- /**
- * Creates a new byte array containing a little-endian 8-byte unsigned long integer.
- * @param n An unsigned long integer.
- * @return A new byte array containing the given value.
- */
- public static byte[] fromUnsignedLong(final BigInteger n) {
- final byte[] b = new byte[8];
- setUnsignedLong(b, n);
- return b;
- }
-
- /**
- * Reads a little-endian 4-byte float from the beginning of the given array.
- * @param b The array to read from.
- * @return a float
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static float getFloat(final byte[] b) {
- return getFloat(b, 0);
- }
-
- /**
- * Reads a little-endian 4-byte float from an offset in the given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return a float
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static float getFloat(final byte[] b, final int offset) {
- return Float.intBitsToFloat(getInt(b, offset));
- }
-
- /**
- * Writes a little-endian 4-byte float at the beginning of the given array.
- * @param b The array to write to.
- * @param n a float
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setFloat(final byte[] b, final float n) {
- setFloat(b, n, 0);
- }
-
- /**
- * Writes a little-endian 4-byte float at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n a float
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setFloat(final byte[] b, final float n, final int offset) {
- setInt(b, Float.floatToIntBits(n), offset);
- }
-
- /**
- * Creates a new byte array containing a little-endian 4-byte float.
- * @param n A float
- * @return A new byte array containing the given value.
- */
- public static byte[] fromFloat(float n) {
- byte[] b = new byte[4];
- setFloat(b, n);
- return b;
- }
-
- /**
- * Reads a little-endian 8-byte double from the beginning of the given array.
- * @param b The array to read from.
- * @return a double
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static double getDouble(final byte[] b) {
- return getDouble(b, 0);
- }
-
- /**
- * Reads a little-endian 8-byte double from an offset in the given array.
- * @param b The array to read from.
- * @param offset The offset in the array to start reading from.
- * @return a double
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static double getDouble(final byte[] b, final int offset) {
- return Double.longBitsToDouble(getLong(b, offset));
- }
-
- /**
- * Writes a little-endian 8-byte double at the beginning of the given array.
- * @param b The array to write to.
- * @param n a double
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setDouble(final byte[] b, final double n) {
- setDouble(b, n, 0);
- }
-
- /**
- * Writes a little-endian 8-byte double at an offset in the given array.
- * @param b The array to write to.
- * @param offset The offset in the array to start writing at.
- * @param n a double
- * @throws IndexOutOfBoundsException if the byte array is too small.
- */
- public static void setDouble(final byte[] b, final double n, final int offset) {
- setLong(b, Double.doubleToLongBits(n), offset);
- }
-
- /**
- * Creates a new byte array containing a little-endian 8-byte double.
- * @param n A double
- * @return A new byte array containing the given value.
- */
- public static byte[] fromDouble(double n) {
- byte[] b = new byte[8];
- setDouble(b, n);
- return b;
- }
-
- /**
- * Extracts the byte array from the given {@link ByteString} without copy.
- * @param buf A buffer from which to extract the array. This buffer must be
- * actually an instance of a {@code LiteralByteString}.
- * @since 1.5
- */
- public static byte[] get(final ByteString buf) {
- return ZeroCopyLiteralByteString.zeroCopyGetBytes(buf);
- }
-
- /** Transforms a string into an UTF-8 encoded byte array. */
- public static byte[] UTF8(final String s) {
- return s.getBytes(CharsetUtil.UTF_8);
- }
-
- /** Transforms a string into an ISO-8859-1 encoded byte array. */
- public static byte[] ISO88591(final String s) {
- return s.getBytes(CharsetUtil.ISO_8859_1);
- }
-
- // ---------------------------- //
- // Pretty-printing byte arrays. //
- // ---------------------------- //
-
- private static final char[] HEX = {
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
- 'A', 'B', 'C', 'D', 'E', 'F'
- };
-
- /**
- * Pretty-prints a byte array into a human-readable output buffer.
- * @param outbuf The buffer where to write the output.
- * @param array The (possibly {@code null}) array to pretty-print.
- */
- public static void pretty(final StringBuilder outbuf, final byte[] array) {
- if (array == null) {
- outbuf.append("null");
- return;
- }
- int ascii = 0;
- final int start_length = outbuf.length();
- final int n = array.length;
- outbuf.ensureCapacity(start_length + 1 + n + 1);
- outbuf.append('"');
- for (int i = 0; i < n; i++) {
- final byte b = array[i];
- if (' ' <= b && b <= '~') {
- ascii++;
- outbuf.append((char) b);
- } else if (b == '\n') {
- outbuf.append('\\').append('n');
- } else if (b == '\t') {
- outbuf.append('\\').append('t');
- } else {
- outbuf.append("\\x")
- .append(HEX[(b >>> 4) & 0x0F])
- .append(HEX[b & 0x0F]);
- }
- }
- if (ascii < n / 2) {
- outbuf.setLength(start_length);
- outbuf.append(Arrays.toString(array));
- } else {
- outbuf.append('"');
- }
- }
-
- /**
- * Pretty-prints an array of byte arrays into a human-readable output buffer.
- * @param outbuf The buffer where to write the output.
- * @param arrays The (possibly {@code null}) array of arrays to pretty-print.
- * @since 1.3
- */
- public static void pretty(final StringBuilder outbuf, final byte[][] arrays) {
- if (arrays == null) {
- outbuf.append("null");
- return;
- } else { // Do some right-sizing.
- int size = 2;
- for (int i = 0; i < arrays.length; i++) {
- size += 2 + 2 + arrays[i].length;
- }
- outbuf.ensureCapacity(outbuf.length() + size);
- }
- outbuf.append('[');
- for (int i = 0; i < arrays.length; i++) {
- Bytes.pretty(outbuf, arrays[i]);
- outbuf.append(", ");
- }
- outbuf.setLength(outbuf.length() - 2); // Remove the last ", "
- outbuf.append(']');
- }
-
- /**
- * Pretty-prints a byte array into a human-readable string.
- * @param array The (possibly {@code null}) array to pretty-print.
- * @return The array in a pretty-printed string.
- */
- public static String pretty(final byte[] array) {
- if (array == null) {
- return "null";
- }
- final StringBuilder buf = new StringBuilder(1 + array.length + 1);
- pretty(buf, array);
- return buf.toString();
- }
-
- /**
- * Convert a byte array to a hex encoded string.
- * @param bytes the bytes to encode
- * @return the hex encoded bytes
- */
- public static String hex(byte[] bytes) {
- StringBuilder sb = new StringBuilder(2 + bytes.length * 2);
- sb.append('0');
- sb.append('x');
- sb.append(BaseEncoding.base16().encode(bytes));
- return sb.toString();
- }
-
- // Ugly stuff
- // ----------
- // Background: when using ReplayingDecoder (which makes it easy to deal with
- // unframed RPC responses), the ChannelBuffer we manipulate is in fact a
- // ReplayingDecoderBuffer, a package-private class that Netty uses. This
- // class, for some reason, throws UnsupportedOperationException on its
- // array() method. This method is unfortunately the only way to easily dump
- // the contents of a ChannelBuffer, which is useful for debugging or logging
- // unexpected buffers. An issue (NETTY-346) has been filed to get access to
- // the buffer, but the resolution was useless: instead of making the array()
- // method work, a new internalBuffer() method was added on ReplayingDecoder,
- // which would require that we keep a reference on the ReplayingDecoder all
- // along in order to properly convert the buffer to a string.
- // So we instead use ugly reflection to gain access to the underlying buffer
- // while taking into account that the implementation of Netty has changed
- // over time, so depending which version of Netty we're working with, we do
- // a different hack. Yes this is horrible, but it's for the greater good as
- // this is what allows us to debug unexpected buffers when deserializing RPCs
- // and what's more important than being able to debug unexpected stuff?
- private static final Class<?> ReplayingDecoderBuffer;
- private static final Field RDB_buffer; // For Netty 3.5.0 and before.
- private static final Method RDB_buf; // For Netty 3.5.1 and above.
- static {
- try {
- ReplayingDecoderBuffer = Class.forName("org.jboss.netty.handler.codec."
- + "replay.ReplayingDecoderBuffer");
- Field field = null;
- try {
- field = ReplayingDecoderBuffer.getDeclaredField("buffer");
- field.setAccessible(true);
- } catch (NoSuchFieldException e) {
- // Ignore. Field has been removed in Netty 3.5.1.
- }
- RDB_buffer = field;
- if (field != null) { // Netty 3.5.0 or before.
- RDB_buf = null;
- } else {
- RDB_buf = ReplayingDecoderBuffer.getDeclaredMethod("buf");
- RDB_buf.setAccessible(true);
- }
- } catch (Exception e) {
- throw new RuntimeException("static initializer failed", e);
- }
- }
-
- /**
- * Pretty-prints all the bytes of a buffer into a human-readable string.
- * @param buf The (possibly {@code null}) buffer to pretty-print.
- * @return The buffer in a pretty-printed string.
- */
- public static String pretty(final ChannelBuffer buf) {
- if (buf == null) {
- return "null";
- }
- byte[] array;
- try {
- if (buf.getClass() != ReplayingDecoderBuffer) {
- array = buf.array();
- } else if (RDB_buf != null) { // Netty 3.5.1 and above.
- array = ((ChannelBuffer) RDB_buf.invoke(buf)).array();
- } else { // Netty 3.5.0 and before.
- final ChannelBuffer wrapped_buf = (ChannelBuffer) RDB_buffer.get(buf);
- array = wrapped_buf.array();
- }
- } catch (UnsupportedOperationException e) {
- return "(failed to extract content of buffer of type "
- + buf.getClass().getName() + ')';
- } catch (IllegalAccessException e) {
- throw new AssertionError("Should not happen: " + e);
- } catch (InvocationTargetException e) {
- throw new AssertionError("Should not happen: " + e);
- }
- return pretty(array);
- }
-
- // ---------------------- //
- // Comparing byte arrays. //
- // ---------------------- //
- // Don't ask me why this isn't in java.util.Arrays.
-
- /**
- * A singleton {@link Comparator} for non-{@code null} byte arrays.
- * @see #memcmp
- */
- public static final MemCmp MEMCMP = new MemCmp();
-
- /** {@link Comparator} for non-{@code null} byte arrays. */
- private static final class MemCmp implements Comparator<byte[]> {
-
- private MemCmp() { // Can't instantiate outside of this class.
- }
-
- @Override
- public int compare(final byte[] a, final byte[] b) {
- return memcmp(a, b);
- }
-
- }
-
- /**
- * {@code memcmp} in Java, hooray.
- * @param a First non-{@code null} byte array to compare.
- * @param b Second non-{@code null} byte array to compare.
- * @return 0 if the two arrays are identical, otherwise the difference
- * between the first two different bytes, otherwise the different between
- * their lengths.
- */
- public static int memcmp(final byte[] a, final byte[] b) {
- final int length = Math.min(a.length, b.length);
- if (a == b) { // Do this after accessing a.length and b.length
- return 0; // in order to NPE if either a or b is null.
- }
- for (int i = 0; i < length; i++) {
- if (a[i] != b[i]) {
- return (a[i] & 0xFF) - (b[i] & 0xFF); // "promote" to unsigned.
- }
- }
- return a.length - b.length;
- }
-
- /**
- * {@code memcmp(3)} with a given offset and length.
- * @param a First non-{@code null} byte array to compare.
- * @param b Second non-{@code null} byte array to compare.
- * @param offset The offset at which to start comparing both arrays.
- * @param length The number of bytes to compare.
- * @return 0 if the two arrays are identical, otherwise the difference
- * between the first two different bytes (treated as unsigned), otherwise
- * the different between their lengths.
- * @throws IndexOutOfBoundsException if either array isn't large enough.
- */
- public static int memcmp(final byte[] a, final byte[] b,
- final int offset, int length) {
- if (a == b && a != null) {
- return 0;
- }
- length += offset;
- for (int i = offset; i < length; i++) {
- if (a[i] != b[i]) {
- return (a[i] & 0xFF) - (b[i] & 0xFF); // "promote" to unsigned.
- }
- }
- return 0;
- }
-
- /**
- * De-duplicates two byte arrays.
- * <p>
- * If two byte arrays have the same contents but are different, this
- * function helps to re-use the old one and discard the new copy.
- * @param old The existing byte array.
- * @param neww The new byte array we're trying to de-duplicate.
- * @return {@code old} if {@code neww} is a different array with the same
- * contents, otherwise {@code neww}.
- */
- public static byte[] deDup(final byte[] old, final byte[] neww) {
- return memcmp(old, neww) == 0 ? old : neww;
- }
-
- /**
- * Tests whether two byte arrays have the same contents.
- * @param a First non-{@code null} byte array to compare.
- * @param b Second non-{@code null} byte array to compare.
- * @return {@code true} if the two arrays are identical,
- * {@code false} otherwise.
- */
- public static boolean equals(final byte[] a, final byte[] b) {
- return memcmp(a, b) == 0;
- }
-
- /**
- * {@code memcmp(3)} in Java for possibly {@code null} arrays, hooray.
- * @param a First possibly {@code null} byte array to compare.
- * @param b Second possibly {@code null} byte array to compare.
- * @return 0 if the two arrays are identical (or both are {@code null}),
- * otherwise the difference between the first two different bytes (treated
- * as unsigned), otherwise the different between their lengths (a {@code
- * null} byte array is considered shorter than an empty byte array).
- */
- public static int memcmpMaybeNull(final byte[] a, final byte[] b) {
- if (a == null) {
- if (b == null) {
- return 0;
- }
- return -1;
- } else if (b == null) {
- return 1;
- }
- return memcmp(a, b);
- }
-
- public static int getBitSetSize(int items) {
- return (items + 7) / 8;
- }
-
- public static byte[] fromBitSet(BitSet bits, int colCount) {
- byte[] bytes = new byte[getBitSetSize(colCount)];
- for (int i = 0; i < bits.length(); i++) {
- if (bits.get(i)) {
- bytes[i / 8] |= 1 << (i % 8);
- }
- }
- return bytes;
- }
-
- public static BitSet toBitSet(byte[] b, int offset, int colCount) {
- BitSet bs = new BitSet(colCount);
- for (int i = 0; i < colCount; i++) {
- if ((b[offset + (i / 8)] >> (i % 8) & 1) == 1) {
- bs.set(i);
- }
- }
- return bs;
- }
-
- /**
- * This method will apply xor on the left most bit of the provided byte. This is used in Kudu to
- * have unsigned data types sorting correctly.
- * @param value byte whose left most bit will be xor'd
- * @return same byte with xor applied on the left most bit
- */
- public static byte xorLeftMostBit(byte value) {
- value ^= (1 << 7);
- return value;
- }
-
- /**
- * Get the byte array representation of this string, with UTF8 encoding
- * @param data String get the byte array from
- * @return UTF8 byte array
- */
- public static byte[] fromString(String data) {
- return UTF8(data);
- }
-
- /**
- * Get a string from the passed byte array, with UTF8 encoding
- * @param b byte array to convert to string, possibly coming from {@link #fromString(String)}
- * @return A new string built with the byte array
- */
- public static String getString(byte[] b) {
- return getString(b, 0, b.length);
- }
-
- public static String getString(Slice slice) {
- return slice.toString(CharsetUtil.UTF_8);
- }
-
- /**
- * Get a string from the passed byte array, at the specified offset and for the specified
- * length, with UTF8 encoding
- * @param b byte array to convert to string, possibly coming from {@link #fromString(String)}
- * @param offset where to start reading from in the byte array
- * @param len how many bytes we should read
- * @return A new string built with the byte array
- */
- public static String getString(byte[] b, int offset, int len) {
- if (len == 0) {
- return "";
- }
- return new String(b, offset, len, CharsetUtil.UTF_8);
- }
-
- /**
- * Utility method to write a byte array to a data output. Equivalent of doing a writeInt of the
- * length followed by a write of the byte array. Convert back with {@link #readByteArray}
- * @param dataOutput
- * @param b
- * @throws IOException
- */
- public static void writeByteArray(DataOutput dataOutput, byte[] b) throws IOException {
- dataOutput.writeInt(b.length);
- dataOutput.write(b);
- }
-
- /**
- * Utility method to read a byte array written the way {@link #writeByteArray} does it.
- * @param dataInput
- * @return
- * @throws IOException
- */
- public static byte[] readByteArray(DataInput dataInput) throws IOException {
- int len = dataInput.readInt();
- byte[] data = new byte[len];
- dataInput.readFully(data);
- return data;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/CallResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/CallResponse.java b/java/kudu-client/src/main/java/org/kududb/client/CallResponse.java
deleted file mode 100644
index 77cbd5e..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/CallResponse.java
+++ /dev/null
@@ -1,162 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import java.util.List;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.rpc.RpcHeader;
-import org.kududb.util.Slice;
-
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * This class handles information received from an RPC response, providing
- * access to sidecars and decoded protobufs from the message.
- */
-@InterfaceAudience.Private
-final class CallResponse {
- private final ChannelBuffer buf;
- private final RpcHeader.ResponseHeader header;
- private final int totalResponseSize;
-
- // Non-header main message slice is generated upon request and cached.
- private Slice message = null;
-
- /**
- * Performs some sanity checks on the sizes recorded in the packet
- * referred to by {@code buf}. Assumes that {@code buf} has not been
- * read from yet, and will only be accessed by this class.
- *
- * Afterwards, this constructs the RpcHeader from the buffer.
- * @param buf Channel buffer which call response reads from.
- * @throws IllegalArgumentException If either the entire recorded packet
- * size or recorded response header PB size are not within reasonable
- * limits as defined by {@link KuduRpc#checkArrayLength(ChannelBuffer, long)}.
- * @throws IndexOutOfBoundsException if the ChannelBuffer does not contain
- * the amount of bytes specified by its length prefix.
- */
- public CallResponse(final ChannelBuffer buf) {
- this.buf = buf;
-
- this.totalResponseSize = buf.readInt();
- KuduRpc.checkArrayLength(buf, this.totalResponseSize);
- TabletClient.ensureReadable(buf, this.totalResponseSize);
-
- final int headerSize = Bytes.readVarInt32(buf);
- final Slice headerSlice = nextBytes(buf, headerSize);
- RpcHeader.ResponseHeader.Builder builder = RpcHeader.ResponseHeader.newBuilder();
- KuduRpc.readProtobuf(headerSlice, builder);
- this.header = builder.build();
- }
-
- /**
- * @return the parsed header
- */
- public RpcHeader.ResponseHeader getHeader() {
- return this.header;
- }
-
- /**
- * @return the total response size
- */
- public int getTotalResponseSize() { return this.totalResponseSize; }
-
- /**
- * @return A slice pointing to the section of the packet reserved for the main
- * protobuf message.
- * @throws IllegalArgumentException If the recorded size for the main message
- * is not within reasonable limits as defined by
- * {@link KuduRpc#checkArrayLength(ChannelBuffer, long)}.
- * @throws IllegalStateException If the offset for the main protobuf message
- * is not valid.
- */
- public Slice getPBMessage() {
- cacheMessage();
- final int mainLength = this.header.getSidecarOffsetsCount() == 0 ?
- this.message.length() : this.header.getSidecarOffsets(0);
- if (mainLength < 0 || mainLength > this.message.length()) {
- throw new IllegalStateException("Main protobuf message invalid. "
- + "Length is " + mainLength + " while the size of the message "
- + "excluding the header is " + this.message.length());
- }
- return subslice(this.message, 0, mainLength);
- }
-
- /**
- * @param sidecar The index of the sidecar to retrieve.
- * @return A slice pointing to the desired sidecar.
- * @throws IllegalStateException If the sidecar offsets specified in the
- * header response PB are not valid offsets for the array.
- * @throws IllegalArgumentException If the sidecar with the specified index
- * does not exist.
- * @throws IllegalArgumentException If the recorded size for the main message
- * is not within reasonable limits as defined by
- * {@link KuduRpc#checkArrayLength(ChannelBuffer, long)}.
- */
- public Slice getSidecar(int sidecar) {
- cacheMessage();
-
- List<Integer> sidecarList = this.header.getSidecarOffsetsList();
- if (sidecar < 0 || sidecar > sidecarList.size()) {
- throw new IllegalArgumentException("Sidecar " + sidecar
- + " not valid, response has " + sidecarList.size() + " sidecars");
- }
-
- final int prevOffset = sidecarList.get(sidecar);
- final int nextOffset = sidecar + 1 == sidecarList.size() ?
- this.message.length() : sidecarList.get(sidecar + 1);
- final int length = nextOffset - prevOffset;
-
- if (prevOffset < 0 || length < 0 || prevOffset + length > this.message.length()) {
- throw new IllegalStateException("Sidecar " + sidecar + " invalid "
- + "(offset = " + prevOffset + ", length = " + length + "). The size "
- + "of the message " + "excluding the header is " + this.message.length());
- }
-
- return subslice(this.message, prevOffset, length);
- }
-
- // Reads the message after the header if not read yet
- private void cacheMessage() {
- if (this.message != null) return;
- final int length = Bytes.readVarInt32(buf);
- this.message = nextBytes(buf, length);
- }
-
- // Accounts for a parent slice's offset when making a new one with relative offsets.
- private static Slice subslice(Slice parent, int offset, int length) {
- return new Slice(parent.getRawArray(), parent.getRawOffset() + offset, length);
- }
-
- // After checking the length, generates a slice for the next 'length'
- // bytes of 'buf'.
- private static Slice nextBytes(final ChannelBuffer buf, final int length) {
- KuduRpc.checkArrayLength(buf, length);
- byte[] payload;
- int offset;
- if (buf.hasArray()) { // Zero copy.
- payload = buf.array();
- offset = buf.arrayOffset() + buf.readerIndex();
- } else { // We have to copy the entire payload out of the buffer :(
- payload = new byte[length];
- buf.readBytes(payload);
- offset = 0;
- }
- return new Slice(payload, offset, length);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ColumnRangePredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ColumnRangePredicate.java b/java/kudu-client/src/main/java/org/kududb/client/ColumnRangePredicate.java
deleted file mode 100644
index 0e24088..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ColumnRangePredicate.java
+++ /dev/null
@@ -1,387 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.tserver.Tserver;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A range predicate on one of the columns in the underlying data.
- * Both boundaries are inclusive.
- * @deprecated use the {@link KuduPredicate} class instead.
- */
-@InterfaceAudience.Public
-@Deprecated
-public class ColumnRangePredicate {
-
- private final Tserver.ColumnRangePredicatePB.Builder pb = Tserver.ColumnRangePredicatePB
- .newBuilder();
- private final ColumnSchema column;
- private byte[] lowerBound = null;
- private byte[] upperBound = null;
-
- /**
- * Create the predicate on the specified column
- * @param column
- */
- public ColumnRangePredicate(ColumnSchema column) {
- this.column = column;
- this.pb.setColumn(ProtobufHelper.columnToPb(column));
- }
-
- private void setLowerBoundInternal(byte[] value) {
- this.lowerBound = value;
- pb.setLowerBound(ZeroCopyLiteralByteString.wrap(this.lowerBound));
- }
-
- private void setUpperBoundInternal(byte[] value) {
- this.upperBound = value;
- pb.setInclusiveUpperBound(ZeroCopyLiteralByteString.wrap(this.upperBound));
- }
-
- /**
- * Convert a bound into a {@link KuduPredicate}.
- * @param column the column
- * @param op the bound comparison operator
- * @param bound the bound
- * @return the {@code KuduPredicate}
- */
- private static KuduPredicate toKuduPredicate(ColumnSchema column,
- KuduPredicate.ComparisonOp op,
- byte[] bound) {
- if (bound == null) { return null; }
- switch (column.getType().getDataType()) {
- case BOOL: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getBoolean(bound));
- case INT8: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getByte(bound));
- case INT16: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getShort(bound));
- case INT32: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getInt(bound));
- case INT64:
- case TIMESTAMP: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getLong(bound));
- case FLOAT: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getFloat(bound));
- case DOUBLE: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getDouble(bound));
- case STRING: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getString(bound));
- case BINARY: return KuduPredicate.newComparisonPredicate(column, op, bound);
- default:
- throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
- }
- }
-
- /**
- * Convert this column range predicate into a {@link KuduPredicate}.
- * @return the column predicate.
- */
- public KuduPredicate toKuduPredicate() {
- KuduPredicate lower =
- toKuduPredicate(column, KuduPredicate.ComparisonOp.GREATER_EQUAL, lowerBound);
- KuduPredicate upper =
- toKuduPredicate(column, KuduPredicate.ComparisonOp.LESS_EQUAL, upperBound);
-
- if (upper != null && lower != null) {
- return upper.merge(lower);
- } else if (upper != null) {
- return upper;
- } else {
- return lower;
- }
- }
-
- /**
- * Set a boolean for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(boolean lowerBound) {
- checkColumn(Type.BOOL);
- setLowerBoundInternal(Bytes.fromBoolean(lowerBound));
- }
-
- /**
- * Set a byte for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(byte lowerBound) {
- checkColumn(Type.INT8);
- setLowerBoundInternal(new byte[] {lowerBound});
- }
-
- /**
- * Set a short for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(short lowerBound) {
- checkColumn(Type.INT16);
- setLowerBoundInternal(Bytes.fromShort(lowerBound));
- }
-
- /**
- * Set an int for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(int lowerBound) {
- checkColumn(Type.INT32);
- setLowerBoundInternal(Bytes.fromInt(lowerBound));
- }
-
- /**
- * Set a long for the lower bound
- *
- * If 'lowerBound' is a timestamp see {@link PartialRow#addLong(String, long)} for the
- * format.
- *
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(long lowerBound) {
- checkColumn(Type.INT64, Type.TIMESTAMP);
- setLowerBoundInternal(Bytes.fromLong(lowerBound));
- }
-
- /**
- * Set a string for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(String lowerBound) {
- checkColumn(Type.STRING);
- setLowerBoundInternal(lowerBound.getBytes());
- }
-
- /**
- * Set a binary value for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(byte[] lowerBound) {
- checkColumn(Type.BINARY);
- setLowerBoundInternal(lowerBound);
- }
-
- /**
- * Set a float for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(float lowerBound) {
- checkColumn(Type.FLOAT);
- setLowerBoundInternal(Bytes.fromFloat(lowerBound));
- }
-
- /**
- * Set a double for the lower bound
- * @param lowerBound value for the lower bound
- */
- public void setLowerBound(double lowerBound) {
- checkColumn(Type.DOUBLE);
- setLowerBoundInternal(Bytes.fromDouble(lowerBound));
- }
-
- /**
- * Set a boolean for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(boolean upperBound) {
- checkColumn(Type.BOOL);
- setUpperBoundInternal(Bytes.fromBoolean(upperBound));
- }
-
- /**
- * Set a byte for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(byte upperBound) {
- checkColumn(Type.INT8);
- setUpperBoundInternal(new byte[] {upperBound});
- }
-
- /**
- * Set a short for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(short upperBound) {
- checkColumn(Type.INT16);
- setUpperBoundInternal(Bytes.fromShort(upperBound));
- }
-
- /**
- * Set an int for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(int upperBound) {
- checkColumn(Type.INT32);
- setUpperBoundInternal(Bytes.fromInt(upperBound));
- }
-
- /**
- * Set a long for the upper bound
- *
- * If 'upperBound' is a timestamp see {@link PartialRow#addLong(String, long)} for the
- * format.
- *
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(long upperBound) {
- checkColumn(Type.INT64, Type.TIMESTAMP);
- setUpperBoundInternal(Bytes.fromLong(upperBound));
- }
-
- /**
- * Set a string for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(String upperBound) {
- checkColumn(Type.STRING);
- setUpperBoundInternal(upperBound.getBytes());
- }
-
- /**
- * Set a binary value for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(byte[] upperBound) {
- checkColumn(Type.BINARY);
- setUpperBoundInternal(upperBound);
- }
-
- /**
- * Set a float for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(float upperBound) {
- checkColumn(Type.FLOAT);
- setUpperBoundInternal(Bytes.fromFloat(upperBound));
- }
-
- /**
- * Set a double for the upper bound
- * @param upperBound value for the upper bound
- */
- public void setUpperBound(double upperBound) {
- checkColumn(Type.DOUBLE);
- setUpperBoundInternal(Bytes.fromDouble(upperBound));
- }
-
- /**
- * Get the column used by this predicate
- * @return the column
- */
- public ColumnSchema getColumn() {
- return column;
- }
-
- /**
- * Get the lower bound in its raw representation
- * @return lower bound as a byte array
- */
- public byte[] getLowerBound() {
- return lowerBound;
- }
-
- /**
- * Get the upper bound in its raw representation
- * @return upper bound as a byte array
- */
- public byte[] getUpperBound() {
- return upperBound;
- }
-
- /**
- * Converts a list of predicates into an opaque byte array. This is a convenience method for use
- * cases that require passing predicates as messages.
- * @param predicates a list of predicates
- * @return an opaque byte array, or null if the list was empty
- */
- public static byte[] toByteArray(List<ColumnRangePredicate> predicates) {
- if (predicates.isEmpty()) {
- return null;
- }
-
- Tserver.ColumnRangePredicateListPB.Builder predicateListBuilder =
- Tserver.ColumnRangePredicateListPB.newBuilder();
-
- for (ColumnRangePredicate crp : predicates) {
- predicateListBuilder.addRangePredicates(crp.getPb());
- }
-
- return predicateListBuilder.build().toByteArray();
- }
-
- /**
- * Converts a given byte array to a list of predicates in their pb format.
- * @param listBytes bytes obtained from {@link #toByteArray(List)}
- * @return a list of predicates
- * @throws IllegalArgumentException thrown when the passed bytes aren't valid
- */
- static List<Tserver.ColumnRangePredicatePB> fromByteArray(byte[] listBytes) {
- List<Tserver.ColumnRangePredicatePB> predicates = new ArrayList<>();
- if (listBytes == null || listBytes.length == 0) {
- return predicates;
- }
- Tserver.ColumnRangePredicateListPB list = ColumnRangePredicate.getPbFromBytes(listBytes);
- return list.getRangePredicatesList();
- }
-
- /**
- * Get the predicate in its protobuf form.
- * @return this predicate in protobuf
- */
- Tserver.ColumnRangePredicatePB getPb() {
- return pb.build();
- }
-
- /**
- * Creates a {@code ColumnRangePredicate} from a protobuf column range predicate message.
- * @param pb the protobuf message
- * @return a column range predicate
- */
- static ColumnRangePredicate fromPb(Tserver.ColumnRangePredicatePB pb) {
- ColumnRangePredicate pred =
- new ColumnRangePredicate(ProtobufHelper.pbToColumnSchema(pb.getColumn()));
- if (pb.hasLowerBound()) {
- pred.setLowerBoundInternal(pb.getLowerBound().toByteArray());
- }
- if (pb.hasInclusiveUpperBound()) {
- pred.setUpperBoundInternal(pb.getInclusiveUpperBound().toByteArray());
- }
- return pred;
- }
-
- /**
- * Convert a list of predicates given in bytes back to its pb format. It also hides the
- * InvalidProtocolBufferException.
- */
- private static Tserver.ColumnRangePredicateListPB getPbFromBytes(byte[] listBytes) {
- try {
- return Tserver.ColumnRangePredicateListPB.parseFrom(listBytes);
- } catch (InvalidProtocolBufferException e) {
- // We shade our pb dependency so we can't send out the exception above since other modules
- // won't know what to expect.
- throw new IllegalArgumentException("Encountered an invalid column range predicate list: "
- + Bytes.pretty(listBytes), e);
- }
- }
-
- private void checkColumn(Type... passedTypes) {
- for (Type type : passedTypes) {
- if (this.column.getType().equals(type)) return;
- }
- throw new IllegalArgumentException(String.format("%s's type isn't %s, it's %s",
- column.getName(), Arrays.toString(passedTypes), column.getType().getName()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java b/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java
deleted file mode 100644
index 20bc4c3..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java
+++ /dev/null
@@ -1,172 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-import java.util.List;
-
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
-
-/**
- * This is a builder class for all the options that can be provided while creating a table.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class CreateTableOptions {
-
- private Master.CreateTableRequestPB.Builder pb = Master.CreateTableRequestPB.newBuilder();
- private final List<PartialRow> splitRows = Lists.newArrayList();
- private final List<Pair<PartialRow, PartialRow>> rangeBounds = Lists.newArrayList();
-
- /**
- * Add a split point for the table. The table in the end will have splits + 1 tablets.
- * The row may be reused or modified safely after this call without changing the split point.
- *
- * @param row a key row for the split point
- * @return this instance
- */
- public CreateTableOptions addSplitRow(PartialRow row) {
- splitRows.add(new PartialRow(row));
- return this;
- }
-
- /**
- * Add a partition range bound to the table with an inclusive lower bound and
- * exclusive upper bound.
- *
- * If either row is empty, then that end of the range will be unbounded. If a
- * range column is missing a value, the logical minimum value for that column
- * type will be used as the default.
- *
- * Multiple range bounds may be added, but they must not overlap. All split
- * rows must fall in one of the range bounds. The lower bound must be less
- * than the upper bound.
- *
- * If not provided, the table's range will be unbounded.
- *
- * @param lower the inclusive lower bound
- * @param upper the exclusive upper bound
- * @return this instance
- */
- public CreateTableOptions addRangeBound(PartialRow lower, PartialRow upper) {
- rangeBounds.add(new Pair<>(new PartialRow(lower), new PartialRow(upper)));
- return this;
- }
-
- /**
- * Add a set of hash partitions to the table.
- *
- * Each column must be a part of the table's primary key, and an individual
- * column may only appear in a single hash component.
- *
- * For each set of hash partitions added to the table, the total number of
- * table partitions is multiplied by the number of buckets. For example, if a
- * table is created with 3 split rows, and two hash partitions with 4 and 5
- * buckets respectively, the total number of table partitions will be 80
- * (4 range partitions * 4 hash buckets * 5 hash buckets).
- *
- * @param columns the columns to hash
- * @param buckets the number of buckets to hash into
- * @return this instance
- */
- public CreateTableOptions addHashPartitions(List<String> columns, int buckets) {
- addHashPartitions(columns, buckets, 0);
- return this;
- }
-
- /**
- * Add a set of hash partitions to the table.
- *
- * This constructor takes a seed value, which can be used to randomize the
- * mapping of rows to hash buckets. Setting the seed may provide some
- * amount of protection against denial of service attacks when the hashed
- * columns contain user provided values.
- *
- * @param columns the columns to hash
- * @param buckets the number of buckets to hash into
- * @param seed a hash seed
- * @return this instance
- */
- public CreateTableOptions addHashPartitions(List<String> columns, int buckets, int seed) {
- Common.PartitionSchemaPB.HashBucketSchemaPB.Builder hashBucket =
- pb.getPartitionSchemaBuilder().addHashBucketSchemasBuilder();
- for (String column : columns) {
- hashBucket.addColumnsBuilder().setName(column);
- }
- hashBucket.setNumBuckets(buckets);
- hashBucket.setSeed(seed);
- return this;
- }
-
- /**
- * Set the columns on which the table will be range-partitioned.
- *
- * Every column must be a part of the table's primary key. If not set or if
- * called with an empty vector, the table will be created without range
- * partitioning.
- *
- * Tables must be created with either range, hash, or range and hash
- * partitioning. To force the use of a single tablet (not recommended),
- * call this method with an empty list and set no split rows and no hash
- * partitions.
- *
- * @param columns the range partitioned columns
- * @return this instance
- */
- public CreateTableOptions setRangePartitionColumns(List<String> columns) {
- Common.PartitionSchemaPB.RangeSchemaPB.Builder rangePartition =
- pb.getPartitionSchemaBuilder().getRangeSchemaBuilder();
- for (String column : columns) {
- rangePartition.addColumnsBuilder().setName(column);
- }
- return this;
- }
-
- /**
- * Sets the number of replicas that each tablet will have. If not specified, it uses the
- * server-side default which is usually 3 unless changed by an administrator.
- *
- * @param numReplicas the number of replicas to use
- * @return this instance
- */
- public CreateTableOptions setNumReplicas(int numReplicas) {
- pb.setNumReplicas(numReplicas);
- return this;
- }
-
- Master.CreateTableRequestPB.Builder getBuilder() {
- if (!splitRows.isEmpty() || !rangeBounds.isEmpty()) {
- pb.setSplitRowsRangeBounds(new Operation.OperationsEncoder()
- .encodeSplitRowsRangeBounds(splitRows, rangeBounds));
- }
- return pb;
- }
-
- List<Integer> getRequiredFeatureFlags() {
- if (rangeBounds.isEmpty()) {
- return ImmutableList.of();
- } else {
- return ImmutableList.of(Master.MasterFeatures.RANGE_PARTITION_BOUNDS_VALUE);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/CreateTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/CreateTableRequest.java b/java/kudu-client/src/main/java/org/kududb/client/CreateTableRequest.java
deleted file mode 100644
index 31ed9a2..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/CreateTableRequest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-
-import java.util.Collection;
-import java.util.List;
-
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * RPC to create new tables
- */
-@InterfaceAudience.Private
-class CreateTableRequest extends KuduRpc<CreateTableResponse> {
-
- static final String CREATE_TABLE = "CreateTable";
-
- private final Schema schema;
- private final String name;
- private final Master.CreateTableRequestPB.Builder builder;
- private final List<Integer> featureFlags;
-
- CreateTableRequest(KuduTable masterTable, String name, Schema schema,
- CreateTableOptions builder) {
- super(masterTable);
- this.schema = schema;
- this.name = name;
- this.builder = builder.getBuilder();
- featureFlags = builder.getRequiredFeatureFlags();
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- this.builder.setName(this.name);
- this.builder.setSchema(ProtobufHelper.schemaToPb(this.schema));
- return toChannelBuffer(header, this.builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return CREATE_TABLE;
- }
-
- @Override
- Pair<CreateTableResponse, Object> deserialize(final CallResponse callResponse,
- String tsUUID) throws Exception {
- final Master.CreateTableResponsePB.Builder builder = Master.CreateTableResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), builder);
- CreateTableResponse response =
- new CreateTableResponse(deadlineTracker.getElapsedMillis(), tsUUID);
- return new Pair<CreateTableResponse, Object>(
- response, builder.hasError() ? builder.getError() : null);
- }
-
- @Override
- Collection<Integer> getRequiredFeatures() {
- return featureFlags;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/CreateTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/CreateTableResponse.java b/java/kudu-client/src/main/java/org/kududb/client/CreateTableResponse.java
deleted file mode 100644
index 7906c5f..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/CreateTableResponse.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-@InterfaceAudience.Private
-public class CreateTableResponse extends KuduRpcResponse {
-
- /**
- * @param ellapsedMillis Time in milliseconds since RPC creation to now.
- */
- CreateTableResponse(long ellapsedMillis, String tsUUID) {
- super(ellapsedMillis, tsUUID);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/DeadlineTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/DeadlineTracker.java b/java/kudu-client/src/main/java/org/kududb/client/DeadlineTracker.java
deleted file mode 100644
index fc30074..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/DeadlineTracker.java
+++ /dev/null
@@ -1,157 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Stopwatch;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * This is a wrapper class around {@link com.google.common.base.Stopwatch} used to track a relative
- * deadline in the future.
- * <p>
- * The watch starts as soon as this object is created with a deadline of 0,
- * meaning that there's no deadline.
- * The deadline has been reached once the stopwatch's elapsed time is equal or greater than the
- * provided deadline.
- */
-public class DeadlineTracker {
- private final Stopwatch stopwatch;
- /** relative deadline in milliseconds **/
- private long deadline = 0;
-
- /**
- * Creates a new tracker, which starts the stopwatch right now.
- */
- public DeadlineTracker() {
- this(Stopwatch.createUnstarted());
- }
-
- /**
- * Creates a new tracker, using the specified stopwatch, and starts it right now.
- * The stopwatch is reset if it was already running.
- * @param stopwatch Specific Stopwatch to use
- */
- public DeadlineTracker(Stopwatch stopwatch) {
- if (stopwatch.isRunning()) {
- stopwatch.reset();
- }
- this.stopwatch = stopwatch.start();
- }
-
- /**
- * Check if we're already past the deadline.
- * @return true if we're past the deadline, otherwise false. Also returns false if no deadline
- * was specified
- */
- public boolean timedOut() {
- if (!hasDeadline()) {
- return false;
- }
- return deadline - stopwatch.elapsed(TimeUnit.MILLISECONDS) <= 0;
- }
-
- /**
- * Get the number of milliseconds before the deadline is reached.
- * <p>
- * This method is used to pass down the remaining deadline to the RPCs, so has special semantics.
- * A deadline of 0 is used to indicate an infinite deadline, and negative deadlines are invalid.
- * Thus, if the deadline has passed (i.e. <tt>deadline - stopwatch.elapsedMillis() <= 0</tt>),
- * the returned value is floored at <tt>1</tt>.
- * <p>
- * Callers who care about this behavior should first check {@link #timedOut()}.
- *
- * @return the remaining millis before the deadline is reached, or 1 if the remaining time is
- * lesser or equal to 0, or Long.MAX_VALUE if no deadline was specified (in which case it
- * should never be called).
- * @throws IllegalStateException if this method is called and no deadline was set
- */
- public long getMillisBeforeDeadline() {
- if (!hasDeadline()) {
- throw new IllegalStateException("This tracker doesn't have a deadline set so it cannot " +
- "answer getMillisBeforeDeadline()");
- }
- long millisBeforeDeadline = deadline - stopwatch.elapsed(TimeUnit.MILLISECONDS);
- millisBeforeDeadline = millisBeforeDeadline <= 0 ? 1 : millisBeforeDeadline;
- return millisBeforeDeadline;
- }
-
- public long getElapsedMillis() {
- return this.stopwatch.elapsed(TimeUnit.MILLISECONDS);
- }
-
- /**
- * Tells if a non-zero deadline was set.
- * @return true if the deadline is greater than 0, false otherwise.
- */
- public boolean hasDeadline() {
- return deadline != 0;
- }
-
- /**
- * Utility method to check if sleeping for a specified amount of time would put us past the
- * deadline.
- * @param plannedSleepTime number of milliseconds for a planned sleep
- * @return if the planned sleeps goes past the deadline.
- */
- public boolean wouldSleepingTimeout(long plannedSleepTime) {
- if (!hasDeadline()) {
- return false;
- }
- return getMillisBeforeDeadline() - plannedSleepTime <= 0;
- }
-
- /**
- * Sets the deadline to 0 (no deadline) and restarts the stopwatch from scratch.
- */
- public void reset() {
- deadline = 0;
- stopwatch.reset();
- stopwatch.start();
- }
-
- /**
- * Get the deadline (in milliseconds).
- * @return the current deadline
- */
- public long getDeadline() {
- return deadline;
- }
-
- /**
- * Set a new deadline for this tracker. It cannot be smaller than 0,
- * and if it is 0 then it means that there is no deadline (which is the default behavior).
- * This method won't call reset().
- * @param deadline a number of milliseconds greater or equal to 0
- * @throws IllegalArgumentException if the deadline is lesser than 0
- */
- public void setDeadline(long deadline) {
- if (deadline < 0) {
- throw new IllegalArgumentException("The deadline must be greater or equal to 0, " +
- "the passed value is " + deadline);
- }
- this.deadline = deadline;
- }
-
- public String toString() {
- StringBuffer buf = new StringBuffer("DeadlineTracker(timeout=");
- buf.append(deadline);
- buf.append(", elapsed=").append(stopwatch.elapsed(TimeUnit.MILLISECONDS));
- buf.append(")");
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Delete.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Delete.java b/java/kudu-client/src/main/java/org/kududb/client/Delete.java
deleted file mode 100644
index 7a068bc..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Delete.java
+++ /dev/null
@@ -1,41 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Class of Operation for whole row removals.
- * Only columns which are part of the key can be set.
- * Instances of this class should not be reused.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Delete extends Operation {
-
- Delete(KuduTable table) {
- super(table);
- }
-
- @Override
- ChangeType getChangeType() {
- return ChangeType.DELETE;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/DeleteTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/DeleteTableRequest.java b/java/kudu-client/src/main/java/org/kududb/client/DeleteTableRequest.java
deleted file mode 100644
index 7f8fa51..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/DeleteTableRequest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * RPC to delete tables
- */
-@InterfaceAudience.Private
-class DeleteTableRequest extends KuduRpc<DeleteTableResponse> {
-
- static final String DELETE_TABLE = "DeleteTable";
-
- private final String name;
-
- DeleteTableRequest(KuduTable table, String name) {
- super(table);
- this.name = name;
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- final Master.DeleteTableRequestPB.Builder builder = Master.DeleteTableRequestPB.newBuilder();
- Master.TableIdentifierPB tableID =
- Master.TableIdentifierPB.newBuilder().setTableName(name).build();
- builder.setTable(tableID);
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return DELETE_TABLE;
- }
-
- @Override
- Pair<DeleteTableResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- final Master.DeleteTableResponsePB.Builder builder = Master.DeleteTableResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), builder);
- DeleteTableResponse response =
- new DeleteTableResponse(deadlineTracker.getElapsedMillis(), tsUUID);
- return new Pair<DeleteTableResponse, Object>(
- response, builder.hasError() ? builder.getError() : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/DeleteTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/DeleteTableResponse.java b/java/kudu-client/src/main/java/org/kududb/client/DeleteTableResponse.java
deleted file mode 100644
index 51f3ba7..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/DeleteTableResponse.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class DeleteTableResponse extends KuduRpcResponse {
-
- /**
- * @param ellapsedMillis Time in milliseconds since RPC creation to now.
- */
- DeleteTableResponse(long ellapsedMillis, String tsUUID) {
- super(ellapsedMillis, tsUUID);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ErrorCollector.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ErrorCollector.java b/java/kudu-client/src/main/java/org/kududb/client/ErrorCollector.java
deleted file mode 100644
index db2952c..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ErrorCollector.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Preconditions;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.util.ArrayDeque;
-import java.util.Queue;
-
-/**
- * Class that helps tracking row errors. All methods are thread-safe.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ErrorCollector {
- private final Queue<RowError> errorQueue;
- private final int maxCapacity;
- private boolean overflowed;
-
- /**
- * Create a new error collector with a maximum capacity.
- * @param maxCapacity how many errors can be stored, has to be higher than 0
- */
- public ErrorCollector(int maxCapacity) {
- Preconditions.checkArgument(maxCapacity > 0, "Need to be able to store at least one row error");
- this.maxCapacity = maxCapacity;
- this.errorQueue = new ArrayDeque<>(maxCapacity);
- }
-
- /**
- * Add a new error to this collector. If it is already at max capacity, the oldest error will be
- * discarded before the new one is added.
- * @param rowError a row error to collect
- */
- public synchronized void addError(RowError rowError) {
- if (errorQueue.size() >= maxCapacity) {
- errorQueue.poll();
- overflowed = true;
- }
- errorQueue.add(rowError);
- }
-
- /**
- * Get the current count of collected row errors. Cannot be greater than the max capacity this
- * instance was configured with.
- * @return the count of errors
- */
- public synchronized int countErrors() {
- return errorQueue.size();
- }
-
- /**
- * Get all the errors that have been collected and an indication if the list overflowed.
- * The list of errors cleared and the overflow state is reset.
- * @return an object that contains both the list of row errors and the overflow status
- */
- public synchronized RowErrorsAndOverflowStatus getErrors() {
- RowError[] returnedErrors = new RowError[errorQueue.size()];
- errorQueue.toArray(returnedErrors);
- errorQueue.clear();
-
- RowErrorsAndOverflowStatus returnObject =
- new RowErrorsAndOverflowStatus(returnedErrors, overflowed);
- overflowed = false;
- return returnObject;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ExternalConsistencyMode.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ExternalConsistencyMode.java b/java/kudu-client/src/main/java/org/kududb/client/ExternalConsistencyMode.java
deleted file mode 100644
index adfb624..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ExternalConsistencyMode.java
+++ /dev/null
@@ -1,42 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * The possible external consistency modes on which Kudu operates.
- * See {@code src/kudu/common/common.proto} for a detailed explanations on the
- * meaning and implications of each mode.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public enum ExternalConsistencyMode {
- CLIENT_PROPAGATED(Common.ExternalConsistencyMode.CLIENT_PROPAGATED),
- COMMIT_WAIT(Common.ExternalConsistencyMode.COMMIT_WAIT);
-
- private Common.ExternalConsistencyMode pbVersion;
- private ExternalConsistencyMode(Common.ExternalConsistencyMode pbVersion) {
- this.pbVersion = pbVersion;
- }
- @InterfaceAudience.Private
- public Common.ExternalConsistencyMode pbVersion() {
- return pbVersion;
- }
-}
[24/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java
new file mode 100644
index 0000000..8a29b7b
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.tablet.Tablet;
+import org.kududb.tserver.Tserver;
+import org.kududb.tserver.TserverService;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@InterfaceAudience.Private
+class ListTabletsRequest extends KuduRpc<ListTabletsResponse> {
+
+ ListTabletsRequest() {
+ super(null);
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ final Tserver.ListTabletsRequestPB.Builder builder =
+ Tserver.ListTabletsRequestPB.newBuilder();
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() { return TABLET_SERVER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return "ListTablets";
+ }
+
+ @Override
+ Pair<ListTabletsResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final Tserver.ListTabletsResponsePB.Builder respBuilder =
+ Tserver.ListTabletsResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), respBuilder);
+ int serversCount = respBuilder.getStatusAndSchemaCount();
+ List<String> tablets = new ArrayList<String>(serversCount);
+ for (Tserver.ListTabletsResponsePB.StatusAndSchemaPB info
+ : respBuilder.getStatusAndSchemaList()) {
+ tablets.add(info.getTabletStatus().getTabletId());
+ }
+ ListTabletsResponse response = new ListTabletsResponse(deadlineTracker.getElapsedMillis(),
+ tsUUID, tablets);
+ return new Pair<ListTabletsResponse, Object>(
+ response, respBuilder.hasError() ? respBuilder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java
new file mode 100644
index 0000000..be2ed65
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+
+import java.util.List;
+
+@InterfaceAudience.Private
+public class ListTabletsResponse extends KuduRpcResponse {
+
+ private final List<String> tabletsList;
+
+ ListTabletsResponse(long ellapsedMillis, String tsUUID, List<String> tabletsList) {
+ super(ellapsedMillis, tsUUID);
+ this.tabletsList = tabletsList;
+ }
+
+ /**
+ * Get the list of tablets as specified in the request.
+ * @return a list of tablet uuids
+ */
+ public List<String> getTabletsList() {
+ return tabletsList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
new file mode 100644
index 0000000..67934db
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
@@ -0,0 +1,132 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.kududb.client;
+
+import java.util.List;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.consensus.Metadata.RaftPeerPB.Role;
+import org.kududb.master.Master.TabletLocationsPB.ReplicaPB;
+
+/**
+ * Information about the locations of tablets in a Kudu table.
+ * This should be treated as immutable data (it does not reflect
+ * any updates the client may have heard since being constructed).
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class LocatedTablet {
+ private final Partition partition;
+ private final byte[] tabletId;
+
+ private final List<Replica> replicas;
+
+ LocatedTablet(AsyncKuduClient.RemoteTablet tablet) {
+ partition = tablet.getPartition();
+ tabletId = tablet.getTabletIdAsBytes();
+ replicas = tablet.getReplicas();
+ }
+
+ public List<Replica> getReplicas() {
+ return replicas;
+ }
+
+ public Partition getPartition() {
+ return partition;
+ }
+
+ /**
+ * DEPRECATED: use {@link #getPartition()}
+ */
+ @Deprecated
+ public byte[] getStartKey() {
+ return getPartition().getPartitionKeyStart();
+ }
+
+ /**
+ * DEPRECATED: use {@link #getPartition()}
+ */
+ @Deprecated()
+ public byte[] getEndKey() {
+ return getPartition().getPartitionKeyEnd();
+ }
+
+ public byte[] getTabletId() {
+ return tabletId;
+ }
+
+ /**
+ * Return the current leader, or null if there is none.
+ */
+ public Replica getLeaderReplica() {
+ return getOneOfRoleOrNull(Role.LEADER);
+ }
+
+ /**
+ * Return the first occurrence for the given role, or null if there is none.
+ */
+ private Replica getOneOfRoleOrNull(Role role) {
+ for (Replica r : replicas) {
+ if (r.getRole() == role.toString()) return r;
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return Bytes.pretty(tabletId) + " " + partition.toString();
+ }
+
+ /**
+ * One of the replicas of the tablet.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public static class Replica {
+ private final ReplicaPB pb;
+
+ Replica(ReplicaPB pb) {
+ this.pb = pb;
+ }
+
+ public String getRpcHost() {
+ if (pb.getTsInfo().getRpcAddressesList().isEmpty()) {
+ return null;
+ }
+ return pb.getTsInfo().getRpcAddressesList().get(0).getHost();
+ }
+
+ public Integer getRpcPort() {
+ if (pb.getTsInfo().getRpcAddressesList().isEmpty()) {
+ return null;
+ }
+ return pb.getTsInfo().getRpcAddressesList().get(0).getPort();
+ }
+
+ public String getRole() {
+ return pb.getRole().toString();
+ }
+
+ public String toString() {
+ return pb.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java
new file mode 100644
index 0000000..1cde694
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.util.List;
+
+/**
+ * Indicates that the request failed because we couldn't find a leader master server.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+final class NoLeaderMasterFoundException extends RecoverableException {
+
+ NoLeaderMasterFoundException(Status status) {
+ super(status);
+ }
+ NoLeaderMasterFoundException(Status status, Exception cause) {
+ super(status, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java
new file mode 100644
index 0000000..1c3b024
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Joiner;
+import com.google.common.primitives.UnsignedBytes;
+
+import java.util.Comparator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A cache of the non-covered range partitions in a Kudu table.
+ *
+ * Currently entries are never invalidated from the cache.
+ */
+@ThreadSafe
+@InterfaceAudience.Private
+class NonCoveredRangeCache {
+ private static final Logger LOG = LoggerFactory.getLogger(NonCoveredRangeCache.class);
+ private static final Comparator<byte[]> COMPARATOR = UnsignedBytes.lexicographicalComparator();
+
+ private final ConcurrentNavigableMap<byte[], byte[]> nonCoveredRanges =
+ new ConcurrentSkipListMap<>(COMPARATOR);
+
+ /**
+ * Retrieves a non-covered range from the cache.
+ *
+ * The pair contains the inclusive start partition key and the exclusive end
+ * partition key containing the provided partition key. If there is no such
+ * cached range, null is returned.
+ *
+ * @param partitionKey the partition key to lookup in the cache
+ * @return the non covered range, or null
+ */
+ public Map.Entry<byte[], byte[]> getNonCoveredRange(byte[] partitionKey) {
+ Map.Entry<byte[], byte[]> range = nonCoveredRanges.floorEntry(partitionKey);
+ if (range == null ||
+ (range.getValue().length != 0 && COMPARATOR.compare(partitionKey, range.getValue()) >= 0)) {
+ return null;
+ } else {
+ return range;
+ }
+ }
+
+ /**
+ * Adds a non-covered range to the cache.
+ *
+ * @param startPartitionKey the inclusive start partition key of the non-covered range
+ * @param endPartitionKey the exclusive end partition key of the non-covered range
+ */
+ public void addNonCoveredRange(byte[] startPartitionKey, byte[] endPartitionKey) {
+ if (startPartitionKey == null || endPartitionKey == null) {
+ throw new IllegalArgumentException("Non-covered partition range keys may not be null");
+ }
+ // Concurrent additions of the same non-covered range key are handled by
+ // serializing puts through the concurrent map.
+ if (nonCoveredRanges.put(startPartitionKey, endPartitionKey) == null) {
+ LOG.info("Discovered non-covered partition range [{}, {})",
+ Bytes.hex(startPartitionKey), Bytes.hex(endPartitionKey));
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('[');
+ boolean isFirst = true;
+ for (Map.Entry<byte[], byte[]> range : nonCoveredRanges.entrySet()) {
+ if (isFirst) {
+ isFirst = false;
+ } else {
+ sb.append(", ");
+ }
+ sb.append('[');
+ sb.append(range.getKey().length == 0 ? "<start>" : Bytes.hex(range.getKey()));
+ sb.append(", ");
+ sb.append(range.getValue().length == 0 ? "<end>" : Bytes.hex(range.getValue()));
+ sb.append(')');
+ }
+ sb.append(']');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java
new file mode 100644
index 0000000..b704441
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Exception indicating that an operation attempted to access a non-covered range partition.
+ */
+@InterfaceAudience.Private
+class NonCoveredRangeException extends NonRecoverableException {
+ private final byte[] nonCoveredRangeStart;
+ private final byte[] nonCoveredRangeEnd;
+
+ public NonCoveredRangeException(byte[] nonCoveredRangeStart, byte[] nonCoveredRangeEnd) {
+ super(Status.NotFound("non-covered range"));
+ this.nonCoveredRangeStart = nonCoveredRangeStart;
+ this.nonCoveredRangeEnd = nonCoveredRangeEnd;
+ }
+
+ byte[] getNonCoveredRangeStart() {
+ return nonCoveredRangeStart;
+ }
+
+ byte[] getNonCoveredRangeEnd() {
+ return nonCoveredRangeEnd;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "NonCoveredRangeException([%s, %s))",
+ nonCoveredRangeStart.length == 0 ? "<start>" : Bytes.hex(nonCoveredRangeStart),
+ nonCoveredRangeEnd.length == 0 ? "<end>" : Bytes.hex(nonCoveredRangeEnd));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java
new file mode 100644
index 0000000..7bcb81d
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+class NonRecoverableException extends KuduException {
+
+ /**
+ * Constructor.
+ * @param status status object containing the reason for the exception
+ * trace.
+ */
+ NonRecoverableException(Status status) {
+ super(status);
+ }
+
+ /**
+ * Constructor.
+ * @param status status object containing the reason for the exception
+ * @param cause The exception that caused this one to be thrown.
+ */
+ NonRecoverableException(Status status, Throwable cause) {
+ super(status, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java
new file mode 100644
index 0000000..e27c222
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java
@@ -0,0 +1,345 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.WireProtocol;
+import org.kududb.WireProtocol.RowOperationsPB;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.Statistics.Statistic;
+import org.kududb.client.Statistics.TabletStatistics;
+import org.kududb.tserver.Tserver;
+import org.kududb.util.Pair;
+import org.kududb.util.Slice;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Base class for the RPCs that related to WriteRequestPB. It contains almost all the logic
+ * and knows how to serialize its child classes.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class Operation extends KuduRpc<OperationResponse> {
+ /**
+ * This size will be set when serialize is called. It stands for the size of the row in this
+ * operation.
+ */
+ private long rowOperationSizeBytes = 0;
+
+ enum ChangeType {
+ INSERT((byte)RowOperationsPB.Type.INSERT.getNumber()),
+ UPDATE((byte)RowOperationsPB.Type.UPDATE.getNumber()),
+ DELETE((byte)RowOperationsPB.Type.DELETE.getNumber()),
+ SPLIT_ROWS((byte)RowOperationsPB.Type.SPLIT_ROW.getNumber()),
+ UPSERT((byte)RowOperationsPB.Type.UPSERT.getNumber()),
+ RANGE_LOWER_BOUND((byte) RowOperationsPB.Type.RANGE_LOWER_BOUND.getNumber()),
+ RANGE_UPPER_BOUND((byte) RowOperationsPB.Type.RANGE_UPPER_BOUND.getNumber());
+
+ ChangeType(byte encodedByte) {
+ this.encodedByte = encodedByte;
+ }
+
+ byte toEncodedByte() {
+ return encodedByte;
+ }
+
+ /** The byte used to encode this in a RowOperationsPB */
+ private byte encodedByte;
+ }
+
+ static final String METHOD = "Write";
+
+ private final PartialRow row;
+
+ /** See {@link SessionConfiguration#setIgnoreAllDuplicateRows(boolean)} */
+ boolean ignoreAllDuplicateRows = false;
+
+ /**
+ * Package-private constructor. Subclasses need to be instantiated via AsyncKuduSession
+ * @param table table with the schema to use for this operation
+ */
+ Operation(KuduTable table) {
+ super(table);
+ this.row = table.getSchema().newPartialRow();
+ }
+
+ /** See {@link SessionConfiguration#setIgnoreAllDuplicateRows(boolean)} */
+ void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows) {
+ this.ignoreAllDuplicateRows = ignoreAllDuplicateRows;
+ }
+
+ /**
+ * Classes extending Operation need to have a specific ChangeType
+ * @return Operation's ChangeType
+ */
+ abstract ChangeType getChangeType();
+
+ /**
+ * Returns the size in bytes of this operation's row after serialization.
+ * @return size in bytes
+ * @throws IllegalStateException thrown if this RPC hasn't been serialized eg sent to a TS
+ */
+ long getRowOperationSizeBytes() {
+ if (this.rowOperationSizeBytes == 0) {
+ throw new IllegalStateException("This row hasn't been serialized yet");
+ }
+ return this.rowOperationSizeBytes;
+ }
+
+ @Override
+ String serviceName() { return TABLET_SERVER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return METHOD;
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ final Tserver.WriteRequestPB.Builder builder =
+ createAndFillWriteRequestPB(ImmutableList.of(this));
+ this.rowOperationSizeBytes = builder.getRowOperations().getRows().size()
+ + builder.getRowOperations().getIndirectData().size();
+ builder.setTabletId(ZeroCopyLiteralByteString.wrap(getTablet().getTabletIdAsBytes()));
+ builder.setExternalConsistencyMode(this.externalConsistencyMode.pbVersion());
+ if (this.propagatedTimestamp != AsyncKuduClient.NO_TIMESTAMP) {
+ builder.setPropagatedTimestamp(this.propagatedTimestamp);
+ }
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ Pair<OperationResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ Tserver.WriteResponsePB.Builder builder = Tserver.WriteResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), builder);
+ Tserver.WriteResponsePB.PerRowErrorPB error = null;
+ if (builder.getPerRowErrorsCount() != 0) {
+ error = builder.getPerRowErrors(0);
+ if (ignoreAllDuplicateRows &&
+ error.getError().getCode() == WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT) {
+ error = null;
+ }
+ }
+ OperationResponse response = new OperationResponse(deadlineTracker.getElapsedMillis(), tsUUID,
+ builder.getTimestamp(), this, error);
+ return new Pair<OperationResponse, Object>(
+ response, builder.hasError() ? builder.getError() : null);
+ }
+
+ @Override
+ public byte[] partitionKey() {
+ return this.getTable().getPartitionSchema().encodePartitionKey(row);
+ }
+
+ @Override
+ boolean isRequestTracked() {
+ return true;
+ }
+
+ /**
+ * Get the underlying row to modify.
+ * @return a partial row that will be sent with this Operation
+ */
+ public PartialRow getRow() {
+ return this.row;
+ }
+
+ @Override
+ void updateStatistics(Statistics statistics, OperationResponse response) {
+ Slice tabletId = this.getTablet().getTabletId();
+ String tableName = this.getTable().getName();
+ TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId);
+ if (response == null) {
+ tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1);
+ tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1);
+ return;
+ }
+ tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1);
+ if (response.hasRowError()) {
+ // If ignoreAllDuplicateRows is set, the already_present exception will be
+ // discarded and wont't be recorded here
+ tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1);
+ } else {
+ tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1);
+ }
+ tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationSizeBytes());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(super.toString());
+ sb.append(" row_key=");
+ sb.append(row.stringifyRowKey());
+ return sb.toString();
+ }
+
+ /**
+ * Helper method that puts a list of Operations together into a WriteRequestPB.
+ * @param operations The list of ops to put together in a WriteRequestPB
+ * @return A fully constructed WriteRequestPB containing the passed rows, or
+ * null if no rows were passed.
+ */
+ static Tserver.WriteRequestPB.Builder createAndFillWriteRequestPB(List<Operation> operations) {
+ if (operations == null || operations.isEmpty()) return null;
+ Schema schema = operations.get(0).table.getSchema();
+ RowOperationsPB rowOps = new OperationsEncoder().encodeOperations(operations);
+ if (rowOps == null) return null;
+
+ Tserver.WriteRequestPB.Builder requestBuilder = Tserver.WriteRequestPB.newBuilder();
+ requestBuilder.setSchema(ProtobufHelper.schemaToPb(schema));
+ requestBuilder.setRowOperations(rowOps);
+ return requestBuilder;
+ }
+
+ static class OperationsEncoder {
+ private Schema schema;
+ private ByteBuffer rows;
+ // We're filling this list as we go through the operations in encodeRow() and at the same time
+ // compute the total size, which will be used to right-size the array in toPB().
+ private List<ByteBuffer> indirect;
+ private long indirectWrittenBytes;
+
+ /**
+ * Initializes the state of the encoder based on the schema and number of operations to encode.
+ *
+ * @param schema the schema of the table which the operations belong to.
+ * @param numOperations the number of operations.
+ */
+ private void init(Schema schema, int numOperations) {
+ this.schema = schema;
+
+ // Set up the encoded data.
+ // Estimate a maximum size for the data. This is conservative, but avoids
+ // having to loop through all the operations twice.
+ final int columnBitSetSize = Bytes.getBitSetSize(schema.getColumnCount());
+ int sizePerRow = 1 /* for the op type */ + schema.getRowSize() + columnBitSetSize;
+ if (schema.hasNullableColumns()) {
+ // nullsBitSet is the same size as the columnBitSet
+ sizePerRow += columnBitSetSize;
+ }
+
+ // TODO: would be more efficient to use a buffer which "chains" smaller allocations
+ // instead of a doubling buffer like BAOS.
+ this.rows = ByteBuffer.allocate(sizePerRow * numOperations)
+ .order(ByteOrder.LITTLE_ENDIAN);
+ this.indirect = new ArrayList<>(schema.getVarLengthColumnCount() * numOperations);
+ }
+
+ /**
+ * Builds the row operations protobuf message with encoded operations.
+ * @return the row operations protobuf message.
+ */
+ private RowOperationsPB toPB() {
+ RowOperationsPB.Builder rowOpsBuilder = RowOperationsPB.newBuilder();
+
+ // TODO: we could implement a ZeroCopy approach here by subclassing LiteralByteString.
+ // We have ZeroCopyLiteralByteString, but that only supports an entire array. Here
+ // we've only partially filled in rows.array(), so we have to make the extra copy.
+ rows.limit(rows.position());
+ rows.flip();
+ rowOpsBuilder.setRows(ByteString.copyFrom(rows));
+ if (indirect.size() > 0) {
+ // TODO: same as above, we could avoid a copy here by using an implementation that allows
+ // zero-copy on a slice of an array.
+ byte[] indirectData = new byte[(int)indirectWrittenBytes];
+ int offset = 0;
+ for (ByteBuffer bb : indirect) {
+ int bbSize = bb.remaining();
+ bb.get(indirectData, offset, bbSize);
+ offset += bbSize;
+ }
+ rowOpsBuilder.setIndirectData(ZeroCopyLiteralByteString.wrap(indirectData));
+ }
+ return rowOpsBuilder.build();
+ }
+
+ private void encodeRow(PartialRow row, ChangeType type) {
+ rows.put(type.toEncodedByte());
+ rows.put(Bytes.fromBitSet(row.getColumnsBitSet(), schema.getColumnCount()));
+ if (schema.hasNullableColumns()) {
+ rows.put(Bytes.fromBitSet(row.getNullsBitSet(), schema.getColumnCount()));
+ }
+ int colIdx = 0;
+ byte[] rowData = row.getRowAlloc();
+ int currentRowOffset = 0;
+ for (ColumnSchema col : row.getSchema().getColumns()) {
+ // Keys should always be specified, maybe check?
+ if (row.isSet(colIdx) && !row.isSetToNull(colIdx)) {
+ if (col.getType() == Type.STRING || col.getType() == Type.BINARY) {
+ ByteBuffer varLengthData = row.getVarLengthData().get(colIdx);
+ varLengthData.reset();
+ rows.putLong(indirectWrittenBytes);
+ int bbSize = varLengthData.remaining();
+ rows.putLong(bbSize);
+ indirect.add(varLengthData);
+ indirectWrittenBytes += bbSize;
+ } else {
+ // This is for cols other than strings
+ rows.put(rowData, currentRowOffset, col.getType().getSize());
+ }
+ }
+ currentRowOffset += col.getType().getSize();
+ colIdx++;
+ }
+ }
+
+ public RowOperationsPB encodeOperations(List<Operation> operations) {
+ if (operations == null || operations.isEmpty()) return null;
+ init(operations.get(0).table.getSchema(), operations.size());
+ for (Operation operation : operations) {
+ encodeRow(operation.row, operation.getChangeType());
+ }
+ return toPB();
+ }
+
+ public RowOperationsPB encodeSplitRowsRangeBounds(List<PartialRow> splitRows,
+ List<Pair<PartialRow, PartialRow>> rangeBounds) {
+ if (splitRows.isEmpty() && rangeBounds.isEmpty()) {
+ return null;
+ }
+
+ Schema schema = splitRows.isEmpty() ? rangeBounds.get(0).getFirst().getSchema()
+ : splitRows.get(0).getSchema();
+ init(schema, splitRows.size() + 2 * rangeBounds.size());
+
+ for (PartialRow row : splitRows) {
+ encodeRow(row, ChangeType.SPLIT_ROWS);
+ }
+
+ for (Pair<PartialRow, PartialRow> bound : rangeBounds) {
+ encodeRow(bound.getFirst(), ChangeType.RANGE_LOWER_BOUND);
+ encodeRow(bound.getSecond(), ChangeType.RANGE_UPPER_BOUND);
+ }
+
+ return toPB();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java
new file mode 100644
index 0000000..bf707ce
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.tserver.Tserver;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class OperationResponse extends KuduRpcResponse {
+
+ private final long writeTimestamp;
+ private final RowError rowError;
+ private final Operation operation;
+
+ /**
+ * Package-private constructor to build an OperationResponse with a row error in the pb format.
+ * @param elapsedMillis time in milliseconds since RPC creation to now
+ * @param writeTimestamp HT's write timestamp
+ * @param operation the operation that created this response
+ * @param errorPB a row error in pb format, can be null
+ */
+ OperationResponse(long elapsedMillis, String tsUUID, long writeTimestamp,
+ Operation operation, Tserver.WriteResponsePB.PerRowErrorPB errorPB) {
+ super(elapsedMillis, tsUUID);
+ this.writeTimestamp = writeTimestamp;
+ this.rowError = errorPB == null ? null : RowError.fromRowErrorPb(errorPB, operation, tsUUID);
+ this.operation = operation;
+ }
+
+ /**
+ * Package-private constructor to build an OperationResponse with a row error.
+ * @param elapsedMillis time in milliseconds since RPC creation to now
+ * @param writeTimestamp HT's write timestamp
+ * @param operation the operation that created this response
+ * @param rowError a parsed row error, can be null
+ */
+ OperationResponse(long elapsedMillis, String tsUUID, long writeTimestamp,
+ Operation operation, RowError rowError) {
+ super(elapsedMillis, tsUUID);
+ this.writeTimestamp = writeTimestamp;
+ this.rowError = rowError;
+ this.operation = operation;
+ }
+
+ /**
+ * Utility method that collects all the row errors from the given list of responses.
+ * @param responses a list of operation responses to collect the row errors from
+ * @return a combined list of row errors
+ */
+ public static List<RowError> collectErrors(List<OperationResponse> responses) {
+ List<RowError> errors = new ArrayList<>(responses.size());
+ for (OperationResponse resp : responses) {
+ if (resp.hasRowError()) {
+ errors.add(resp.getRowError());
+ }
+ }
+ return errors;
+ }
+
+ /**
+ * Gives the write timestamp that was returned by the Tablet Server.
+ * @return a timestamp in milliseconds, 0 if the external consistency mode set
+ * in AsyncKuduSession wasn't CLIENT_PROPAGATED, or if the operation failed.
+ */
+ public long getWriteTimestamp() {
+ return writeTimestamp;
+ }
+
+ /**
+ * Returns a row error. If {@link #hasRowError()} returns false, then this method returns null.
+ * @return a row error, or null if the operation was successful
+ */
+ public RowError getRowError() {
+ return rowError;
+ }
+
+ /**
+ * Tells if this operation response contains a row error.
+ * @return true if this operation response has errors, else false
+ */
+ public boolean hasRowError() {
+ return rowError != null;
+ }
+
+ /**
+ * Returns the operation associated with this response.
+ * @return an operation, cannot be null
+ */
+ Operation getOperation() {
+ return operation;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java b/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
new file mode 100644
index 0000000..b5f3069
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
@@ -0,0 +1,626 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Class used to represent parts of a row along with its schema.<p>
+ *
+ * Values can be replaced as often as needed, but once the enclosing {@link Operation} is applied
+ * then they cannot be changed again. This means that a PartialRow cannot be reused.<p>
+ *
+ * Each PartialRow is backed by an byte array where all the cells (except strings and binary data)
+ * are written. The others are kept in a List.<p>
+ *
+ * This class isn't thread-safe.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class PartialRow {
+
+ private final Schema schema;
+
+ // Variable length data. If string, will be UTF-8 encoded. Elements of this list _must_ have a
+ // mark that we can reset() to. Readers of these fields (encoders, etc) must call reset() before
+ // attempting to read these values.
+ private final List<ByteBuffer> varLengthData;
+ private final byte[] rowAlloc;
+
+ private final BitSet columnsBitSet;
+ private final BitSet nullsBitSet;
+
+ private boolean frozen = false;
+
+ /**
+ * This is not a stable API, prefer using {@link Schema#newPartialRow()}
+ * to create a new partial row.
+ * @param schema the schema to use for this row
+ */
+ public PartialRow(Schema schema) {
+ this.schema = schema;
+ this.columnsBitSet = new BitSet(this.schema.getColumnCount());
+ this.nullsBitSet = schema.hasNullableColumns() ?
+ new BitSet(this.schema.getColumnCount()) : null;
+ this.rowAlloc = new byte[schema.getRowSize()];
+ // Pre-fill the array with nulls. We'll only replace cells that have varlen values.
+ this.varLengthData = Arrays.asList(new ByteBuffer[this.schema.getColumnCount()]);
+ }
+
+ /**
+ * Creates a new partial row by deep-copying the data-fields of the provided partial row.
+ * @param row the partial row to copy
+ */
+ PartialRow(PartialRow row) {
+ this.schema = row.schema;
+
+ this.varLengthData = Lists.newArrayListWithCapacity(row.varLengthData.size());
+ for (ByteBuffer data: row.varLengthData) {
+ if (data == null) {
+ this.varLengthData.add(null);
+ } else {
+ data.reset();
+ // Deep copy the ByteBuffer.
+ ByteBuffer clone = ByteBuffer.allocate(data.remaining());
+ clone.put(data);
+ clone.flip();
+
+ clone.mark(); // We always expect a mark.
+ this.varLengthData.add(clone);
+ }
+ }
+
+ this.rowAlloc = row.rowAlloc.clone();
+ this.columnsBitSet = (BitSet) row.columnsBitSet.clone();
+ this.nullsBitSet = row.nullsBitSet == null ? null : (BitSet) row.nullsBitSet.clone();
+ }
+
+ /**
+ * Add a boolean for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addBoolean(int columnIndex, boolean val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.BOOL);
+ rowAlloc[getPositionInRowAllocAndSetBitSet(columnIndex)] = (byte) (val ? 1 : 0);
+ }
+
+ /**
+ * Add a boolean for the specified column.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addBoolean(String columnName, boolean val) {
+ addBoolean(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add a byte for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addByte(int columnIndex, byte val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.INT8);
+ rowAlloc[getPositionInRowAllocAndSetBitSet(columnIndex)] = val;
+ }
+
+ /**
+ * Add a byte for the specified column.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addByte(String columnName, byte val) {
+ addByte(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add a short for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addShort(int columnIndex, short val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.INT16);
+ Bytes.setShort(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
+ }
+
+ /**
+ * Add a short for the specified column.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addShort(String columnName, short val) {
+ addShort(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add an int for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addInt(int columnIndex, int val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.INT32);
+ Bytes.setInt(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
+ }
+
+ /**
+ * Add an int for the specified column.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addInt(String columnName, int val) {
+ addInt(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add an long for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addLong(int columnIndex, long val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.INT64, Type.TIMESTAMP);
+ Bytes.setLong(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
+ }
+
+ /**
+ * Add an long for the specified column.
+ *
+ * If this is a TIMESTAMP column, the long value provided should be the number of microseconds
+ * between a given time and January 1, 1970 UTC.
+ * For example, to encode the current time, use setLong(System.currentTimeMillis() * 1000);
+ *
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addLong(String columnName, long val) {
+ addLong(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add an float for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addFloat(int columnIndex, float val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.FLOAT);
+ Bytes.setFloat(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
+ }
+
+ /**
+ * Add an float for the specified column.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addFloat(String columnName, float val) {
+ addFloat(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add an double for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addDouble(int columnIndex, double val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.DOUBLE);
+ Bytes.setDouble(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
+ }
+
+ /**
+ * Add an double for the specified column.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addDouble(String columnName, double val) {
+ addDouble(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add a String for the specified column.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addString(int columnIndex, String val) {
+ addStringUtf8(columnIndex, Bytes.fromString(val));
+ }
+
+ /**
+ * Add a String for the specified column.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addString(String columnName, String val) {
+ addStringUtf8(columnName, Bytes.fromString(val));
+ }
+
+ /**
+ * Add a String for the specified value, encoded as UTF8.
+ * Note that the provided value must not be mutated after this.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addStringUtf8(int columnIndex, byte[] val) {
+ // TODO: use Utf8.isWellFormed from Guava 16 to verify that
+ // the user isn't putting in any garbage data.
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.STRING);
+ addVarLengthData(columnIndex, val);
+ }
+
+ /**
+ * Add a String for the specified value, encoded as UTF8.
+ * Note that the provided value must not be mutated after this.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ *
+ */
+ public void addStringUtf8(String columnName, byte[] val) {
+ addStringUtf8(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add binary data with the specified value.
+ * Note that the provided value must not be mutated after this.
+ * @param columnIndex the column's index in the schema
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addBinary(int columnIndex, byte[] val) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.BINARY);
+ addVarLengthData(columnIndex, val);
+ }
+
+ /**
+ * Add binary data with the specified value, from the current ByteBuffer's position to its limit.
+ * This method duplicates the ByteBuffer but doesn't copy the data. This means that the wrapped
+ * data must not be mutated after this.
+ * @param columnIndex the column's index in the schema
+ * @param value byte buffer to get the value from
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addBinary(int columnIndex, ByteBuffer value) {
+ checkColumn(schema.getColumnByIndex(columnIndex), Type.BINARY);
+ addVarLengthData(columnIndex, value);
+ }
+
+ /**
+ * Add binary data with the specified value.
+ * Note that the provided value must not be mutated after this.
+ * @param columnName Name of the column
+ * @param val value to add
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addBinary(String columnName, byte[] val) {
+ addBinary(schema.getColumnIndex(columnName), val);
+ }
+
+ /**
+ * Add binary data with the specified value, from the current ByteBuffer's position to its limit.
+ * This method duplicates the ByteBuffer but doesn't copy the data. This means that the wrapped
+ * data must not be mutated after this.
+ * @param columnName Name of the column
+ * @param value byte buffer to get the value from
+ * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
+ * the column's type
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void addBinary(String columnName, ByteBuffer value) {
+ addBinary(schema.getColumnIndex(columnName), value);
+ }
+
+ private void addVarLengthData(int columnIndex, byte[] val) {
+ addVarLengthData(columnIndex, ByteBuffer.wrap(val));
+ }
+
+ private void addVarLengthData(int columnIndex, ByteBuffer val) {
+ // A duplicate will copy all the original's metadata but still point to the same content.
+ ByteBuffer duplicate = val.duplicate();
+ // Mark the current position so we can reset to it.
+ duplicate.mark();
+
+ varLengthData.set(columnIndex, duplicate);
+ // Set the usage bit but we don't care where it is.
+ getPositionInRowAllocAndSetBitSet(columnIndex);
+ // We don't set anything in row alloc, it will be managed at encoding time.
+ }
+
+ /**
+ * Set the specified column to null
+ * @param columnIndex the column's index in the schema
+ * @throws IllegalArgumentException if the column doesn't exist or cannot be set to null
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void setNull(int columnIndex) {
+ setNull(this.schema.getColumnByIndex(columnIndex));
+ }
+
+ /**
+ * Set the specified column to null
+ * @param columnName Name of the column
+ * @throws IllegalArgumentException if the column doesn't exist or cannot be set to null
+ * @throws IllegalStateException if the row was already applied
+ */
+ public void setNull(String columnName) {
+ setNull(this.schema.getColumn(columnName));
+ }
+
+ private void setNull(ColumnSchema column) {
+ assert nullsBitSet != null;
+ checkNotFrozen();
+ checkColumnExists(column);
+ if (!column.isNullable()) {
+ throw new IllegalArgumentException(column.getName() + " cannot be set to null");
+ }
+ int idx = schema.getColumns().indexOf(column);
+ columnsBitSet.set(idx);
+ nullsBitSet.set(idx);
+ }
+
+ /**
+ * Verifies if the column exists and belongs to one of the specified types
+ * It also does some internal accounting
+ * @param column column the user wants to set
+ * @param types types we expect
+ * @throws IllegalArgumentException if the column or type was invalid
+ * @throws IllegalStateException if the row was already applied
+ */
+ private void checkColumn(ColumnSchema column, Type... types) {
+ checkNotFrozen();
+ checkColumnExists(column);
+ for(Type type : types) {
+ if (column.getType().equals(type)) return;
+ }
+ throw new IllegalArgumentException(String.format("%s isn't %s, it's %s", column.getName(),
+ Arrays.toString(types), column.getType().getName()));
+ }
+
+ /**
+ * @param column column the user wants to set
+ * @throws IllegalArgumentException if the column doesn't exist
+ */
+ private void checkColumnExists(ColumnSchema column) {
+ if (column == null)
+ throw new IllegalArgumentException("Column name isn't present in the table's schema");
+ }
+
+ /**
+ * @throws IllegalStateException if the row was already applied
+ */
+ private void checkNotFrozen() {
+ if (frozen) {
+ throw new IllegalStateException("This row was already applied and cannot be modified.");
+ }
+ }
+
+ /**
+ * Sets the column bit set for the column index, and returns the column's offset.
+ * @param columnIndex the index of the column to get the position for and mark as set
+ * @return the offset in rowAlloc for the column
+ */
+ private int getPositionInRowAllocAndSetBitSet(int columnIndex) {
+ columnsBitSet.set(columnIndex);
+ return schema.getColumnOffset(columnIndex);
+ }
+
+ /**
+ * Tells if the specified column was set by the user
+ * @param column column's index in the schema
+ * @return true if it was set, else false
+ */
+ boolean isSet(int column) {
+ return this.columnsBitSet.get(column);
+ }
+
+ /**
+ * Tells if the specified column was set to null by the user
+ * @param column column's index in the schema
+ * @return true if it was set, else false
+ */
+ boolean isSetToNull(int column) {
+ if (this.nullsBitSet == null) {
+ return false;
+ }
+ return this.nullsBitSet.get(column);
+ }
+
+ /**
+ * Returns the encoded primary key of the row.
+ * @return a byte array containing an encoded primary key
+ */
+ public byte[] encodePrimaryKey() {
+ return new KeyEncoder().encodePrimaryKey(this);
+ }
+
+ /**
+ * Transforms the row key into a string representation where each column is in the format:
+ * "type col_name=value".
+ * @return a string representation of the operation's row key
+ */
+ public String stringifyRowKey() {
+ int numRowKeys = schema.getPrimaryKeyColumnCount();
+ StringBuilder sb = new StringBuilder();
+ sb.append("(");
+ for (int i = 0; i < numRowKeys; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+
+ ColumnSchema col = schema.getColumnByIndex(i);
+ assert !col.isNullable();
+ Preconditions.checkState(columnsBitSet.get(i),
+ "Full row key not specified, missing at least col: " + col.getName());
+ Type type = col.getType();
+ sb.append(type.getName());
+ sb.append(" ");
+ sb.append(col.getName());
+ sb.append("=");
+
+ if (type == Type.STRING || type == Type.BINARY) {
+ ByteBuffer value = getVarLengthData().get(i).duplicate();
+ value.reset(); // Make sure we start at the beginning.
+ byte[] data = new byte[value.limit()];
+ value.get(data);
+ if (type == Type.STRING) {
+ sb.append(Bytes.getString(data));
+ } else {
+ sb.append(Bytes.pretty(data));
+ }
+ } else {
+ switch (type) {
+ case INT8:
+ sb.append(Bytes.getByte(rowAlloc, schema.getColumnOffset(i)));
+ break;
+ case INT16:
+ sb.append(Bytes.getShort(rowAlloc, schema.getColumnOffset(i)));
+ break;
+ case INT32:
+ sb.append(Bytes.getInt(rowAlloc, schema.getColumnOffset(i)));
+ break;
+ case INT64:
+ sb.append(Bytes.getLong(rowAlloc, schema.getColumnOffset(i)));
+ break;
+ case TIMESTAMP:
+ sb.append(Bytes.getLong(rowAlloc, schema.getColumnOffset(i)));
+ break;
+ default:
+ throw new IllegalArgumentException(String.format(
+ "The column type %s is not a valid key component type", type));
+ }
+ }
+ }
+ sb.append(")");
+
+ return sb.toString();
+ }
+
+ /**
+ * Get the schema used for this row.
+ * @return a schema that came from KuduTable
+ */
+ Schema getSchema() {
+ return schema;
+ }
+
+ /**
+ * Get the list variable length data cells that were added to this row.
+ * @return a list of binary data, may be empty
+ */
+ List<ByteBuffer> getVarLengthData() {
+ return varLengthData;
+ }
+
+ /**
+ * Get the byte array that contains all the data added to this partial row. Variable length data
+ * is contained separately, see {@link #getVarLengthData()}. In their place you'll find their
+ * index in that list and their size.
+ * @return a byte array containing the data for this row, except strings
+ */
+ byte[] getRowAlloc() {
+ return rowAlloc;
+ }
+
+ /**
+ * Get the bit set that indicates which columns were set.
+ * @return a bit set for columns with data
+ */
+ BitSet getColumnsBitSet() {
+ return columnsBitSet;
+ }
+
+ /**
+ * Get the bit set for the columns that were specifically set to null
+ * @return a bit set for null columns
+ */
+ BitSet getNullsBitSet() {
+ return nullsBitSet;
+ }
+
+ /**
+ * Prevents this PartialRow from being modified again. Can be called multiple times.
+ */
+ void freeze() {
+ this.frozen = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java
new file mode 100644
index 0000000..bdc089b
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Objects;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A Partition describes the set of rows that a Tablet is responsible for
+ * serving. Each tablet is assigned a single Partition.<p>
+ *
+ * Partitions consist primarily of a start and end partition key. Every row with
+ * a partition key that falls in a Tablet's Partition will be served by that
+ * tablet.<p>
+ *
+ * In addition to the start and end partition keys, a Partition holds metadata
+ * to determine if a scan can prune, or skip, a partition based on the scan's
+ * start and end primary keys, and predicates.
+ *
+ * This class is new, and not considered stable or suitable for public use.
+ */
+@InterfaceAudience.LimitedPrivate("Impala")
+@InterfaceStability.Unstable
+public class Partition implements Comparable<Partition> {
+ final byte[] partitionKeyStart;
+ final byte[] partitionKeyEnd;
+
+ final byte[] rangeKeyStart;
+ final byte[] rangeKeyEnd;
+
+ final List<Integer> hashBuckets;
+
+ /**
+ * Size of an encoded hash bucket component in a partition key.
+ */
+ private static final int ENCODED_BUCKET_SIZE = 4;
+
+ /**
+ * Creates a new partition with the provided start and end keys, and hash buckets.
+ * @param partitionKeyStart the start partition key
+ * @param partitionKeyEnd the end partition key
+ * @param hashBuckets the partition hash buckets
+ */
+ Partition(byte[] partitionKeyStart,
+ byte[] partitionKeyEnd,
+ List<Integer> hashBuckets) {
+ this.partitionKeyStart = partitionKeyStart;
+ this.partitionKeyEnd = partitionKeyEnd;
+ this.hashBuckets = hashBuckets;
+ this.rangeKeyStart = rangeKey(partitionKeyStart, hashBuckets.size());
+ this.rangeKeyEnd = rangeKey(partitionKeyEnd, hashBuckets.size());
+ }
+
+ /**
+ * Gets the start partition key.
+ * @return the start partition key
+ */
+ public byte[] getPartitionKeyStart() {
+ return partitionKeyStart;
+ }
+
+ /**
+ * Gets the end partition key.
+ * @return the end partition key
+ */
+ public byte[] getPartitionKeyEnd() {
+ return partitionKeyEnd;
+ }
+
+ /**
+ * Gets the start range key.
+ * @return the start range key
+ */
+ public byte[] getRangeKeyStart() {
+ return rangeKeyStart;
+ }
+
+ /**
+ * Gets the end range key.
+ * @return the end range key
+ */
+ public byte[] getRangeKeyEnd() {
+ return rangeKeyEnd;
+ }
+
+ /**
+ * Gets the partition hash buckets.
+ * @return the partition hash buckets
+ */
+ public List<Integer> getHashBuckets() {
+ return hashBuckets;
+ }
+
+ /**
+ * @return true if the partition is the absolute end partition
+ */
+ public boolean isEndPartition() {
+ return partitionKeyEnd.length == 0;
+ }
+
+ /**
+ * Equality only holds for partitions from the same table. Partition equality only takes into
+ * account the partition keys, since there is a 1 to 1 correspondence between partition keys and
+ * the hash buckets and range keys.
+ *
+ * @return the hash code
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Partition partition = (Partition) o;
+ return Arrays.equals(partitionKeyStart, partition.partitionKeyStart)
+ && Arrays.equals(partitionKeyEnd, partition.partitionKeyEnd);
+ }
+
+ /**
+ * The hash code only takes into account the partition keys, since there is a 1 to 1
+ * correspondence between partition keys and the hash buckets and range keys.
+ *
+ * @return the hash code
+ */
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(Arrays.hashCode(partitionKeyStart), Arrays.hashCode(partitionKeyEnd));
+ }
+
+ /**
+ * Partition comparison is only reasonable when comparing partitions from the same table, and
+ * since Kudu does not yet allow partition splitting, no two distinct partitions can have the
+ * same start partition key. Accordingly, partitions are compared strictly by the start partition
+ * key.
+ *
+ * @param other the other partition of the same table
+ * @return the comparison of the partitions
+ */
+ @Override
+ public int compareTo(Partition other) {
+ return Bytes.memcmp(this.partitionKeyStart, other.partitionKeyStart);
+ }
+
+ /**
+ * Returns the range key portion of a partition key given the number of buckets in the partition
+ * schema.
+ * @param partitionKey the partition key containing the range key
+ * @param numHashBuckets the number of hash bucket components of the table
+ * @return the range key
+ */
+ private static byte[] rangeKey(byte[] partitionKey, int numHashBuckets) {
+ int bucketsLen = numHashBuckets * ENCODED_BUCKET_SIZE;
+ if (partitionKey.length > bucketsLen) {
+ return Arrays.copyOfRange(partitionKey, bucketsLen, partitionKey.length);
+ } else {
+ return AsyncKuduClient.EMPTY_ARRAY;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format("[%s, %s)",
+ partitionKeyStart.length == 0 ? "<start>" : Bytes.hex(partitionKeyStart),
+ partitionKeyEnd.length == 0 ? "<end>" : Bytes.hex(partitionKeyEnd));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java b/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
new file mode 100644
index 0000000..fdee32e
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
@@ -0,0 +1,142 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.util.List;
+
+/**
+ * A partition schema describes how the rows of a table are distributed among
+ * tablets.
+ *
+ * Primarily, a table's partition schema is responsible for translating the
+ * primary key column values of a row into a partition key that can be used to
+ * find the tablet containing the key.
+ *
+ * The partition schema is made up of zero or more hash bucket components,
+ * followed by a single range component.
+ *
+ * Each hash bucket component includes one or more columns from the primary key
+ * column set, with the restriction that an individual primary key column may
+ * only be included in a single hash component.
+ *
+ * This class is new, and not considered stable or suitable for public use.
+ */
+@InterfaceAudience.LimitedPrivate("Impala")
+@InterfaceStability.Unstable
+public class PartitionSchema {
+
+ private final RangeSchema rangeSchema;
+ private final List<HashBucketSchema> hashBucketSchemas;
+ private final boolean isSimple;
+
+ /**
+ * Creates a new partition schema from the range and hash bucket schemas.
+ *
+ * @param rangeSchema the range schema
+ * @param hashBucketSchemas the hash bucket schemas
+ * @param schema the table schema
+ */
+ PartitionSchema(RangeSchema rangeSchema,
+ List<HashBucketSchema> hashBucketSchemas,
+ Schema schema) {
+ this.rangeSchema = rangeSchema;
+ this.hashBucketSchemas = hashBucketSchemas;
+
+ boolean isSimple = hashBucketSchemas.isEmpty()
+ && rangeSchema.columns.size() == schema.getPrimaryKeyColumnCount();
+ if (isSimple) {
+ int i = 0;
+ for (Integer id : rangeSchema.columns) {
+ if (schema.getColumnIndex(id) != i++) {
+ isSimple = false;
+ break;
+ }
+ }
+ }
+ this.isSimple = isSimple;
+ }
+
+ /**
+ * Returns the encoded partition key of the row.
+ * @return a byte array containing the encoded partition key of the row
+ */
+ public byte[] encodePartitionKey(PartialRow row) {
+ return new KeyEncoder().encodePartitionKey(row, this);
+ }
+
+ public RangeSchema getRangeSchema() {
+ return rangeSchema;
+ }
+
+ public List<HashBucketSchema> getHashBucketSchemas() {
+ return hashBucketSchemas;
+ }
+
+ /**
+ * Returns true if the partition schema if the partition schema does not include any hash
+ * components, and the range columns match the table's primary key columns.
+ *
+ * @return whether the partition schema is the default simple range partitioning.
+ */
+ boolean isSimpleRangePartitioning() {
+ return isSimple;
+ }
+
+ public static class RangeSchema {
+ private final List<Integer> columns;
+
+ RangeSchema(List<Integer> columns) {
+ this.columns = columns;
+ }
+
+ public List<Integer> getColumns() {
+ return columns;
+ }
+ }
+
+ public static class HashBucketSchema {
+ private final List<Integer> columnIds;
+ private int numBuckets;
+ private int seed;
+
+ HashBucketSchema(List<Integer> columnIds, int numBuckets, int seed) {
+ this.columnIds = columnIds;
+ this.numBuckets = numBuckets;
+ this.seed = seed;
+ }
+
+ /**
+ * Gets the column IDs of the columns in the hash partition.
+ * @return the column IDs of the columns in the has partition
+ */
+ public List<Integer> getColumnIds() {
+ return columnIds;
+ }
+
+ public int getNumBuckets() {
+ return numBuckets;
+ }
+
+ public int getSeed() {
+ return seed;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java
new file mode 100644
index 0000000..3ca98e2
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import com.stumbleupon.async.Deferred;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * This exception notifies the application to throttle its use of Kudu.
+ * <p>
+ * Since all APIs of {@link AsyncKuduSession} are asynchronous and non-blocking,
+ * it's possible that the application would produce RPCs at a rate higher
+ * than Kudu is able to handle. When this happens, {@link AsyncKuduSession}
+ * will typically do some buffering up to a certain point beyond which RPCs
+ * will fail-fast with this exception, to prevent the application from
+ * running itself out of memory.
+ * <p>
+ * This exception is expected to be handled by having the application
+ * throttle or pause itself for a short period of time before retrying the
+ * RPC that failed with this exception as well as before sending other RPCs.
+ * The reason this exception inherits from {@link NonRecoverableException}
+ * instead of {@link RecoverableException} is that the usual course of action
+ * when handling a {@link RecoverableException} is to retry right away, which
+ * would defeat the whole purpose of this exception. Here, we want the
+ * application to <b>retry after a reasonable delay</b> as well as <b>throttle
+ * the pace of creation of new RPCs</b>. What constitutes a "reasonable
+ * delay" depends on the nature of RPCs and rate at which they're produced.
+ * <p>
+ * One effective strategy to handle this exception is to set a flag to true
+ * when this exception is first emitted that causes the application to pause
+ * or throttle its use of Kudu. Then you can retry the RPC that failed
+ * (which is accessible through {@link #getFailedRpc}) and add a callback to
+ * it in order to unset the flag once the RPC completes successfully.
+ * Note that low-throughput applications will typically rarely (if ever)
+ * hit this exception, so they don't need complex throttling logic.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+public final class PleaseThrottleException extends RecoverableException
+ implements HasFailedRpcException {
+
+ /** The RPC that was failed with this exception. */
+ private final Operation rpc;
+
+ /** A deferred one can wait on before retrying the failed RPC. */
+ private final Deferred deferred;
+
+ /**
+ * Constructor.
+ * @param status status object containing the reason for the exception
+ * @param cause The exception that requires the application to throttle
+ * itself (can be {@code null})
+ * @param rpc The RPC that was made to fail with this exception
+ * @param deferred A deferred one can wait on before retrying the failed RPC
+ */
+ PleaseThrottleException(Status status,
+ KuduException cause,
+ Operation rpc,
+ Deferred deferred) {
+ super(status, cause);
+ this.rpc = rpc;
+ this.deferred = deferred;
+ }
+
+ /**
+ * The RPC that was made to fail with this exception.
+ */
+ public Operation getFailedRpc() {
+ return rpc;
+ }
+
+ /**
+ * Returns a deferred one can wait on before retrying the failed RPC.
+ * @since 1.3
+ */
+ public Deferred getDeferred() {
+ return deferred;
+ }
+
+}
[04/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
new file mode 100644
index 0000000..d732b71
--- /dev/null
+++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/KuduSinkTest.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.kududb.flume.sink;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.flume.Channel;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.Sink;
+import org.apache.flume.Transaction;
+import org.apache.flume.channel.MemoryChannel;
+import org.apache.flume.conf.Configurables;
+import org.apache.flume.event.EventBuilder;
+import org.junit.Assert;
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.client.BaseKuduTest;
+import org.kududb.client.CreateTableOptions;
+import org.kududb.client.KuduTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class KuduSinkTest extends BaseKuduTest {
+ private static final Logger LOG = LoggerFactory.getLogger(KuduSinkTest.class);
+
+ private KuduTable createNewTable(String tableName) throws Exception {
+ LOG.info("Creating new table...");
+
+ ArrayList<ColumnSchema> columns = new ArrayList<>(1);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("payload", Type.BINARY).key(true).build());
+ CreateTableOptions createOptions =
+ new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("payload"))
+ .setNumReplicas(1);
+ KuduTable table = createTable(tableName, new Schema(columns), createOptions);
+
+ LOG.info("Created new table.");
+
+ return table;
+ }
+
+ @Test
+ public void testMandatoryParameters() {
+ LOG.info("Testing mandatory parameters...");
+
+ KuduSink sink = new KuduSink(syncClient);
+
+ HashMap<String, String> parameters = new HashMap<>();
+ Context context = new Context(parameters);
+ try {
+ Configurables.configure(sink, context);
+ Assert.fail("Should have failed due to missing properties");
+ } catch (NullPointerException npe) {
+ //good
+ }
+
+ parameters.put(KuduSinkConfigurationConstants.TABLE_NAME, "tableName");
+ context = new Context(parameters);
+ try {
+ Configurables.configure(sink, context);
+ Assert.fail("Should have failed due to missing properties");
+ } catch (NullPointerException npe) {
+ //good
+ }
+
+ LOG.info("Testing mandatory parameters finished successfully.");
+ }
+
+ @Test(expected = FlumeException.class)
+ public void testMissingTable() throws Exception {
+ LOG.info("Testing missing table...");
+
+ KuduSink sink = createSink("missingTable");
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+
+ LOG.info("Testing missing table finished successfully.");
+ }
+
+ @Test
+ public void testEmptyChannelWithDefaults() throws Exception {
+ testEventsWithDefaults(0);
+ }
+
+ @Test
+ public void testOneEventWithDefaults() throws Exception {
+ testEventsWithDefaults(1);
+ }
+
+ @Test
+ public void testThreeEventsWithDefaults() throws Exception {
+ testEventsWithDefaults(3);
+ }
+
+ @Test
+ public void testDuplicateRowsWithDuplicatesIgnored() throws Exception {
+ doTestDuplicateRows(true);
+ }
+
+ @Test
+ public void testDuplicateRowsWithDuplicatesNotIgnored() throws Exception {
+ doTestDuplicateRows(false);
+ }
+
+ private void doTestDuplicateRows(boolean ignoreDuplicateRows) throws Exception {
+ KuduTable table = createNewTable("testDuplicateRows" + ignoreDuplicateRows);
+ String tableName = table.getName();
+ Context sinkContext = new Context();
+ sinkContext.put(KuduSinkConfigurationConstants.IGNORE_DUPLICATE_ROWS,
+ Boolean.toString(ignoreDuplicateRows));
+ KuduSink sink = createSink(tableName, sinkContext);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+
+ for (int i = 0; i < 2; i++) {
+ Event e = EventBuilder.withBody("key-0", Charsets.UTF_8); // Duplicate keys.
+ channel.put(e);
+ }
+
+ tx.commit();
+ tx.close();
+
+ try {
+ Sink.Status status = sink.process();
+ if (!ignoreDuplicateRows) {
+ fail("Incorrectly ignored duplicate rows!");
+ }
+ assertTrue("incorrect status for empty channel", status == Sink.Status.READY);
+ } catch (EventDeliveryException e) {
+ if (ignoreDuplicateRows) {
+ throw new AssertionError("Failed to ignore duplicate rows!", e);
+ } else {
+ LOG.info("Correctly did not ignore duplicate rows", e);
+ return;
+ }
+ }
+
+ // We only get here if the process() succeeded.
+ try {
+ List<String> rows = scanTableToStrings(table);
+ assertEquals("1 row expected", 1, rows.size());
+ } catch (Exception e) {
+ Throwables.propagate(e);
+ }
+
+ LOG.info("Testing duplicate events finished successfully.");
+ }
+
+ private void testEventsWithDefaults(int eventCount) throws Exception {
+ LOG.info("Testing {} events...", eventCount);
+
+ KuduTable table = createNewTable("test" + eventCount + "events");
+ String tableName = table.getName();
+ KuduSink sink = createSink(tableName);
+
+ Channel channel = new MemoryChannel();
+ Configurables.configure(channel, new Context());
+ sink.setChannel(channel);
+ sink.start();
+
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+
+ for (int i = 0; i < eventCount; i++) {
+ Event e = EventBuilder.withBody(String.format("payload body %s", i).getBytes());
+ channel.put(e);
+ }
+
+ tx.commit();
+ tx.close();
+
+ Sink.Status status = sink.process();
+ if (eventCount == 0) {
+ assertTrue("incorrect status for empty channel", status == Sink.Status.BACKOFF);
+ } else {
+ assertTrue("incorrect status for non-empty channel", status != Sink.Status.BACKOFF);
+ }
+
+ List<String> rows = scanTableToStrings(table);
+ assertEquals(eventCount + " row(s) expected", eventCount, rows.size());
+
+ for (int i = 0; i < eventCount; i++) {
+ assertTrue("incorrect payload", rows.get(i).contains("payload body " + i));
+ }
+
+ LOG.info("Testing {} events finished successfully.", eventCount);
+ }
+
+ private KuduSink createSink(String tableName) {
+ return createSink(tableName, new Context());
+ }
+
+ private KuduSink createSink(String tableName, Context ctx) {
+ LOG.info("Creating Kudu sink for '{}' table...", tableName);
+
+ KuduSink sink = new KuduSink(syncClient);
+ HashMap<String, String> parameters = new HashMap<>();
+ parameters.put(KuduSinkConfigurationConstants.TABLE_NAME, tableName);
+ parameters.put(KuduSinkConfigurationConstants.MASTER_ADDRESSES, getMasterAddresses());
+ Context context = new Context(parameters);
+ context.putAll(ctx.getParameters());
+ Configurables.configure(sink, context);
+
+ LOG.info("Created Kudu sink for '{}' table.", tableName);
+
+ return sink;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/test/java/org/kududb/flume/sink/KuduSinkTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/java/org/kududb/flume/sink/KuduSinkTest.java b/java/kudu-flume-sink/src/test/java/org/kududb/flume/sink/KuduSinkTest.java
deleted file mode 100644
index d732b71..0000000
--- a/java/kudu-flume-sink/src/test/java/org/kududb/flume/sink/KuduSinkTest.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.kududb.flume.sink;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-
-import org.apache.flume.Channel;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Sink;
-import org.apache.flume.Transaction;
-import org.apache.flume.channel.MemoryChannel;
-import org.apache.flume.conf.Configurables;
-import org.apache.flume.event.EventBuilder;
-import org.junit.Assert;
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.client.CreateTableOptions;
-import org.kududb.client.KuduTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class KuduSinkTest extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(KuduSinkTest.class);
-
- private KuduTable createNewTable(String tableName) throws Exception {
- LOG.info("Creating new table...");
-
- ArrayList<ColumnSchema> columns = new ArrayList<>(1);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("payload", Type.BINARY).key(true).build());
- CreateTableOptions createOptions =
- new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("payload"))
- .setNumReplicas(1);
- KuduTable table = createTable(tableName, new Schema(columns), createOptions);
-
- LOG.info("Created new table.");
-
- return table;
- }
-
- @Test
- public void testMandatoryParameters() {
- LOG.info("Testing mandatory parameters...");
-
- KuduSink sink = new KuduSink(syncClient);
-
- HashMap<String, String> parameters = new HashMap<>();
- Context context = new Context(parameters);
- try {
- Configurables.configure(sink, context);
- Assert.fail("Should have failed due to missing properties");
- } catch (NullPointerException npe) {
- //good
- }
-
- parameters.put(KuduSinkConfigurationConstants.TABLE_NAME, "tableName");
- context = new Context(parameters);
- try {
- Configurables.configure(sink, context);
- Assert.fail("Should have failed due to missing properties");
- } catch (NullPointerException npe) {
- //good
- }
-
- LOG.info("Testing mandatory parameters finished successfully.");
- }
-
- @Test(expected = FlumeException.class)
- public void testMissingTable() throws Exception {
- LOG.info("Testing missing table...");
-
- KuduSink sink = createSink("missingTable");
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, new Context());
- sink.setChannel(channel);
- sink.start();
-
- LOG.info("Testing missing table finished successfully.");
- }
-
- @Test
- public void testEmptyChannelWithDefaults() throws Exception {
- testEventsWithDefaults(0);
- }
-
- @Test
- public void testOneEventWithDefaults() throws Exception {
- testEventsWithDefaults(1);
- }
-
- @Test
- public void testThreeEventsWithDefaults() throws Exception {
- testEventsWithDefaults(3);
- }
-
- @Test
- public void testDuplicateRowsWithDuplicatesIgnored() throws Exception {
- doTestDuplicateRows(true);
- }
-
- @Test
- public void testDuplicateRowsWithDuplicatesNotIgnored() throws Exception {
- doTestDuplicateRows(false);
- }
-
- private void doTestDuplicateRows(boolean ignoreDuplicateRows) throws Exception {
- KuduTable table = createNewTable("testDuplicateRows" + ignoreDuplicateRows);
- String tableName = table.getName();
- Context sinkContext = new Context();
- sinkContext.put(KuduSinkConfigurationConstants.IGNORE_DUPLICATE_ROWS,
- Boolean.toString(ignoreDuplicateRows));
- KuduSink sink = createSink(tableName, sinkContext);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, new Context());
- sink.setChannel(channel);
- sink.start();
-
- Transaction tx = channel.getTransaction();
- tx.begin();
-
- for (int i = 0; i < 2; i++) {
- Event e = EventBuilder.withBody("key-0", Charsets.UTF_8); // Duplicate keys.
- channel.put(e);
- }
-
- tx.commit();
- tx.close();
-
- try {
- Sink.Status status = sink.process();
- if (!ignoreDuplicateRows) {
- fail("Incorrectly ignored duplicate rows!");
- }
- assertTrue("incorrect status for empty channel", status == Sink.Status.READY);
- } catch (EventDeliveryException e) {
- if (ignoreDuplicateRows) {
- throw new AssertionError("Failed to ignore duplicate rows!", e);
- } else {
- LOG.info("Correctly did not ignore duplicate rows", e);
- return;
- }
- }
-
- // We only get here if the process() succeeded.
- try {
- List<String> rows = scanTableToStrings(table);
- assertEquals("1 row expected", 1, rows.size());
- } catch (Exception e) {
- Throwables.propagate(e);
- }
-
- LOG.info("Testing duplicate events finished successfully.");
- }
-
- private void testEventsWithDefaults(int eventCount) throws Exception {
- LOG.info("Testing {} events...", eventCount);
-
- KuduTable table = createNewTable("test" + eventCount + "events");
- String tableName = table.getName();
- KuduSink sink = createSink(tableName);
-
- Channel channel = new MemoryChannel();
- Configurables.configure(channel, new Context());
- sink.setChannel(channel);
- sink.start();
-
- Transaction tx = channel.getTransaction();
- tx.begin();
-
- for (int i = 0; i < eventCount; i++) {
- Event e = EventBuilder.withBody(String.format("payload body %s", i).getBytes());
- channel.put(e);
- }
-
- tx.commit();
- tx.close();
-
- Sink.Status status = sink.process();
- if (eventCount == 0) {
- assertTrue("incorrect status for empty channel", status == Sink.Status.BACKOFF);
- } else {
- assertTrue("incorrect status for non-empty channel", status != Sink.Status.BACKOFF);
- }
-
- List<String> rows = scanTableToStrings(table);
- assertEquals(eventCount + " row(s) expected", eventCount, rows.size());
-
- for (int i = 0; i < eventCount; i++) {
- assertTrue("incorrect payload", rows.get(i).contains("payload body " + i));
- }
-
- LOG.info("Testing {} events finished successfully.", eventCount);
- }
-
- private KuduSink createSink(String tableName) {
- return createSink(tableName, new Context());
- }
-
- private KuduSink createSink(String tableName, Context ctx) {
- LOG.info("Creating Kudu sink for '{}' table...", tableName);
-
- KuduSink sink = new KuduSink(syncClient);
- HashMap<String, String> parameters = new HashMap<>();
- parameters.put(KuduSinkConfigurationConstants.TABLE_NAME, tableName);
- parameters.put(KuduSinkConfigurationConstants.MASTER_ADDRESSES, getMasterAddresses());
- Context context = new Context(parameters);
- context.putAll(ctx.getParameters());
- Configurables.configure(sink, context);
-
- LOG.info("Created Kudu sink for '{}' table.", tableName);
-
- return sink;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java
new file mode 100644
index 0000000..05c18f2
--- /dev/null
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/CommandLineParser.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.AsyncKuduClient;
+import org.apache.hadoop.conf.Configuration;
+import org.kududb.client.KuduClient;
+
+/**
+ * Utility class that manages common configurations to all MR jobs. For example,
+ * any job that uses {#KuduTableMapReduceUtil} to setup an input or output format
+ * and that has parsed the command line arguments with
+ * {@link org.apache.hadoop.util.GenericOptionsParser} can simply be passed:
+ * <code>
+ * -Dmaster.address=ADDRESS
+ * </code>
+ * in order to specify where the master is.
+ * Use {@link CommandLineParser#getHelpSnippet()} to provide usage text for the configurations
+ * managed by this class.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class CommandLineParser {
+ private final Configuration conf;
+ public static final String MASTER_ADDRESSES_KEY = "kudu.master.addresses";
+ public static final String MASTER_ADDRESSES_DEFAULT = "127.0.0.1";
+ public static final String OPERATION_TIMEOUT_MS_KEY = "kudu.operation.timeout.ms";
+ public static final long OPERATION_TIMEOUT_MS_DEFAULT =
+ AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
+ public static final String ADMIN_OPERATION_TIMEOUT_MS_KEY = "kudu.admin.operation.timeout.ms";
+ public static final String SOCKET_READ_TIMEOUT_MS_KEY = "kudu.socket.read.timeout.ms";
+ public static final long SOCKET_READ_TIMEOUT_MS_DEFAULT =
+ AsyncKuduClient.DEFAULT_SOCKET_READ_TIMEOUT_MS;
+ public static final String NUM_REPLICAS_KEY = "kudu.num.replicas";
+ public static final int NUM_REPLICAS_DEFAULT = 3;
+
+ /**
+ * Constructor that uses a Configuration that has already been through
+ * {@link org.apache.hadoop.util.GenericOptionsParser}'s command line parsing.
+ * @param conf the configuration from which job configurations will be extracted
+ */
+ public CommandLineParser(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Get the configured master's config.
+ * @return a string that contains the passed config, or the default value
+ */
+ public String getMasterAddresses() {
+ return conf.get(MASTER_ADDRESSES_KEY, MASTER_ADDRESSES_DEFAULT);
+ }
+
+ /**
+ * Get the configured timeout for operations on sessions and scanners.
+ * @return a long that represents the passed timeout, or the default value
+ */
+ public long getOperationTimeoutMs() {
+ return conf.getLong(OPERATION_TIMEOUT_MS_KEY, OPERATION_TIMEOUT_MS_DEFAULT);
+ }
+
+ /**
+ * Get the configured timeout for admin operations.
+ * @return a long that represents the passed timeout, or the default value
+ */
+ public long getAdminOperationTimeoutMs() {
+ return conf.getLong(ADMIN_OPERATION_TIMEOUT_MS_KEY, OPERATION_TIMEOUT_MS_DEFAULT);
+ }
+
+ /**
+ * Get the configured timeout for socket reads.
+ * @return a long that represents the passed timeout, or the default value
+ */
+ public long getSocketReadTimeoutMs() {
+ return conf.getLong(SOCKET_READ_TIMEOUT_MS_KEY, SOCKET_READ_TIMEOUT_MS_DEFAULT);
+ }
+
+ /**
+ * Get the number of replicas to use when configuring a new table.
+ * @return an int that represents the passed number of replicas to use, or the default value.
+ */
+ public int getNumReplicas() {
+ return conf.getInt(NUM_REPLICAS_KEY, NUM_REPLICAS_DEFAULT);
+ }
+
+ /**
+ * Get an async client connected to the configured Master(s).
+ * @return an async kudu client
+ */
+ public AsyncKuduClient getAsyncClient() {
+ return new AsyncKuduClient.AsyncKuduClientBuilder(getMasterAddresses())
+ .defaultOperationTimeoutMs(getOperationTimeoutMs())
+ .defaultAdminOperationTimeoutMs(getAdminOperationTimeoutMs())
+ .defaultSocketReadTimeoutMs(getSocketReadTimeoutMs())
+ .build();
+ }
+
+ /**
+ * Get a client connected to the configured Master(s).
+ * @return a kudu client
+ */
+ public KuduClient getClient() {
+ return new KuduClient.KuduClientBuilder(getMasterAddresses())
+ .defaultOperationTimeoutMs(getOperationTimeoutMs())
+ .defaultAdminOperationTimeoutMs(getAdminOperationTimeoutMs())
+ .defaultSocketReadTimeoutMs(getSocketReadTimeoutMs())
+ .build();
+ }
+
+ /**
+ * This method returns a single multi-line string that contains the help snippet to append to
+ * the tail of a usage() or help() type of method.
+ * @return a string with all the available configurations and their defaults
+ */
+ public static String getHelpSnippet() {
+ return "\nAdditionally, the following options are available:" +
+ " -D" + OPERATION_TIMEOUT_MS_KEY + "=TIME - timeout for read and write " +
+ "operations, defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n"+
+ " -D" + ADMIN_OPERATION_TIMEOUT_MS_KEY + "=TIME - timeout for admin operations " +
+ ", defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n"+
+ " -D" + SOCKET_READ_TIMEOUT_MS_KEY + "=TIME - timeout for socket reads " +
+ ", defaults to " + SOCKET_READ_TIMEOUT_MS_DEFAULT + " \n"+
+ " -D" + MASTER_ADDRESSES_KEY + "=ADDRESSES - addresses to reach the Masters, " +
+ "defaults to " + MASTER_ADDRESSES_DEFAULT + " which is usually wrong.\n" +
+ " -D " + NUM_REPLICAS_KEY + "=NUM - number of replicas to use when configuring a new " +
+ "table, defaults to " + NUM_REPLICAS_DEFAULT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
new file mode 100644
index 0000000..57593db
--- /dev/null
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/JarFinder.java
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import com.google.common.base.Preconditions;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.MessageFormat;
+import java.util.Enumeration;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * Finds the Jar for a class. If the class is in a directory in the
+ * classpath, it creates a Jar on the fly with the contents of the directory
+ * and returns the path to that Jar. If a Jar is created, it is created in
+ * the system temporary directory.
+ *
+ * This file was forked from hbase/branches/master@4ce6f48.
+ */
+public class JarFinder {
+
+ private static void copyToZipStream(File file, ZipEntry entry,
+ ZipOutputStream zos) throws IOException {
+ InputStream is = new FileInputStream(file);
+ try {
+ zos.putNextEntry(entry);
+ byte[] arr = new byte[4096];
+ int read = is.read(arr);
+ while (read > -1) {
+ zos.write(arr, 0, read);
+ read = is.read(arr);
+ }
+ } finally {
+ try {
+ is.close();
+ } finally {
+ zos.closeEntry();
+ }
+ }
+ }
+
+ public static void jarDir(File dir, String relativePath, ZipOutputStream zos)
+ throws IOException {
+ Preconditions.checkNotNull(relativePath, "relativePath");
+ Preconditions.checkNotNull(zos, "zos");
+
+ // by JAR spec, if there is a manifest, it must be the first entry in the
+ // ZIP.
+ File manifestFile = new File(dir, JarFile.MANIFEST_NAME);
+ ZipEntry manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
+ if (!manifestFile.exists()) {
+ zos.putNextEntry(manifestEntry);
+ new Manifest().write(new BufferedOutputStream(zos));
+ zos.closeEntry();
+ } else {
+ copyToZipStream(manifestFile, manifestEntry, zos);
+ }
+ zos.closeEntry();
+ zipDir(dir, relativePath, zos, true);
+ zos.close();
+ }
+
+ private static void zipDir(File dir, String relativePath, ZipOutputStream zos,
+ boolean start) throws IOException {
+ String[] dirList = dir.list();
+ for (String aDirList : dirList) {
+ File f = new File(dir, aDirList);
+ if (!f.isHidden()) {
+ if (f.isDirectory()) {
+ if (!start) {
+ ZipEntry dirEntry = new ZipEntry(relativePath + f.getName() + "/");
+ zos.putNextEntry(dirEntry);
+ zos.closeEntry();
+ }
+ String filePath = f.getPath();
+ File file = new File(filePath);
+ zipDir(file, relativePath + f.getName() + "/", zos, false);
+ }
+ else {
+ String path = relativePath + f.getName();
+ if (!path.equals(JarFile.MANIFEST_NAME)) {
+ ZipEntry anEntry = new ZipEntry(path);
+ copyToZipStream(f, anEntry, zos);
+ }
+ }
+ }
+ }
+ }
+
+ private static void createJar(File dir, File jarFile) throws IOException {
+ Preconditions.checkNotNull(dir, "dir");
+ Preconditions.checkNotNull(jarFile, "jarFile");
+ File jarDir = jarFile.getParentFile();
+ if (!jarDir.exists()) {
+ if (!jarDir.mkdirs()) {
+ throw new IOException(MessageFormat.format("could not create dir [{0}]",
+ jarDir));
+ }
+ }
+ JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile));
+ jarDir(dir, "", zos);
+ }
+
+ /**
+ * Returns the full path to the Jar containing the class. It always returns a
+ * JAR.
+ *
+ * @param klass class.
+ *
+ * @return path to the Jar containing the class.
+ */
+ public static String getJar(Class klass) {
+ Preconditions.checkNotNull(klass, "klass");
+ ClassLoader loader = klass.getClassLoader();
+ if (loader != null) {
+ String class_file = klass.getName().replaceAll("\\.", "/") + ".class";
+ try {
+ for (Enumeration itr = loader.getResources(class_file);
+ itr.hasMoreElements(); ) {
+ URL url = (URL) itr.nextElement();
+ String path = url.getPath();
+ if (path.startsWith("file:")) {
+ path = path.substring("file:".length());
+ }
+ path = URLDecoder.decode(path, "UTF-8");
+ if ("jar".equals(url.getProtocol())) {
+ path = URLDecoder.decode(path, "UTF-8");
+ return path.replaceAll("!.*$", "");
+ }
+ else if ("file".equals(url.getProtocol())) {
+ String klassName = klass.getName();
+ klassName = klassName.replace(".", "/") + ".class";
+ path = path.substring(0, path.length() - klassName.length());
+ File baseDir = new File(path);
+ File testDir = new File(System.getProperty("test.build.dir", "target/test-dir"));
+ testDir = testDir.getAbsoluteFile();
+ if (!testDir.exists()) {
+ testDir.mkdirs();
+ }
+ File tempJar = File.createTempFile("hadoop-", "", testDir);
+ tempJar = new File(tempJar.getAbsolutePath() + ".jar");
+ tempJar.deleteOnExit();
+ createJar(baseDir, tempJar);
+ return tempJar.getAbsolutePath();
+ }
+ }
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
new file mode 100644
index 0000000..25235cb
--- /dev/null
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableInputFormat.java
@@ -0,0 +1,444 @@
+/**
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.mapreduce;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import com.google.common.primitives.UnsignedBytes;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.naming.NamingException;
+
+import org.apache.commons.net.util.Base64;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.net.DNS;
+import org.kududb.Common;
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.AsyncKuduClient;
+import org.kududb.client.Bytes;
+import org.kududb.client.KuduClient;
+import org.kududb.client.KuduPredicate;
+import org.kududb.client.KuduScanner;
+import org.kududb.client.KuduTable;
+import org.kududb.client.LocatedTablet;
+import org.kududb.client.RowResult;
+import org.kududb.client.RowResultIterator;
+import org.kududb.client.KuduScanToken;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * This input format generates one split per tablet and the only location for each split is that
+ * tablet's leader.
+ * </p>
+ *
+ * <p>
+ * Hadoop doesn't have the concept of "closing" the input format so in order to release the
+ * resources we assume that once either {@link #getSplits(org.apache.hadoop.mapreduce.JobContext)}
+ * or {@link KuduTableInputFormat.TableRecordReader#close()} have been called that
+ * the object won't be used again and the AsyncKuduClient is shut down.
+ * </p>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult>
+ implements Configurable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KuduTableInputFormat.class);
+
+ /** Job parameter that specifies the input table. */
+ static final String INPUT_TABLE_KEY = "kudu.mapreduce.input.table";
+
+ /** Job parameter that specifies if the scanner should cache blocks or not (default: false). */
+ static final String SCAN_CACHE_BLOCKS = "kudu.mapreduce.input.scan.cache.blocks";
+
+ /** Job parameter that specifies where the masters are. */
+ static final String MASTER_ADDRESSES_KEY = "kudu.mapreduce.master.address";
+
+ /** Job parameter that specifies how long we wait for operations to complete (default: 10s). */
+ static final String OPERATION_TIMEOUT_MS_KEY = "kudu.mapreduce.operation.timeout.ms";
+
+ /** Job parameter that specifies the address for the name server. */
+ static final String NAME_SERVER_KEY = "kudu.mapreduce.name.server";
+
+ /** Job parameter that specifies the encoded column predicates (may be empty). */
+ static final String ENCODED_PREDICATES_KEY =
+ "kudu.mapreduce.encoded.predicates";
+
+ /**
+ * Job parameter that specifies the column projection as a comma-separated list of column names.
+ *
+ * Not specifying this at all (i.e. setting to null) or setting to the special string
+ * '*' means to project all columns.
+ *
+ * Specifying the empty string means to project no columns (i.e just count the rows).
+ */
+ static final String COLUMN_PROJECTION_KEY = "kudu.mapreduce.column.projection";
+
+ /**
+ * The reverse DNS lookup cache mapping: address from Kudu => hostname for Hadoop. This cache is
+ * used in order to not do DNS lookups multiple times for each tablet server.
+ */
+ private final Map<String, String> reverseDNSCacheMap = new HashMap<>();
+
+ private Configuration conf;
+ private KuduClient client;
+ private KuduTable table;
+ private long operationTimeoutMs;
+ private String nameServer;
+ private boolean cacheBlocks;
+ private List<String> projectedCols;
+ private List<KuduPredicate> predicates;
+
+ @Override
+ public List<InputSplit> getSplits(JobContext jobContext)
+ throws IOException, InterruptedException {
+ try {
+ if (table == null) {
+ throw new IOException("No table was provided");
+ }
+
+ KuduScanToken.KuduScanTokenBuilder tokenBuilder = client.newScanTokenBuilder(table)
+ .setProjectedColumnNames(projectedCols)
+ .cacheBlocks(cacheBlocks)
+ .setTimeout(operationTimeoutMs);
+ for (KuduPredicate predicate : predicates) {
+ tokenBuilder.addPredicate(predicate);
+ }
+ List<KuduScanToken> tokens = tokenBuilder.build();
+
+ List<InputSplit> splits = new ArrayList<>(tokens.size());
+ for (KuduScanToken token : tokens) {
+ List<String> locations = new ArrayList<>(token.getTablet().getReplicas().size());
+ for (LocatedTablet.Replica replica : token.getTablet().getReplicas()) {
+ locations.add(reverseDNS(replica.getRpcHost(), replica.getRpcPort()));
+ }
+ splits.add(new TableSplit(token, locations.toArray(new String[locations.size()])));
+ }
+ return splits;
+ } finally {
+ shutdownClient();
+ }
+ }
+
+ private void shutdownClient() throws IOException {
+ try {
+ client.shutdown();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * This method might seem alien, but we do this in order to resolve the hostnames the same way
+ * Hadoop does. This ensures we get locality if Kudu is running along MR/YARN.
+ * @param host hostname we got from the master
+ * @param port port we got from the master
+ * @return reverse DNS'd address
+ */
+ private String reverseDNS(String host, Integer port) {
+ String location = this.reverseDNSCacheMap.get(host);
+ if (location != null) {
+ return location;
+ }
+ // The below InetSocketAddress creation does a name resolution.
+ InetSocketAddress isa = new InetSocketAddress(host, port);
+ if (isa.isUnresolved()) {
+ LOG.warn("Failed address resolve for: " + isa);
+ }
+ InetAddress tabletInetAddress = isa.getAddress();
+ try {
+ location = domainNamePointerToHostName(
+ DNS.reverseDns(tabletInetAddress, this.nameServer));
+ this.reverseDNSCacheMap.put(host, location);
+ } catch (NamingException e) {
+ LOG.warn("Cannot resolve the host name for " + tabletInetAddress + " because of " + e);
+ location = host;
+ }
+ return location;
+ }
+
+ @Override
+ public RecordReader<NullWritable, RowResult> createRecordReader(InputSplit inputSplit,
+ TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+ return new TableRecordReader();
+ }
+
+ @Override
+ public void setConf(Configuration entries) {
+ this.conf = new Configuration(entries);
+
+ String tableName = conf.get(INPUT_TABLE_KEY);
+ String masterAddresses = conf.get(MASTER_ADDRESSES_KEY);
+ this.operationTimeoutMs = conf.getLong(OPERATION_TIMEOUT_MS_KEY,
+ AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS);
+ this.nameServer = conf.get(NAME_SERVER_KEY);
+ this.cacheBlocks = conf.getBoolean(SCAN_CACHE_BLOCKS, false);
+
+ this.client = new KuduClient.KuduClientBuilder(masterAddresses)
+ .defaultOperationTimeoutMs(operationTimeoutMs)
+ .build();
+ try {
+ this.table = client.openTable(tableName);
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not obtain the table from the master, " +
+ "is the master running and is this table created? tablename=" + tableName + " and " +
+ "master address= " + masterAddresses, ex);
+ }
+
+ String projectionConfig = conf.get(COLUMN_PROJECTION_KEY);
+ if (projectionConfig == null || projectionConfig.equals("*")) {
+ this.projectedCols = null; // project the whole table
+ } else if ("".equals(projectionConfig)) {
+ this.projectedCols = new ArrayList<>();
+ } else {
+ this.projectedCols = Lists.newArrayList(Splitter.on(',').split(projectionConfig));
+
+ // Verify that the column names are valid -- better to fail with an exception
+ // before we submit the job.
+ Schema tableSchema = table.getSchema();
+ for (String columnName : projectedCols) {
+ if (tableSchema.getColumn(columnName) == null) {
+ throw new IllegalArgumentException("Unknown column " + columnName);
+ }
+ }
+ }
+
+ this.predicates = new ArrayList<>();
+ try {
+ InputStream is =
+ new ByteArrayInputStream(Base64.decodeBase64(conf.get(ENCODED_PREDICATES_KEY, "")));
+ while (is.available() > 0) {
+ this.predicates.add(KuduPredicate.fromPB(table.getSchema(),
+ Common.ColumnPredicatePB.parseDelimitedFrom(is)));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("unable to deserialize predicates from the configuration", e);
+ }
+ }
+
+ /**
+ * Given a PTR string generated via reverse DNS lookup, return everything
+ * except the trailing period. Example for host.example.com., return
+ * host.example.com
+ * @param dnPtr a domain name pointer (PTR) string.
+ * @return Sanitized hostname with last period stripped off.
+ *
+ */
+ private static String domainNamePointerToHostName(String dnPtr) {
+ if (dnPtr == null)
+ return null;
+ return dnPtr.endsWith(".") ? dnPtr.substring(0, dnPtr.length() - 1) : dnPtr;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ static class TableSplit extends InputSplit implements Writable, Comparable<TableSplit> {
+
+ /** The scan token that the split will use to scan the Kudu table. */
+ private byte[] scanToken;
+
+ /** The start partition key of the scan. Used for sorting splits. */
+ private byte[] partitionKey;
+
+ /** Tablet server locations which host the tablet to be scanned. */
+ private String[] locations;
+
+ public TableSplit() { } // Writable
+
+ public TableSplit(KuduScanToken token, String[] locations) throws IOException {
+ this.scanToken = token.serialize();
+ this.partitionKey = token.getTablet().getPartition().getPartitionKeyStart();
+ this.locations = locations;
+ }
+
+ public byte[] getScanToken() {
+ return scanToken;
+ }
+
+ public byte[] getPartitionKey() {
+ return partitionKey;
+ }
+
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ // TODO Guesstimate a size
+ return 0;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return locations;
+ }
+
+ @Override
+ public int compareTo(TableSplit other) {
+ return UnsignedBytes.lexicographicalComparator().compare(partitionKey, other.partitionKey);
+ }
+
+ @Override
+ public void write(DataOutput dataOutput) throws IOException {
+ Bytes.writeByteArray(dataOutput, scanToken);
+ Bytes.writeByteArray(dataOutput, partitionKey);
+ dataOutput.writeInt(locations.length);
+ for (String location : locations) {
+ byte[] str = Bytes.fromString(location);
+ Bytes.writeByteArray(dataOutput, str);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput dataInput) throws IOException {
+ scanToken = Bytes.readByteArray(dataInput);
+ partitionKey = Bytes.readByteArray(dataInput);
+ locations = new String[dataInput.readInt()];
+ for (int i = 0; i < locations.length; i++) {
+ byte[] str = Bytes.readByteArray(dataInput);
+ locations[i] = Bytes.getString(str);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ // We currently just care about the partition key since we're within the same table.
+ return Arrays.hashCode(partitionKey);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TableSplit that = (TableSplit) o;
+
+ return this.compareTo(that) == 0;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("partitionKey", Bytes.pretty(partitionKey))
+ .add("locations", Arrays.toString(locations))
+ .toString();
+ }
+ }
+
+ class TableRecordReader extends RecordReader<NullWritable, RowResult> {
+
+ private final NullWritable currentKey = NullWritable.get();
+ private RowResult currentValue;
+ private RowResultIterator iterator;
+ private KuduScanner scanner;
+ private TableSplit split;
+
+ @Override
+ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+ if (!(inputSplit instanceof TableSplit)) {
+ throw new IllegalArgumentException("TableSplit is the only accepted input split");
+ }
+
+ split = (TableSplit) inputSplit;
+
+ try {
+ scanner = KuduScanToken.deserializeIntoScanner(split.getScanToken(), client);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ // Calling this now to set iterator.
+ tryRefreshIterator();
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ if (!iterator.hasNext()) {
+ tryRefreshIterator();
+ if (!iterator.hasNext()) {
+ // Means we still have the same iterator, we're done
+ return false;
+ }
+ }
+ currentValue = iterator.next();
+ return true;
+ }
+
+ /**
+ * If the scanner has more rows, get a new iterator else don't do anything.
+ * @throws IOException
+ */
+ private void tryRefreshIterator() throws IOException {
+ if (!scanner.hasMoreRows()) {
+ return;
+ }
+ try {
+ iterator = scanner.nextRows();
+ } catch (Exception e) {
+ throw new IOException("Couldn't get scan data", e);
+ }
+ }
+
+ @Override
+ public NullWritable getCurrentKey() throws IOException, InterruptedException {
+ return currentKey;
+ }
+
+ @Override
+ public RowResult getCurrentValue() throws IOException, InterruptedException {
+ return currentValue;
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ // TODO Guesstimate progress
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ scanner.close();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ shutdownClient();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
new file mode 100644
index 0000000..0b919d9
--- /dev/null
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableMapReduceUtil.java
@@ -0,0 +1,541 @@
+/**
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.mapreduce;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.net.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.TaskInputOutputContext;
+import org.apache.hadoop.util.StringUtils;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.AsyncKuduClient;
+import org.kududb.client.ColumnRangePredicate;
+import org.kududb.client.KuduPredicate;
+import org.kududb.client.KuduTable;
+import org.kududb.client.Operation;
+
+/**
+ * Utility class to setup MR jobs that use Kudu as an input and/or output.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduTableMapReduceUtil {
+ // Mostly lifted from HBase's TableMapReduceUtil
+
+ private static final Log LOG = LogFactory.getLog(KuduTableMapReduceUtil.class);
+
+ /**
+ * Doesn't need instantiation
+ */
+ private KuduTableMapReduceUtil() { }
+
+
+ /**
+ * Base class for MR I/O formats, contains the common configurations.
+ */
+ private static abstract class AbstractMapReduceConfigurator<S> {
+ protected final Job job;
+ protected final String table;
+
+ protected boolean addDependencies = true;
+
+ /**
+ * Constructor for the required fields to configure.
+ * @param job a job to configure
+ * @param table a string that contains the name of the table to read from
+ */
+ private AbstractMapReduceConfigurator(Job job, String table) {
+ this.job = job;
+ this.table = table;
+ }
+
+ /**
+ * Sets whether this job should add Kudu's dependencies to the distributed cache. Turned on
+ * by default.
+ * @param addDependencies a boolean that says if we should add the dependencies
+ * @return this instance
+ */
+ @SuppressWarnings("unchecked")
+ public S addDependencies(boolean addDependencies) {
+ this.addDependencies = addDependencies;
+ return (S) this;
+ }
+
+ /**
+ * Configures the job using the passed parameters.
+ * @throws IOException If addDependencies is enabled and a problem is encountered reading
+ * files on the filesystem
+ */
+ public abstract void configure() throws IOException;
+ }
+
+ /**
+ * Builder-like class that sets up the required configurations and classes to write to Kudu.
+ * <p>
+ * Use either child classes when configuring the table output format.
+ */
+ private static abstract class AbstractTableOutputFormatConfigurator
+ <S extends AbstractTableOutputFormatConfigurator<? super S>>
+ extends AbstractMapReduceConfigurator<S> {
+
+ protected String masterAddresses;
+ protected long operationTimeoutMs = AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
+
+ /**
+ * {@inheritDoc}
+ */
+ private AbstractTableOutputFormatConfigurator(Job job, String table) {
+ super(job, table);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public void configure() throws IOException {
+ job.setOutputFormatClass(KuduTableOutputFormat.class);
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(Operation.class);
+
+ Configuration conf = job.getConfiguration();
+ conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, masterAddresses);
+ conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, table);
+ conf.setLong(KuduTableOutputFormat.OPERATION_TIMEOUT_MS_KEY, operationTimeoutMs);
+ if (addDependencies) {
+ addDependencyJars(job);
+ }
+ }
+ }
+
+ /**
+ * Builder-like class that sets up the required configurations and classes to read from Kudu.
+ * By default, block caching is disabled.
+ * <p>
+ * Use either child classes when configuring the table input format.
+ */
+ private static abstract class AbstractTableInputFormatConfigurator
+ <S extends AbstractTableInputFormatConfigurator<? super S>>
+ extends AbstractMapReduceConfigurator<S> {
+
+ protected String masterAddresses;
+ protected long operationTimeoutMs = AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
+ protected final String columnProjection;
+ protected boolean cacheBlocks;
+ protected List<KuduPredicate> predicates = new ArrayList<>();
+
+ /**
+ * Constructor for the required fields to configure.
+ * @param job a job to configure
+ * @param table a string that contains the name of the table to read from
+ * @param columnProjection a string containing a comma-separated list of columns to read.
+ * It can be null in which case we read empty rows
+ */
+ private AbstractTableInputFormatConfigurator(Job job, String table, String columnProjection) {
+ super(job, table);
+ this.columnProjection = columnProjection;
+ }
+
+ /**
+ * Sets the block caching configuration for the scanners. Turned off by default.
+ * @param cacheBlocks whether the job should use scanners that cache blocks.
+ * @return this instance
+ */
+ public S cacheBlocks(boolean cacheBlocks) {
+ this.cacheBlocks = cacheBlocks;
+ return (S) this;
+ }
+
+ /**
+ * Configures the job with all the passed parameters.
+ * @throws IOException If addDependencies is enabled and a problem is encountered reading
+ * files on the filesystem
+ */
+ public void configure() throws IOException {
+ job.setInputFormatClass(KuduTableInputFormat.class);
+
+ Configuration conf = job.getConfiguration();
+
+ conf.set(KuduTableInputFormat.MASTER_ADDRESSES_KEY, masterAddresses);
+ conf.set(KuduTableInputFormat.INPUT_TABLE_KEY, table);
+ conf.setLong(KuduTableInputFormat.OPERATION_TIMEOUT_MS_KEY, operationTimeoutMs);
+ conf.setBoolean(KuduTableInputFormat.SCAN_CACHE_BLOCKS, cacheBlocks);
+
+ if (columnProjection != null) {
+ conf.set(KuduTableInputFormat.COLUMN_PROJECTION_KEY, columnProjection);
+ }
+
+ conf.set(KuduTableInputFormat.ENCODED_PREDICATES_KEY, base64EncodePredicates(predicates));
+
+ if (addDependencies) {
+ addDependencyJars(job);
+ }
+ }
+ }
+
+ /**
+ * Returns the provided predicates as a Base64 encoded string.
+ * @param predicates the predicates to encode
+ * @return the encoded predicates
+ */
+ static String base64EncodePredicates(List<KuduPredicate> predicates) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (KuduPredicate predicate : predicates) {
+ predicate.toPB().writeDelimitedTo(baos);
+ }
+ return Base64.encodeBase64String(baos.toByteArray());
+ }
+
+
+ /**
+ * Table output format configurator to use to specify the parameters directly.
+ */
+ public static class TableOutputFormatConfigurator
+ extends AbstractTableOutputFormatConfigurator<TableOutputFormatConfigurator> {
+
+ /**
+ * Constructor for the required fields to configure.
+ * @param job a job to configure
+ * @param table a string that contains the name of the table to read from
+ * @param masterAddresses a comma-separated list of masters' hosts and ports
+ */
+ public TableOutputFormatConfigurator(Job job, String table, String masterAddresses) {
+ super(job, table);
+ this.masterAddresses = masterAddresses;
+ }
+
+ /**
+ * Sets the timeout for all the operations. The default is 10 seconds.
+ * @param operationTimeoutMs a long that represents the timeout for operations to complete,
+ * must be a positive value or 0
+ * @return this instance
+ * @throws IllegalArgumentException if the operation timeout is lower than 0
+ */
+ public TableOutputFormatConfigurator operationTimeoutMs(long operationTimeoutMs) {
+ if (operationTimeoutMs < 0) {
+ throw new IllegalArgumentException("The operation timeout must be => 0, " +
+ "passed value is: " + operationTimeoutMs);
+ }
+ this.operationTimeoutMs = operationTimeoutMs;
+ return this;
+ }
+ }
+
+ /**
+ * Table output format that uses a {@link CommandLineParser} in order to set the
+ * master config and the operation timeout.
+ */
+ public static class TableOutputFormatConfiguratorWithCommandLineParser extends
+ AbstractTableOutputFormatConfigurator<TableOutputFormatConfiguratorWithCommandLineParser> {
+
+ /**
+ * {@inheritDoc}
+ */
+ public TableOutputFormatConfiguratorWithCommandLineParser(Job job, String table) {
+ super(job, table);
+ CommandLineParser parser = new CommandLineParser(job.getConfiguration());
+ this.masterAddresses = parser.getMasterAddresses();
+ this.operationTimeoutMs = parser.getOperationTimeoutMs();
+ }
+ }
+
+ /**
+ * Table input format configurator to use to specify the parameters directly.
+ */
+ public static class TableInputFormatConfigurator
+ extends AbstractTableInputFormatConfigurator<TableInputFormatConfigurator> {
+
+ /**
+ * Constructor for the required fields to configure.
+ * @param job a job to configure
+ * @param table a string that contains the name of the table to read from
+ * @param columnProjection a string containing a comma-separated list of columns to read.
+ * It can be null in which case we read empty rows
+ * @param masterAddresses a comma-separated list of masters' hosts and ports
+ */
+ public TableInputFormatConfigurator(Job job, String table, String columnProjection,
+ String masterAddresses) {
+ super(job, table, columnProjection);
+ this.masterAddresses = masterAddresses;
+ }
+
+ /**
+ * Sets the timeout for all the operations. The default is 10 seconds.
+ * @param operationTimeoutMs a long that represents the timeout for operations to complete,
+ * must be a positive value or 0
+ * @return this instance
+ * @throws IllegalArgumentException if the operation timeout is lower than 0
+ */
+ public TableInputFormatConfigurator operationTimeoutMs(long operationTimeoutMs) {
+ if (operationTimeoutMs < 0) {
+ throw new IllegalArgumentException("The operation timeout must be => 0, " +
+ "passed value is: " + operationTimeoutMs);
+ }
+ this.operationTimeoutMs = operationTimeoutMs;
+ return this;
+ }
+
+ /**
+ * Adds a new predicate that will be pushed down to all the tablets.
+ * @param predicate a predicate to add
+ * @return this instance
+ * @deprecated use {@link #addPredicate}
+ */
+ @Deprecated
+ public TableInputFormatConfigurator addColumnRangePredicate(ColumnRangePredicate predicate) {
+ return addPredicate(predicate.toKuduPredicate());
+ }
+
+ /**
+ * Adds a new predicate that will be pushed down to all the tablets.
+ * @param predicate a predicate to add
+ * @return this instance
+ */
+ public TableInputFormatConfigurator addPredicate(KuduPredicate predicate) {
+ this.predicates.add(predicate);
+ return this;
+ }
+ }
+
+ /**
+ * Table input format that uses a {@link CommandLineParser} in order to set the
+ * master config and the operation timeout.
+ * This version cannot set column range predicates.
+ */
+ public static class TableInputFormatConfiguratorWithCommandLineParser extends
+ AbstractTableInputFormatConfigurator<TableInputFormatConfiguratorWithCommandLineParser> {
+
+ /**
+ * {@inheritDoc}
+ */
+ public TableInputFormatConfiguratorWithCommandLineParser(Job job,
+ String table,
+ String columnProjection) {
+ super(job, table, columnProjection);
+ CommandLineParser parser = new CommandLineParser(job.getConfiguration());
+ this.masterAddresses = parser.getMasterAddresses();
+ this.operationTimeoutMs = parser.getOperationTimeoutMs();
+ }
+ }
+
+ /**
+ * Use this method when setting up a task to get access to the KuduTable in order to create
+ * Inserts, Updates, and Deletes.
+ * @param context Map context
+ * @return The kudu table object as setup by the output format
+ */
+ @SuppressWarnings("rawtypes")
+ public static KuduTable getTableFromContext(TaskInputOutputContext context) {
+ String multitonKey = context.getConfiguration().get(KuduTableOutputFormat.MULTITON_KEY);
+ return KuduTableOutputFormat.getKuduTable(multitonKey);
+ }
+
+ /**
+ * Add the Kudu dependency jars as well as jars for any of the configured
+ * job classes to the job configuration, so that JobClient will ship them
+ * to the cluster and add them to the DistributedCache.
+ */
+ public static void addDependencyJars(Job job) throws IOException {
+ addKuduDependencyJars(job.getConfiguration());
+ try {
+ addDependencyJars(job.getConfiguration(),
+ // when making changes here, consider also mapred.TableMapReduceUtil
+ // pull job classes
+ job.getMapOutputKeyClass(),
+ job.getMapOutputValueClass(),
+ job.getInputFormatClass(),
+ job.getOutputKeyClass(),
+ job.getOutputValueClass(),
+ job.getOutputFormatClass(),
+ job.getPartitionerClass(),
+ job.getCombinerClass());
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Add the jars containing the given classes to the job's configuration
+ * such that JobClient will ship them to the cluster and add them to
+ * the DistributedCache.
+ */
+ public static void addDependencyJars(Configuration conf,
+ Class<?>... classes) throws IOException {
+
+ FileSystem localFs = FileSystem.getLocal(conf);
+ Set<String> jars = new HashSet<String>();
+ // Add jars that are already in the tmpjars variable
+ jars.addAll(conf.getStringCollection("tmpjars"));
+
+ // add jars as we find them to a map of contents jar name so that we can avoid
+ // creating new jars for classes that have already been packaged.
+ Map<String, String> packagedClasses = new HashMap<String, String>();
+
+ // Add jars containing the specified classes
+ for (Class<?> clazz : classes) {
+ if (clazz == null) continue;
+
+ Path path = findOrCreateJar(clazz, localFs, packagedClasses);
+ if (path == null) {
+ LOG.warn("Could not find jar for class " + clazz +
+ " in order to ship it to the cluster.");
+ continue;
+ }
+ if (!localFs.exists(path)) {
+ LOG.warn("Could not validate jar file " + path + " for class "
+ + clazz);
+ continue;
+ }
+ jars.add(path.toString());
+ }
+ if (jars.isEmpty()) return;
+
+ conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
+ }
+
+ /**
+ * Add Kudu and its dependencies (only) to the job configuration.
+ * <p>
+ * This is intended as a low-level API, facilitating code reuse between this
+ * class and its mapred counterpart. It also of use to external tools that
+ * need to build a MapReduce job that interacts with Kudu but want
+ * fine-grained control over the jars shipped to the cluster.
+ * </p>
+ * @param conf The Configuration object to extend with dependencies.
+ * @see KuduTableMapReduceUtil
+ * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
+ */
+ public static void addKuduDependencyJars(Configuration conf) throws IOException {
+ addDependencyJars(conf,
+ // explicitly pull a class from each module
+ Operation.class, // kudu-client
+ KuduTableMapReduceUtil.class, // kudu-mapreduce
+ // pull necessary dependencies
+ com.stumbleupon.async.Deferred.class);
+ }
+
+ /**
+ * Finds the Jar for a class or creates it if it doesn't exist. If the class
+ * is in a directory in the classpath, it creates a Jar on the fly with the
+ * contents of the directory and returns the path to that Jar. If a Jar is
+ * created, it is created in the system temporary directory. Otherwise,
+ * returns an existing jar that contains a class of the same name. Maintains
+ * a mapping from jar contents to the tmp jar created.
+ * @param my_class the class to find.
+ * @param fs the FileSystem with which to qualify the returned path.
+ * @param packagedClasses a map of class name to path.
+ * @return a jar file that contains the class.
+ * @throws IOException
+ */
+ @SuppressWarnings("deprecation")
+ private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
+ Map<String, String> packagedClasses)
+ throws IOException {
+ // attempt to locate an existing jar for the class.
+ String jar = findContainingJar(my_class, packagedClasses);
+ if (null == jar || jar.isEmpty()) {
+ jar = JarFinder.getJar(my_class);
+ updateMap(jar, packagedClasses);
+ }
+
+ if (null == jar || jar.isEmpty()) {
+ return null;
+ }
+
+ LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
+ return new Path(jar).makeQualified(fs);
+ }
+
+ /**
+ * Find a jar that contains a class of the same name, if any. It will return
+ * a jar file, even if that is not the first thing on the class path that
+ * has a class with the same name. Looks first on the classpath and then in
+ * the <code>packagedClasses</code> map.
+ * @param my_class the class to find.
+ * @return a jar file that contains the class, or null.
+ * @throws IOException
+ */
+ private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
+ throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+
+ // first search the classpath
+ for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
+ URL url = itr.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ return toReturn.replaceAll("!.*$", "");
+ }
+ }
+
+ // now look in any jars we've packaged using JarFinder. Returns null when
+ // no jar is found.
+ return packagedClasses.get(class_file);
+ }
+
+ /**
+ * Add entries to <code>packagedClasses</code> corresponding to class files
+ * contained in <code>jar</code>.
+ * @param jar The jar who's content to list.
+ * @param packagedClasses map[class -> jar]
+ */
+ private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
+ if (null == jar || jar.isEmpty()) {
+ return;
+ }
+ ZipFile zip = null;
+ try {
+ zip = new ZipFile(jar);
+ for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
+ ZipEntry entry = iter.nextElement();
+ if (entry.getName().endsWith("class")) {
+ packagedClasses.put(entry.getName(), jar);
+ }
+ }
+ } finally {
+ if (null != zip) zip.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java
new file mode 100644
index 0000000..8af750b
--- /dev/null
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputCommitter.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.io.IOException;
+
+/**
+ * Small committer class that does not do anything.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduTableOutputCommitter extends OutputCommitter {
+ @Override
+ public void setupJob(JobContext jobContext) throws IOException {
+
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException {
+ return false;
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java
new file mode 100644
index 0000000..e80b73f
--- /dev/null
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/KuduTableOutputFormat.java
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.*;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * <p>
+ * Use {@link
+ * KuduTableMapReduceUtil.TableOutputFormatConfigurator}
+ * to correctly setup this output format, then {@link
+ * KuduTableMapReduceUtil#getTableFromContext(org.apache.hadoop.mapreduce.TaskInputOutputContext)}
+ * to get a KuduTable.
+ * </p>
+ *
+ * <p>
+ * Hadoop doesn't have the concept of "closing" the output format so in order to release the
+ * resources we assume that once either
+ * {@link #checkOutputSpecs(org.apache.hadoop.mapreduce.JobContext)}
+ * or {@link TableRecordWriter#close(org.apache.hadoop.mapreduce.TaskAttemptContext)}
+ * have been called that the object won't be used again and the KuduClient is shut down.
+ * </p>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduTableOutputFormat extends OutputFormat<NullWritable,Operation>
+ implements Configurable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KuduTableOutputFormat.class);
+
+ /** Job parameter that specifies the output table. */
+ static final String OUTPUT_TABLE_KEY = "kudu.mapreduce.output.table";
+
+ /** Job parameter that specifies where the masters are */
+ static final String MASTER_ADDRESSES_KEY = "kudu.mapreduce.master.addresses";
+
+ /** Job parameter that specifies how long we wait for operations to complete */
+ static final String OPERATION_TIMEOUT_MS_KEY = "kudu.mapreduce.operation.timeout.ms";
+
+ /** Number of rows that are buffered before flushing to the tablet server */
+ static final String BUFFER_ROW_COUNT_KEY = "kudu.mapreduce.buffer.row.count";
+
+ /**
+ * Job parameter that specifies which key is to be used to reach the KuduTableOutputFormat
+ * belonging to the caller
+ */
+ static final String MULTITON_KEY = "kudu.mapreduce.multitonkey";
+
+ /**
+ * This multiton is used so that the tasks using this output format/record writer can find
+ * their KuduTable without having a direct dependency on this class,
+ * with the additional complexity that the output format cannot be shared between threads.
+ */
+ private static final ConcurrentHashMap<String, KuduTableOutputFormat> MULTITON = new
+ ConcurrentHashMap<String, KuduTableOutputFormat>();
+
+ /**
+ * This counter helps indicate which task log to look at since rows that weren't applied will
+ * increment this counter.
+ */
+ public enum Counters { ROWS_WITH_ERRORS }
+
+ private Configuration conf = null;
+
+ private KuduClient client;
+ private KuduTable table;
+ private KuduSession session;
+ private long operationTimeoutMs;
+
+ @Override
+ public void setConf(Configuration entries) {
+ this.conf = new Configuration(entries);
+
+ String masterAddress = this.conf.get(MASTER_ADDRESSES_KEY);
+ String tableName = this.conf.get(OUTPUT_TABLE_KEY);
+ this.operationTimeoutMs = this.conf.getLong(OPERATION_TIMEOUT_MS_KEY,
+ AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS);
+ int bufferSpace = this.conf.getInt(BUFFER_ROW_COUNT_KEY, 1000);
+
+ this.client = new KuduClient.KuduClientBuilder(masterAddress)
+ .defaultOperationTimeoutMs(operationTimeoutMs)
+ .build();
+ try {
+ this.table = client.openTable(tableName);
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not obtain the table from the master, " +
+ "is the master running and is this table created? tablename=" + tableName + " and " +
+ "master address= " + masterAddress, ex);
+ }
+ this.session = client.newSession();
+ this.session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
+ this.session.setMutationBufferSpace(bufferSpace);
+ this.session.setIgnoreAllDuplicateRows(true);
+ String multitonKey = String.valueOf(Thread.currentThread().getId());
+ assert(MULTITON.get(multitonKey) == null);
+ MULTITON.put(multitonKey, this);
+ entries.set(MULTITON_KEY, multitonKey);
+ }
+
+ private void shutdownClient() throws IOException {
+ try {
+ client.shutdown();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ public static KuduTable getKuduTable(String multitonKey) {
+ return MULTITON.get(multitonKey).getKuduTable();
+ }
+
+ private KuduTable getKuduTable() {
+ return this.table;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public RecordWriter<NullWritable, Operation> getRecordWriter(TaskAttemptContext taskAttemptContext)
+ throws IOException, InterruptedException {
+ return new TableRecordWriter(this.session);
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext jobContext) throws IOException, InterruptedException {
+ shutdownClient();
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) throws
+ IOException, InterruptedException {
+ return new KuduTableOutputCommitter();
+ }
+
+ protected class TableRecordWriter extends RecordWriter<NullWritable, Operation> {
+
+ private final AtomicLong rowsWithErrors = new AtomicLong();
+ private final KuduSession session;
+
+ public TableRecordWriter(KuduSession session) {
+ this.session = session;
+ }
+
+ @Override
+ public void write(NullWritable key, Operation operation)
+ throws IOException, InterruptedException {
+ try {
+ session.apply(operation);
+ } catch (Exception e) {
+ throw new IOException("Encountered an error while writing", e);
+ }
+ }
+
+ @Override
+ public void close(TaskAttemptContext taskAttemptContext) throws IOException,
+ InterruptedException {
+ try {
+ processRowErrors(session.close());
+ shutdownClient();
+ } catch (Exception e) {
+ throw new IOException("Encountered an error while closing this task", e);
+ } finally {
+ if (taskAttemptContext != null) {
+ // This is the only place where we have access to the context in the record writer,
+ // so set the counter here.
+ taskAttemptContext.getCounter(Counters.ROWS_WITH_ERRORS).setValue(rowsWithErrors.get());
+ }
+ }
+ }
+
+ private void processRowErrors(List<OperationResponse> responses) {
+ List<RowError> errors = OperationResponse.collectErrors(responses);
+ if (!errors.isEmpty()) {
+ int rowErrorsCount = errors.size();
+ rowsWithErrors.addAndGet(rowErrorsCount);
+ LOG.warn("Got per errors for " + rowErrorsCount + " rows, " +
+ "the first one being " + errors.get(0).getStatus());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java
new file mode 100644
index 0000000..7cf3ada
--- /dev/null
+++ b/java/kudu-mapreduce/src/main/java/org/apache/kudu/mapreduce/TableReducer.java
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.Operation;
+import org.apache.hadoop.mapreduce.Reducer;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class TableReducer<KEYIN, VALUEIN, KEYOUT>
+ extends Reducer<KEYIN, VALUEIN, KEYOUT, Operation> {
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/CommandLineParser.java b/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/CommandLineParser.java
deleted file mode 100644
index 05c18f2..0000000
--- a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/CommandLineParser.java
+++ /dev/null
@@ -1,144 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
-import org.apache.hadoop.conf.Configuration;
-import org.kududb.client.KuduClient;
-
-/**
- * Utility class that manages common configurations to all MR jobs. For example,
- * any job that uses {#KuduTableMapReduceUtil} to setup an input or output format
- * and that has parsed the command line arguments with
- * {@link org.apache.hadoop.util.GenericOptionsParser} can simply be passed:
- * <code>
- * -Dmaster.address=ADDRESS
- * </code>
- * in order to specify where the master is.
- * Use {@link CommandLineParser#getHelpSnippet()} to provide usage text for the configurations
- * managed by this class.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class CommandLineParser {
- private final Configuration conf;
- public static final String MASTER_ADDRESSES_KEY = "kudu.master.addresses";
- public static final String MASTER_ADDRESSES_DEFAULT = "127.0.0.1";
- public static final String OPERATION_TIMEOUT_MS_KEY = "kudu.operation.timeout.ms";
- public static final long OPERATION_TIMEOUT_MS_DEFAULT =
- AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
- public static final String ADMIN_OPERATION_TIMEOUT_MS_KEY = "kudu.admin.operation.timeout.ms";
- public static final String SOCKET_READ_TIMEOUT_MS_KEY = "kudu.socket.read.timeout.ms";
- public static final long SOCKET_READ_TIMEOUT_MS_DEFAULT =
- AsyncKuduClient.DEFAULT_SOCKET_READ_TIMEOUT_MS;
- public static final String NUM_REPLICAS_KEY = "kudu.num.replicas";
- public static final int NUM_REPLICAS_DEFAULT = 3;
-
- /**
- * Constructor that uses a Configuration that has already been through
- * {@link org.apache.hadoop.util.GenericOptionsParser}'s command line parsing.
- * @param conf the configuration from which job configurations will be extracted
- */
- public CommandLineParser(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * Get the configured master's config.
- * @return a string that contains the passed config, or the default value
- */
- public String getMasterAddresses() {
- return conf.get(MASTER_ADDRESSES_KEY, MASTER_ADDRESSES_DEFAULT);
- }
-
- /**
- * Get the configured timeout for operations on sessions and scanners.
- * @return a long that represents the passed timeout, or the default value
- */
- public long getOperationTimeoutMs() {
- return conf.getLong(OPERATION_TIMEOUT_MS_KEY, OPERATION_TIMEOUT_MS_DEFAULT);
- }
-
- /**
- * Get the configured timeout for admin operations.
- * @return a long that represents the passed timeout, or the default value
- */
- public long getAdminOperationTimeoutMs() {
- return conf.getLong(ADMIN_OPERATION_TIMEOUT_MS_KEY, OPERATION_TIMEOUT_MS_DEFAULT);
- }
-
- /**
- * Get the configured timeout for socket reads.
- * @return a long that represents the passed timeout, or the default value
- */
- public long getSocketReadTimeoutMs() {
- return conf.getLong(SOCKET_READ_TIMEOUT_MS_KEY, SOCKET_READ_TIMEOUT_MS_DEFAULT);
- }
-
- /**
- * Get the number of replicas to use when configuring a new table.
- * @return an int that represents the passed number of replicas to use, or the default value.
- */
- public int getNumReplicas() {
- return conf.getInt(NUM_REPLICAS_KEY, NUM_REPLICAS_DEFAULT);
- }
-
- /**
- * Get an async client connected to the configured Master(s).
- * @return an async kudu client
- */
- public AsyncKuduClient getAsyncClient() {
- return new AsyncKuduClient.AsyncKuduClientBuilder(getMasterAddresses())
- .defaultOperationTimeoutMs(getOperationTimeoutMs())
- .defaultAdminOperationTimeoutMs(getAdminOperationTimeoutMs())
- .defaultSocketReadTimeoutMs(getSocketReadTimeoutMs())
- .build();
- }
-
- /**
- * Get a client connected to the configured Master(s).
- * @return a kudu client
- */
- public KuduClient getClient() {
- return new KuduClient.KuduClientBuilder(getMasterAddresses())
- .defaultOperationTimeoutMs(getOperationTimeoutMs())
- .defaultAdminOperationTimeoutMs(getAdminOperationTimeoutMs())
- .defaultSocketReadTimeoutMs(getSocketReadTimeoutMs())
- .build();
- }
-
- /**
- * This method returns a single multi-line string that contains the help snippet to append to
- * the tail of a usage() or help() type of method.
- * @return a string with all the available configurations and their defaults
- */
- public static String getHelpSnippet() {
- return "\nAdditionally, the following options are available:" +
- " -D" + OPERATION_TIMEOUT_MS_KEY + "=TIME - timeout for read and write " +
- "operations, defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n"+
- " -D" + ADMIN_OPERATION_TIMEOUT_MS_KEY + "=TIME - timeout for admin operations " +
- ", defaults to " + OPERATION_TIMEOUT_MS_DEFAULT + " \n"+
- " -D" + SOCKET_READ_TIMEOUT_MS_KEY + "=TIME - timeout for socket reads " +
- ", defaults to " + SOCKET_READ_TIMEOUT_MS_DEFAULT + " \n"+
- " -D" + MASTER_ADDRESSES_KEY + "=ADDRESSES - addresses to reach the Masters, " +
- "defaults to " + MASTER_ADDRESSES_DEFAULT + " which is usually wrong.\n" +
- " -D " + NUM_REPLICAS_KEY + "=NUM - number of replicas to use when configuring a new " +
- "table, defaults to " + NUM_REPLICAS_DEFAULT;
- }
-}
[35/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 2)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java
index 4bd2eaf..16b1072 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java
@@ -23,12 +23,12 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.stumbleupon.async.DeferredGroupException;
import com.stumbleupon.async.TimeoutException;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
index dea94c4..02abc30 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
@@ -15,19 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.primitives.UnsignedBytes;
import com.google.protobuf.ByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Common;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java
index 8ebd89d..df46815 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java
@@ -23,7 +23,7 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.CodedOutputStream;
@@ -32,16 +32,16 @@ import com.google.protobuf.Message;
import com.stumbleupon.async.Deferred;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.util.Pair;
+import org.apache.kudu.util.Slice;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Collection;
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+import static org.apache.kudu.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
/**
* Abstract base class for all RPC requests going out to Kudu.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java
index 981e04a..c13bb19 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java
@@ -14,9 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
/**
* Base class for RPC responses.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
index 382a4f0..cfecb67 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
@@ -15,18 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Client.ScanTokenPB;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Common;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.Client.ScanTokenPB;
import java.io.IOException;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java
index 87db768..558f404 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.stumbleupon.async.Deferred;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduScanner.ReadMode;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.AsyncKuduScanner.ReadMode;
/**
* Synchronous version of {@link AsyncKuduScanner}. Offers the same API but with blocking methods.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java
index 61718b2..112b42a 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.stumbleupon.async.DeferredGroupException;
import com.stumbleupon.async.TimeoutException;
-import org.kududb.annotations.*;
+import org.apache.kudu.annotations.*;
import com.stumbleupon.async.Deferred;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java
index 7f59e47..1265f80 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import com.stumbleupon.async.Deferred;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java
index 78725f0..61b27c8 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java
index 70daee2..92e8a0f 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java
index bf26626..4c6a7c9 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import static org.kududb.master.Master.*;
+import static org.apache.kudu.master.Master.*;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java
index 373a14d..dc3767a 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java
index 8a29b7b..f8b835d 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsRequest.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.tablet.Tablet;
-import org.kududb.tserver.Tserver;
-import org.kududb.tserver.TserverService;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.tablet.Tablet;
+import org.apache.kudu.tserver.Tserver;
+import org.apache.kudu.tserver.TserverService;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java
index be2ed65..db12fa2 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletsResponse.java
@@ -14,9 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
index 67934db..3117f64 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
@@ -17,14 +17,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import java.util.List;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.consensus.Metadata.RaftPeerPB.Role;
-import org.kududb.master.Master.TabletLocationsPB.ReplicaPB;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.consensus.Metadata.RaftPeerPB.Role;
+import org.apache.kudu.master.Master.TabletLocationsPB.ReplicaPB;
/**
* Information about the locations of tablets in a Kudu table.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java
index 1cde694..587741e 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NoLeaderMasterFoundException.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java
index 1c3b024..ac72057 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeCache.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Joiner;
import com.google.common.primitives.UnsignedBytes;
@@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import javax.annotation.concurrent.ThreadSafe;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java
index b704441..801da77 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NonCoveredRangeException.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Exception indicating that an operation attempted to access a non-covered range partition.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java
index 7bcb81d..e080043 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/NonRecoverableException.java
@@ -23,10 +23,10 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
@InterfaceAudience.Private
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java
index e27c222..0af0063 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Operation.java
@@ -14,25 +14,25 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.WireProtocol;
-import org.kududb.WireProtocol.RowOperationsPB;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Statistics.Statistic;
-import org.kududb.client.Statistics.TabletStatistics;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.WireProtocol;
+import org.apache.kudu.WireProtocol.RowOperationsPB;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.Statistics.Statistic;
+import org.apache.kudu.client.Statistics.TabletStatistics;
+import org.apache.kudu.tserver.Tserver;
+import org.apache.kudu.util.Pair;
+import org.apache.kudu.util.Slice;
import org.jboss.netty.buffer.ChannelBuffer;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java
index bf707ce..794e341 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/OperationResponse.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.tserver.Tserver;
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java b/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
index b5f3069..0deee84 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PartialRow.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -23,11 +23,11 @@ import java.util.List;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Class used to represent parts of a row along with its schema.<p>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java
index bdc089b..c95fa34 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Partition.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Objects;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.util.Arrays;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java b/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
index fdee32e..19174ca 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PartitionSchema.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java
index 3ca98e2..6b5adb8 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/PleaseThrottleException.java
@@ -23,11 +23,11 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.stumbleupon.async.Deferred;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* This exception notifies the application to throttle its use of Kudu.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
index 2b2cf64..ef31d6d 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
@@ -14,18 +14,18 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.net.HostAndPort;
import com.google.protobuf.ByteString;
import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Common;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.nio.charset.Charset;
import java.util.ArrayList;
@@ -229,7 +229,7 @@ public class ProtobufHelper {
}
/**
- * Convert a {@link com.google.common.net.HostAndPort} to {@link org.kududb.Common.HostPortPB}
+ * Convert a {@link com.google.common.net.HostAndPort} to {@link org.apache.kudu.Common.HostPortPB}
* protobuf message for serialization.
* @param hostAndPort The host and port object. Both host and port must be specified.
* @return An initialized HostPortPB object.
@@ -242,7 +242,7 @@ public class ProtobufHelper {
}
/**
- * Convert a {@link org.kududb.Common.HostPortPB} to {@link com.google.common.net.HostAndPort}.
+ * Convert a {@link org.apache.kudu.Common.HostPortPB} to {@link com.google.common.net.HostAndPort}.
* @param hostPortPB The fully initialized HostPortPB object. Must have both host and port
* specified.
* @return An initialized initialized HostAndPort object.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java
index 25c0fe0..02dfe65 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java
@@ -23,10 +23,10 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* An exception that's possible to retry.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java
index 229b64f..213b182 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java
@@ -14,9 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.util.Queue;
import java.util.concurrent.PriorityBlockingQueue;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java
index b4c8f36..11b53fc 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
+import org.apache.kudu.WireProtocol;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.tserver.Tserver;
/**
* Wrapper class for a single row error.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java
index 17a4778..9c7f074 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Container class used as a response when retrieving pending row errors.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
index 7692c53..6238608 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.Slice;
+package org.apache.kudu.client;
+
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.util.Slice;
import java.nio.ByteBuffer;
import java.text.DateFormat;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java
index 5705ea3..2d4c2d0 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import java.util.Iterator;
-import org.kududb.Schema;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.Slice;
+import org.apache.kudu.Schema;
+import org.apache.kudu.WireProtocol;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.util.Slice;
/**
* Class that contains the rows sent by a tablet server, exhausting this iterator only means
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java b/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java
index 53e108a..8b48efa 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java
@@ -23,7 +23,7 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
@@ -33,8 +33,8 @@ import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.Channels;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.rpc.RpcHeader;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.rpc.RpcHeader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java b/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java
index 94e0a66..a651ae6 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Interface that defines the methods used to configure a session. It also exposes ways to
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java
index ef45f9e..18a9ff8 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.Sets;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.Slice;
-import org.kududb.util.Slices;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.util.Slice;
+import org.apache.kudu.util.Slices;
import java.nio.charset.Charset;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java
index cd0a17d..2592e17 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.master.Master;
-import org.kududb.tserver.Tserver;
+import org.apache.kudu.WireProtocol;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.tserver.Tserver;
/**
* Representation of an error code and message.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java
index dafb6fc..59ecf3b 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java
@@ -24,18 +24,18 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.annotations.VisibleForTesting;
import com.stumbleupon.async.Deferred;
import org.jboss.netty.handler.timeout.ReadTimeoutException;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.rpc.RpcHeader;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.Pair;
+import org.apache.kudu.WireProtocol;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.rpc.RpcHeader;
+import org.apache.kudu.tserver.Tserver;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java
index 3db2026..5f12dff 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Operation to update columns on an existing row. Instances of this class should not be reused.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java
index 4ba2635..3f7bcb5 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Represents a single row upsert. Instances of this class should not be reused.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java
index a93d1b9..c69ec3f 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.util;
+package org.apache.kudu.util;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
/**
* Utility methods for various parts of async, such as Deferred.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java
index 31436e7..2ee68f4 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java
@@ -14,9 +14,9 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.util;
+package org.apache.kudu.util;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.util.concurrent.TimeUnit;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java
index 1ff77a2..589cb8f 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.util;
+package org.apache.kudu.util;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
@@ -22,7 +22,7 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.net.HostAndPort;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java b/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java
index 341ec10..c2a0944 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.util;
+package org.apache.kudu.util;
import com.google.common.base.Objects;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
@InterfaceAudience.Private
public class Pair<A, B> {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java b/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java
index c9d2719..7a3dcae 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java
@@ -16,13 +16,13 @@
* Copyright 2011 Dain Sundstrom <da...@iq80.com>
* Copyright 2011 FuseSource Corp. http://fusesource.com
*/
-package org.kududb.util;
+package org.apache.kudu.util;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import com.google.common.primitives.Shorts;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.io.IOException;
import java.io.InputStream;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java b/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java
index c2cdbde..2fc6555 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java
@@ -19,10 +19,10 @@
* Copyright 2011 Dain Sundstrom <da...@iq80.com>
* Copyright 2011 FuseSource Corp. http://fusesource.com
*/
-package org.kududb.util;
+package org.apache.kudu.util;
import com.google.common.base.Preconditions;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
index 1464fa4..fbc8b09 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
@@ -24,10 +24,10 @@ import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.master.Master;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.master.Master;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
index cb8e968..e582e2f 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java
index 13465f9..0253ce0 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.Lists;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.kududb.Schema;
+import org.apache.kudu.Schema;
import java.util.Random;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
index 955e6ab..49663b6 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
@@ -11,7 +11,7 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
@@ -22,7 +22,7 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.net.HostAndPort;
import org.apache.commons.io.FileUtils;
-import org.kududb.util.NetUtil;
+import org.apache.kudu.util.NetUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
index abec53f..14cf5ec 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Charsets;
import com.google.common.base.Stopwatch;
@@ -27,9 +27,9 @@ import java.util.concurrent.TimeUnit;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.kududb.Common;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
+import org.apache.kudu.Common;
+import org.apache.kudu.consensus.Metadata;
+import org.apache.kudu.master.Master;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
index ba69305..62095da 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.Schema;
-import org.kududb.WireProtocol.AppStatusPB;
-import org.kududb.client.AsyncKuduClient.RemoteTablet;
-import org.kududb.tserver.Tserver.TabletServerErrorPB;
+import org.apache.kudu.Schema;
+import org.apache.kudu.WireProtocol.AppStatusPB;
+import org.apache.kudu.client.AsyncKuduClient.RemoteTablet;
+import org.apache.kudu.tserver.Tserver.TabletServerErrorPB;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java
index ab27e63..ce532cb 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java
index 11c2035..4cf03de 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java
index bf13f2d..8b39a27 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.Lists;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.tserver.Tserver;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Type;
+import org.apache.kudu.tserver.Tserver;
import java.io.IOException;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java
index 20ee06a..b36bd9b 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java
index 883b2e1..09e8ef7 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
index dafd74a..c9b0361 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
@@ -23,9 +23,9 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import org.junit.Before;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
import java.util.ArrayList;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java
index 666ac38..a74594e 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import com.stumbleupon.async.Deferred;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@@ -30,9 +30,9 @@ import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import static org.kududb.Type.STRING;
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
-import static org.kududb.util.HybridTimeUtil.*;
+import static org.apache.kudu.Type.STRING;
+import static org.apache.kudu.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+import static org.apache.kudu.util.HybridTimeUtil.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
index e446445..9b0ed96 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -22,13 +22,13 @@ import static org.junit.Assert.assertTrue;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.ColumnSchema.ColumnSchemaBuilder;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.PartitionSchema.HashBucketSchema;
-import org.kududb.client.PartitionSchema.RangeSchema;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder;
+import org.apache.kudu.Common;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.PartitionSchema.HashBucketSchema;
+import org.apache.kudu.client.PartitionSchema.RangeSchema;
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
index 3591b7b..e16b657 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -14,17 +14,17 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
-import static org.kududb.client.RowResult.timestampToString;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.GREATER;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.LESS;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
+import static org.apache.kudu.client.RowResult.timestampToString;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterators;
@@ -38,9 +38,9 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Before;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java
index 4915a18..3ad39fe 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java
@@ -15,20 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Preconditions;
import org.junit.Assert;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-
-import static org.kududb.client.KuduPredicate.ComparisonOp.EQUAL;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
-import static org.kududb.client.KuduPredicate.PredicateType.RANGE;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Type;
+
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.EQUAL;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.GREATER;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.LESS;
+import static org.apache.kudu.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
+import static org.apache.kudu.client.KuduPredicate.PredicateType.RANGE;
public class TestKuduPredicate {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
index df2367f..bf98157 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
index 4e41a29..97362a6 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.Rule;
import org.junit.rules.TestName;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java
index 49ac502..08a3bdf 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java
index 2f91a6e..9adc36e 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.BeforeClass;
import org.junit.Ignore;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java
index 82ffacb..8b7d207 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java
@@ -11,7 +11,7 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.After;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
index f305fbf..161283b 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -22,12 +22,12 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.WireProtocol.RowOperationsPB;
-import org.kududb.client.Operation.ChangeType;
-import org.kududb.tserver.Tserver.WriteRequestPBOrBuilder;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.WireProtocol.RowOperationsPB;
+import org.apache.kudu.client.Operation.ChangeType;
+import org.apache.kudu.tserver.Tserver.WriteRequestPBOrBuilder;
import org.mockito.Mockito;
import com.google.common.collect.ImmutableList;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java
index 7528de6..83e247d 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java
index 90d11aa..f4929b5 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java
index 1b302c1..0daf3f3 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.kududb.Type;
+import org.apache.kudu.Type;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java
index d67380b..8d6d06e 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java
@@ -15,16 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedSet;
import org.junit.Assert;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.KuduPredicate.ComparisonOp;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.KuduPredicate.ComparisonOp;
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
index 251057f..71739d4 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
@@ -14,20 +14,20 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.stumbleupon.async.Deferred;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.ArrayList;
import static org.junit.Assert.assertNull;
-import static org.kududb.Type.STRING;
+import static org.apache.kudu.Type.STRING;
import static org.junit.Assert.assertEquals;
public class TestScannerMultiTablet extends BaseKuduTest {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java
index 6cfbee7..365f95e 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.kududb.client.Statistics.Statistic;
+import org.apache.kudu.client.Statistics.Statistic;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java
index 11bd23b..dcd061a 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.Test;
-import org.kududb.client.Status;
+import org.apache.kudu.client.Status;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java
index b150f8e..856a5e5 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import org.junit.After;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java
index 3e78918..28d15bf 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
[02/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/HadoopTestingUtility.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/HadoopTestingUtility.java b/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/HadoopTestingUtility.java
deleted file mode 100644
index 1e2cb41..0000000
--- a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/HadoopTestingUtility.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.mapreduce;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- * This class is analog to HBaseTestingUtility except that we only need it for the MR tests.
- */
-public class HadoopTestingUtility {
-
- private static final Log LOG = LogFactory.getLog(HadoopTestingUtility.class);
-
- private File testDir;
-
- private Configuration conf = new Configuration();
-
- /**
- * System property key to get base test directory value
- */
- public static final String BASE_TEST_DIRECTORY_KEY =
- "test.build.data.basedirectory";
-
- /**
- * Default base directory for test output.
- */
- private static final String DEFAULT_BASE_TEST_DIRECTORY = "target/mr-data";
-
- public Configuration getConfiguration() {
- return this.conf;
- }
-
- /**
- * Sets up a temporary directory for the test to run in. Call cleanup() at the end of your
- * tests to remove it.
- * @param testName Will be used to build a part of the directory name for the test
- * @return Where the test is homed
- */
- public File setupAndGetTestDir(String testName, Configuration conf) {
- if (this.testDir != null) {
- return this.testDir;
- }
- Path testPath = new Path(getBaseTestDir(), testName + System.currentTimeMillis());
- this.testDir = new File(testPath.toString()).getAbsoluteFile();
- this.testDir.mkdirs();
- // Set this property so when mapreduce jobs run, they will use this as their home dir.
- System.setProperty("test.build.dir", this.testDir.toString());
- System.setProperty("hadoop.home.dir", this.testDir.toString());
- conf.set("hadoop.tmp.dir", this.testDir.toString() + "/mapred");
-
- LOG.info("Test configured to write to " + this.testDir);
- return this.testDir;
- }
-
- private Path getBaseTestDir() {
- String pathName = System.getProperty(BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
- return new Path(pathName);
- }
-
- public void cleanup() throws IOException {
- FileSystem.closeAll();
- if (this.testDir != null) {
- delete(this.testDir);
- }
- }
-
- private void delete(File dir) throws IOException {
- if (dir == null || !dir.exists()) {
- return;
- }
- try {
- FileUtils.deleteDirectory(dir);
- } catch (IOException ex) {
- LOG.warn("Failed to delete " + dir.getAbsolutePath());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITInputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITInputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITInputFormatJob.java
deleted file mode 100644
index 3d04043..0000000
--- a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITInputFormatJob.java
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.client.KuduPredicate;
-import org.kududb.client.RowResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-public class ITInputFormatJob extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(ITInputFormatJob.class);
-
- private static final String TABLE_NAME =
- ITInputFormatJob.class.getName() + "-" + System.currentTimeMillis();
-
- private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
-
- /** Counter enumeration to count the actual rows. */
- private static enum Counters { ROWS }
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- try {
- BaseKuduTest.tearDownAfterClass();
- } finally {
- HADOOP_UTIL.cleanup();
- }
- }
-
- @Test
- @SuppressWarnings("deprecation")
- public void test() throws Exception {
-
- createFourTabletsTableWithNineRows(TABLE_NAME);
-
- Configuration conf = new Configuration();
- HADOOP_UTIL.setupAndGetTestDir(ITInputFormatJob.class.getName(), conf).getAbsolutePath();
-
- createAndTestJob(conf, new ArrayList<KuduPredicate>(), 9);
-
- KuduPredicate pred1 = KuduPredicate.newComparisonPredicate(
- basicSchema.getColumnByIndex(0), KuduPredicate.ComparisonOp.GREATER_EQUAL, 20);
- createAndTestJob(conf, Lists.newArrayList(pred1), 6);
-
- KuduPredicate pred2 = KuduPredicate.newComparisonPredicate(
- basicSchema.getColumnByIndex(2), KuduPredicate.ComparisonOp.LESS_EQUAL, 1);
- createAndTestJob(conf, Lists.newArrayList(pred1, pred2), 2);
- }
-
- private void createAndTestJob(Configuration conf,
- List<KuduPredicate> predicates, int expectedCount)
- throws Exception {
- String jobName = ITInputFormatJob.class.getName();
- Job job = new Job(conf, jobName);
-
- Class<TestMapperTableInput> mapperClass = TestMapperTableInput.class;
- job.setJarByClass(mapperClass);
- job.setMapperClass(mapperClass);
- job.setNumReduceTasks(0);
- job.setOutputFormatClass(NullOutputFormat.class);
- KuduTableMapReduceUtil.TableInputFormatConfigurator configurator =
- new KuduTableMapReduceUtil.TableInputFormatConfigurator(
- job,
- TABLE_NAME,
- "*",
- getMasterAddresses())
- .operationTimeoutMs(DEFAULT_SLEEP)
- .addDependencies(false)
- .cacheBlocks(false);
- for (KuduPredicate predicate : predicates) {
- configurator.addPredicate(predicate);
- }
- configurator.configure();
-
- assertTrue("Test job did not end properly", job.waitForCompletion(true));
-
- assertEquals(expectedCount, job.getCounters().findCounter(Counters.ROWS).getValue());
- }
-
- /**
- * Simple row counter and printer
- */
- static class TestMapperTableInput extends
- Mapper<NullWritable, RowResult, NullWritable, NullWritable> {
-
- @Override
- protected void map(NullWritable key, RowResult value, Context context) throws IOException,
- InterruptedException {
- context.getCounter(Counters.ROWS).increment(1);
- LOG.info(value.toStringLongFormat()); // useful to visual debugging
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableInputFormat.java b/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableInputFormat.java
deleted file mode 100644
index ff4d81a..0000000
--- a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableInputFormat.java
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.kududb.Schema;
-import org.kududb.client.*;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-public class ITKuduTableInputFormat extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- ITKuduTableInputFormat.class.getName() + "-" + System.currentTimeMillis();
-
- @Test
- public void test() throws Exception {
- createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
-
- KuduTable table = openTable(TABLE_NAME);
- Schema schema = getBasicSchema();
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addInt(0, 1);
- row.addInt(1, 2);
- row.addInt(2, 3);
- row.addString(3, "a string");
- row.addBoolean(4, true);
- AsyncKuduSession session = client.newSession();
- session.apply(insert).join(DEFAULT_SLEEP);
- session.close().join(DEFAULT_SLEEP);
-
- // Test getting all the columns back
- RecordReader<NullWritable, RowResult> reader = createRecordReader("*", null);
- assertTrue(reader.nextKeyValue());
- assertEquals(5, reader.getCurrentValue().getColumnProjection().getColumnCount());
- assertFalse(reader.nextKeyValue());
-
- // Test getting two columns back
- reader = createRecordReader(schema.getColumnByIndex(3).getName() + "," +
- schema.getColumnByIndex(2).getName(), null);
- assertTrue(reader.nextKeyValue());
- assertEquals(2, reader.getCurrentValue().getColumnProjection().getColumnCount());
- assertEquals("a string", reader.getCurrentValue().getString(0));
- assertEquals(3, reader.getCurrentValue().getInt(1));
- try {
- reader.getCurrentValue().getString(2);
- fail("Should only be getting 2 columns back");
- } catch (IndexOutOfBoundsException e) {
- // expected
- }
-
- // Test getting one column back
- reader = createRecordReader(schema.getColumnByIndex(1).getName(), null);
- assertTrue(reader.nextKeyValue());
- assertEquals(1, reader.getCurrentValue().getColumnProjection().getColumnCount());
- assertEquals(2, reader.getCurrentValue().getInt(0));
- try {
- reader.getCurrentValue().getString(1);
- fail("Should only be getting 1 column back");
- } catch (IndexOutOfBoundsException e) {
- // expected
- }
-
- // Test getting empty rows back
- reader = createRecordReader("", null);
- assertTrue(reader.nextKeyValue());
- assertEquals(0, reader.getCurrentValue().getColumnProjection().getColumnCount());
- assertFalse(reader.nextKeyValue());
-
- // Test getting an unknown table, will not work
- try {
- createRecordReader("unknown", null);
- fail("Should not be able to scan a column that doesn't exist");
- } catch (IllegalArgumentException e) {
- // expected
- }
-
- // Test using a predicate that filters the row out.
- KuduPredicate pred1 = KuduPredicate.newComparisonPredicate(
- schema.getColumnByIndex(1), KuduPredicate.ComparisonOp.GREATER_EQUAL, 3);
- reader = createRecordReader("*", Lists.newArrayList(pred1));
- assertFalse(reader.nextKeyValue());
- }
-
- private RecordReader<NullWritable, RowResult> createRecordReader(String columnProjection,
- List<KuduPredicate> predicates) throws IOException, InterruptedException {
- KuduTableInputFormat input = new KuduTableInputFormat();
- Configuration conf = new Configuration();
- conf.set(KuduTableInputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
- conf.set(KuduTableInputFormat.INPUT_TABLE_KEY, TABLE_NAME);
- if (columnProjection != null) {
- conf.set(KuduTableInputFormat.COLUMN_PROJECTION_KEY, columnProjection);
- }
- if (predicates != null) {
- String encodedPredicates = KuduTableMapReduceUtil.base64EncodePredicates(predicates);
- conf.set(KuduTableInputFormat.ENCODED_PREDICATES_KEY, encodedPredicates);
- }
- input.setConf(conf);
- List<InputSplit> splits = input.getSplits(null);
-
- // We need to re-create the input format to reconnect the client.
- input = new KuduTableInputFormat();
- input.setConf(conf);
- RecordReader<NullWritable, RowResult> reader = input.createRecordReader(null, null);
- reader.initialize(Iterables.getOnlyElement(splits), null);
- return reader;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableOutputFormat.java b/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableOutputFormat.java
deleted file mode 100644
index 86452ed..0000000
--- a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITKuduTableOutputFormat.java
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import org.kududb.client.*;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class ITKuduTableOutputFormat extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- ITKuduTableOutputFormat.class.getName() + "-" + System.currentTimeMillis();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- }
-
- @Test
- public void test() throws Exception {
- createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
-
- KuduTableOutputFormat output = new KuduTableOutputFormat();
- Configuration conf = new Configuration();
- conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
- conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, TABLE_NAME);
- output.setConf(conf);
-
- String multitonKey = conf.get(KuduTableOutputFormat.MULTITON_KEY);
- KuduTable table = KuduTableOutputFormat.getKuduTable(multitonKey);
- assertNotNull(table);
-
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addInt(0, 1);
- row.addInt(1, 2);
- row.addInt(2, 3);
- row.addString(3, "a string");
- row.addBoolean(4, true);
-
- RecordWriter<NullWritable, Operation> rw = output.getRecordWriter(null);
- rw.write(NullWritable.get(), insert);
- rw.close(null);
- AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
- assertEquals(1, countRowsInScan(builder.build()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITOutputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITOutputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITOutputFormatJob.java
deleted file mode 100644
index dff2400..0000000
--- a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/ITOutputFormatJob.java
+++ /dev/null
@@ -1,131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import org.kududb.client.*;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.*;
-
-public class ITOutputFormatJob extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- ITOutputFormatJob.class.getName() + "-" + System.currentTimeMillis();
-
- private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- try {
- BaseKuduTest.tearDownAfterClass();
- } finally {
- HADOOP_UTIL.cleanup();
- }
- }
-
- @Test
- @SuppressWarnings("deprecation")
- public void test() throws Exception {
- Configuration conf = new Configuration();
- String testHome =
- HADOOP_UTIL.setupAndGetTestDir(ITOutputFormatJob.class.getName(), conf).getAbsolutePath();
- String jobName = ITOutputFormatJob.class.getName();
- Job job = new Job(conf, jobName);
-
-
- // Create a 2 lines input file
- File data = new File(testHome, "data.txt");
- writeDataFile(data);
- FileInputFormat.setInputPaths(job, data.toString());
-
- // Configure the job to map the file and write to kudu, without reducers
- Class<TestMapperTableOutput> mapperClass = TestMapperTableOutput.class;
- job.setJarByClass(mapperClass);
- job.setMapperClass(mapperClass);
- job.setInputFormatClass(TextInputFormat.class);
- job.setNumReduceTasks(0);
- new KuduTableMapReduceUtil.TableOutputFormatConfigurator(
- job,
- TABLE_NAME,
- getMasterAddresses())
- .operationTimeoutMs(DEFAULT_SLEEP)
- .addDependencies(false)
- .configure();
-
- assertTrue("Test job did not end properly", job.waitForCompletion(true));
-
- // Make sure the data's there
- KuduTable table = openTable(TABLE_NAME);
- AsyncKuduScanner.AsyncKuduScannerBuilder builder =
- client.newScannerBuilder(table);
- assertEquals(2, countRowsInScan(builder.build()));
- }
-
- /**
- * Simple Mapper that writes one row per line, the key is the line number and the STRING column
- * is the data from that line
- */
- static class TestMapperTableOutput extends
- Mapper<LongWritable, Text, NullWritable, Operation> {
-
- private KuduTable table;
- @Override
- protected void map(LongWritable key, Text value, Context context) throws IOException,
- InterruptedException {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addInt(0, (int) key.get());
- row.addInt(1, 1);
- row.addInt(2, 2);
- row.addString(3, value.toString());
- row.addBoolean(4, true);
- context.write(NullWritable.get(), insert);
- }
-
- @Override
- protected void setup(Context context) throws IOException, InterruptedException {
- super.setup(context);
- table = KuduTableMapReduceUtil.getTableFromContext(context);
- }
- }
-
- private void writeDataFile(File data) throws IOException {
- FileOutputStream fos = new FileOutputStream(data);
- fos.write("VALUE1\nVALUE2\n".getBytes());
- fos.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/TestJarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/TestJarFinder.java b/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/TestJarFinder.java
deleted file mode 100644
index 3801a0c..0000000
--- a/java/kudu-mapreduce/src/test/java/org/kududb/mapreduce/TestJarFinder.java
+++ /dev/null
@@ -1,128 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import org.apache.commons.logging.LogFactory;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.Writer;
-import java.text.MessageFormat;
-import java.util.Properties;
-import java.util.jar.JarInputStream;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-
-/**
- * This file was forked from hbase/branches/master@4ce6f48.
- */
-public class TestJarFinder {
-
- @Test
- public void testJar() throws Exception {
-
- // Picking a class that is for sure in a JAR in the classpath
- String jar = JarFinder.getJar(LogFactory.class);
- Assert.assertTrue(new File(jar).exists());
- }
-
- private static void delete(File file) throws IOException {
- if (file.getAbsolutePath().length() < 5) {
- throw new IllegalArgumentException(
- MessageFormat.format("Path [{0}] is too short, not deleting",
- file.getAbsolutePath()));
- }
- if (file.exists()) {
- if (file.isDirectory()) {
- File[] children = file.listFiles();
- if (children != null) {
- for (File child : children) {
- delete(child);
- }
- }
- }
- if (!file.delete()) {
- throw new RuntimeException(
- MessageFormat.format("Could not delete path [{0}]",
- file.getAbsolutePath()));
- }
- }
- }
-
- @Test
- public void testExpandedClasspath() throws Exception {
- // Picking a class that is for sure in a directory in the classpath
- // In this case, the JAR is created on the fly
- String jar = JarFinder.getJar(TestJarFinder.class);
- Assert.assertTrue(new File(jar).exists());
- }
-
- @Test
- public void testExistingManifest() throws Exception {
- File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
- TestJarFinder.class.getName() + "-testExistingManifest");
- delete(dir);
- dir.mkdirs();
-
- File metaInfDir = new File(dir, "META-INF");
- metaInfDir.mkdirs();
- File manifestFile = new File(metaInfDir, "MANIFEST.MF");
- Manifest manifest = new Manifest();
- OutputStream os = new FileOutputStream(manifestFile);
- manifest.write(os);
- os.close();
-
- File propsFile = new File(dir, "props.properties");
- Writer writer = new FileWriter(propsFile);
- new Properties().store(writer, "");
- writer.close();
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- JarOutputStream zos = new JarOutputStream(baos);
- JarFinder.jarDir(dir, "", zos);
- JarInputStream jis =
- new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
- Assert.assertNotNull(jis.getManifest());
- jis.close();
- }
-
- @Test
- public void testNoManifest() throws Exception {
- File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
- TestJarFinder.class.getName() + "-testNoManifest");
- delete(dir);
- dir.mkdirs();
- File propsFile = new File(dir, "props.properties");
- Writer writer = new FileWriter(propsFile);
- new Properties().store(writer, "");
- writer.close();
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- JarOutputStream zos = new JarOutputStream(baos);
- JarFinder.jarDir(dir, "", zos);
- JarInputStream jis =
- new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
- Assert.assertNotNull(jis.getManifest());
- jis.close();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
new file mode 100644
index 0000000..95145d5
--- /dev/null
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.kududb.spark.kudu
+
+import java.sql.Timestamp
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.sources._
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
+import org.kududb.Type
+import org.kududb.annotations.InterfaceStability
+import org.kududb.client._
+import org.kududb.client.KuduPredicate.ComparisonOp
+import org.kududb.client.SessionConfiguration.FlushMode
+import org.apache.spark.sql.SaveMode._
+
+import scala.collection.JavaConverters._
+
+/**
+ * DefaultSource for integration with Spark's dataframe datasources.
+ * This class will produce a relationProvider based on input given to it from spark.
+ */
+@InterfaceStability.Unstable
+class DefaultSource extends RelationProvider with CreatableRelationProvider {
+
+ val TABLE_KEY = "kudu.table"
+ val KUDU_MASTER = "kudu.master"
+
+ /**
+ * Construct a BaseRelation using the provided context and parameters.
+ *
+ * @param sqlContext SparkSQL context
+ * @param parameters parameters given to us from SparkSQL
+ * @return a BaseRelation Object
+ */
+ override def createRelation(sqlContext: SQLContext,
+ parameters: Map[String, String]):
+ BaseRelation = {
+ val tableName = parameters.getOrElse(TABLE_KEY,
+ throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
+ s"using key '$TABLE_KEY'"))
+ val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+
+ new KuduRelation(tableName, kuduMaster)(sqlContext)
+ }
+
+ /**
+ * Creates a relation and inserts data to specified table.
+ *
+ * @param sqlContext
+ * @param mode Append will not overwrite existing data, Overwrite will perform update, but will
+ * not insert data, use upsert on KuduContext if you require both
+ * @param parameters Nessisary parameters for kudu.table and kudu.master
+ * @param data Dataframe to save into kudu
+ * @return returns populated base relation
+ */
+ override def createRelation(sqlContext: SQLContext, mode: SaveMode,
+ parameters: Map[String, String], data: DataFrame): BaseRelation = {
+ val tableName = parameters.getOrElse(TABLE_KEY,
+ throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
+ s"using key '$TABLE_KEY'"))
+
+ val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+
+ val kuduRelation = new KuduRelation(tableName, kuduMaster)(sqlContext)
+ mode match {
+ case Append | Ignore => kuduRelation.insert(data, overwrite = false)
+ case Overwrite => kuduRelation.insert(data, overwrite = true)
+ case ErrorIfExists =>
+ throw new UnsupportedOperationException(
+ "ErrorIfExists is currently not supported")
+ }
+
+ kuduRelation
+ }
+}
+
+/**
+ * Implementation of Spark BaseRelation.
+ *
+ * @param tableName Kudu table that we plan to read from
+ * @param kuduMaster Kudu master addresses
+ * @param sqlContext SparkSQL context
+ */
+@InterfaceStability.Unstable
+class KuduRelation(private val tableName: String,
+ private val kuduMaster: String)(
+ val sqlContext: SQLContext)
+extends BaseRelation
+with PrunedFilteredScan
+with InsertableRelation {
+
+ import KuduRelation._
+
+ private val context: KuduContext = new KuduContext(kuduMaster)
+ private val table: KuduTable = context.syncClient.openTable(tableName)
+
+ override def unhandledFilters(filters: Array[Filter]): Array[Filter] =
+ filters.filterNot(supportsFilter)
+
+ /**
+ * Generates a SparkSQL schema object so SparkSQL knows what is being
+ * provided by this BaseRelation.
+ *
+ * @return schema generated from the Kudu table's schema
+ */
+ override def schema: StructType = {
+ val fields: Array[StructField] =
+ table.getSchema.getColumns.asScala.map { columnSchema =>
+ val sparkType = kuduTypeToSparkType(columnSchema.getType)
+ new StructField(columnSchema.getName, sparkType, columnSchema.isNullable)
+ }.toArray
+
+ new StructType(fields)
+ }
+
+ /**
+ * Build the RDD to scan rows.
+ *
+ * @param requiredColumns columns that are being requested by the requesting query
+ * @param filters filters that are being applied by the requesting query
+ * @return RDD will all the results from Kudu
+ */
+ override def buildScan(requiredColumns: Array[String], filters: Array[Filter]): RDD[Row] = {
+ val predicates = filters.flatMap(filterToPredicate)
+ new KuduRDD(kuduMaster, 1024 * 1024 * 20, requiredColumns, predicates,
+ table, context, sqlContext.sparkContext)
+ }
+
+ /**
+ * Converts a Spark [[Filter]] to a Kudu [[KuduPredicate]].
+ *
+ * @param filter the filter to convert
+ * @return the converted filter
+ */
+ private def filterToPredicate(filter : Filter) : Array[KuduPredicate] = {
+ filter match {
+ case EqualTo(column, value) =>
+ Array(comparisonPredicate(column, ComparisonOp.EQUAL, value))
+ case GreaterThan(column, value) =>
+ Array(comparisonPredicate(column, ComparisonOp.GREATER, value))
+ case GreaterThanOrEqual(column, value) =>
+ Array(comparisonPredicate(column, ComparisonOp.GREATER_EQUAL, value))
+ case LessThan(column, value) =>
+ Array(comparisonPredicate(column, ComparisonOp.LESS, value))
+ case LessThanOrEqual(column, value) =>
+ Array(comparisonPredicate(column, ComparisonOp.LESS_EQUAL, value))
+ case And(left, right) => filterToPredicate(left) ++ filterToPredicate(right)
+ case _ => Array()
+ }
+ }
+
+ /**
+ * Creates a new comparison predicate for the column, comparison operator, and comparison value.
+ *
+ * @param column the column name
+ * @param operator the comparison operator
+ * @param value the comparison value
+ * @return the comparison predicate
+ */
+ private def comparisonPredicate(column: String,
+ operator: ComparisonOp,
+ value: Any): KuduPredicate = {
+ val columnSchema = table.getSchema.getColumn(column)
+ value match {
+ case value: Boolean => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Byte => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Short => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Int => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Long => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Timestamp => KuduPredicate.newComparisonPredicate(columnSchema, operator, timestampToMicros(value))
+ case value: Float => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Double => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: String => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Array[Byte] => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ }
+ }
+
+ /**
+ * Inserts data into an existing Kudu table.
+ * @param data [[DataFrame]] to be inserted into Kudu
+ * @param overwrite If True it will update existing records, but will not perform inserts.
+ */
+ override def insert(data: DataFrame, overwrite: Boolean): Unit = {
+ context.writeRows(data, tableName, overwrite)
+ }
+}
+
+private[spark] object KuduRelation {
+ /**
+ * Converts a Kudu [[Type]] to a Spark SQL [[DataType]].
+ *
+ * @param t the Kudu type
+ * @return the corresponding Spark SQL type
+ */
+ private def kuduTypeToSparkType(t: Type): DataType = t match {
+ case Type.BOOL => BooleanType
+ case Type.INT8 => ByteType
+ case Type.INT16 => ShortType
+ case Type.INT32 => IntegerType
+ case Type.INT64 => LongType
+ case Type.TIMESTAMP => TimestampType
+ case Type.FLOAT => FloatType
+ case Type.DOUBLE => DoubleType
+ case Type.STRING => StringType
+ case Type.BINARY => BinaryType
+ }
+
+ /**
+ * Returns `true` if the filter is able to be pushed down to Kudu.
+ *
+ * @param filter the filter to test
+ */
+ private def supportsFilter(filter: Filter): Boolean = filter match {
+ case EqualTo(_, _)
+ | GreaterThan(_, _)
+ | GreaterThanOrEqual(_, _)
+ | LessThan(_, _)
+ | LessThanOrEqual(_, _) => true
+ case And(left, right) => supportsFilter(left) && supportsFilter(right)
+ case _ => false
+ }
+
+ /**
+ * Converts a [[Timestamp]] to microseconds since the Unix epoch (1970-01-01T00:00:00Z).
+ *
+ * @param timestamp the timestamp to convert to microseconds
+ * @return the microseconds since the Unix epoch
+ */
+ def timestampToMicros(timestamp: Timestamp): Long = {
+ // Number of whole milliseconds since the Unix epoch, in microseconds.
+ val millis = timestamp.getTime * 1000
+ // Sub millisecond time since the Unix epoch, in microseconds.
+ val micros = (timestamp.getNanos % 1000000) / 1000
+ if (micros >= 0) {
+ millis + micros
+ } else {
+ millis + 1000000 + micros
+ }
+ }
+
+ /**
+ * Converts a microsecond offset from the Unix epoch (1970-01-01T00:00:00Z) to a [[Timestamp]].
+ *
+ * @param micros the offset in microseconds since the Unix epoch
+ * @return the corresponding timestamp
+ */
+ def microsToTimestamp(micros: Long): Timestamp = {
+ var millis = micros / 1000
+ var nanos = (micros % 1000000) * 1000
+ if (nanos < 0) {
+ millis -= 1
+ nanos += 1000000000
+ }
+
+ val timestamp = new Timestamp(millis)
+ timestamp.setNanos(nanos.asInstanceOf[Int])
+ timestamp
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
new file mode 100644
index 0000000..167ee13
--- /dev/null
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.kududb.spark.kudu
+
+import java.util
+import org.apache.hadoop.util.ShutdownHookManager
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.types.{StructType, DataType, DataTypes}
+import org.kududb.{ColumnSchema, Schema, Type}
+import org.kududb.annotations.InterfaceStability
+import org.kududb.client.SessionConfiguration.FlushMode
+import org.kududb.client._
+
+
+import scala.collection.mutable
+
+/**
+ * KuduContext is a serializable container for Kudu client connections.
+ *
+ * If a Kudu client connection is needed as part of a Spark application, a
+ * [[KuduContext]] should used as a broadcast variable in the job in order to
+ * share connections among the tasks in a JVM.
+ */
+@InterfaceStability.Unstable
+class KuduContext(kuduMaster: String) extends Serializable {
+
+ @transient lazy val syncClient = {
+ KuduConnection.getSyncClient(kuduMaster)
+ }
+
+ @transient lazy val asyncClient = {
+ KuduConnection.getAsyncClient(kuduMaster)
+ }
+
+ /**
+ * Create an RDD from a Kudu table.
+ *
+ * @param tableName table to read from
+ * @param columnProjection list of columns to read. Not specifying this at all
+ * (i.e. setting to null) or setting to the special
+ * string '*' means to project all columns.
+ * @return a new RDD that maps over the given table for the selected columns
+ */
+ def kuduRDD(sc: SparkContext,
+ tableName: String,
+ columnProjection: Seq[String] = Nil): RDD[Row] = {
+ new KuduRDD(kuduMaster, 1024*1024*20, columnProjection.toArray, Array(),
+ syncClient.openTable(tableName), this, sc)
+ }
+
+ /**
+ * Check if kudu table already exists
+ * @param tableName tablename to check
+ * @return true if table exists, false if table does not exist
+ */
+ def tableExists(tableName: String): Boolean = syncClient.tableExists(tableName)
+
+ /**
+ * Delete kudu table
+ * @param tableName tablename to delete
+ * @return DeleteTableResponse
+ */
+ def deleteTable(tableName: String): DeleteTableResponse = syncClient.deleteTable(tableName)
+
+ /**
+ * Creates a kudu table for the given schema. Partitioning can be specified through options.
+ * @param tableName table to create
+ * @param schema struct schema of table
+ * @param keys primary keys of the table
+ * @param options replication and partitioning options for the table
+ */
+ def createTable(tableName: String,
+ schema: StructType,
+ keys: Seq[String],
+ options: CreateTableOptions): KuduTable = {
+ val kuduCols = new util.ArrayList[ColumnSchema]()
+ // add the key columns first, in the order specified
+ for (key <- keys) {
+ val f = schema.fields(schema.fieldIndex(key))
+ kuduCols.add(new ColumnSchema.ColumnSchemaBuilder(f.name, kuduType(f.dataType)).key(true).build())
+ }
+ // now add the non-key columns
+ for (f <- schema.fields.filter(field=> !keys.contains(field.name))) {
+ kuduCols.add(new ColumnSchema.ColumnSchemaBuilder(f.name, kuduType(f.dataType)).nullable(f.nullable).key(false).build())
+ }
+
+ syncClient.createTable(tableName, new Schema(kuduCols), options)
+ }
+
+ /** Map Spark SQL type to Kudu type */
+ def kuduType(dt: DataType) : Type = dt match {
+ case DataTypes.BinaryType => Type.BINARY
+ case DataTypes.BooleanType => Type.BOOL
+ case DataTypes.StringType => Type.STRING
+ case DataTypes.TimestampType => Type.TIMESTAMP
+ case DataTypes.ByteType => Type.INT8
+ case DataTypes.ShortType => Type.INT16
+ case DataTypes.IntegerType => Type.INT32
+ case DataTypes.LongType => Type.INT64
+ case DataTypes.FloatType => Type.FLOAT
+ case DataTypes.DoubleType => Type.DOUBLE
+ case _ => throw new IllegalArgumentException(s"No support for Spark SQL type $dt")
+ }
+
+ /**
+ * Inserts or updates rows in kudu from a [[DataFrame]].
+ * @param data `DataFrame` to insert/update
+ * @param tableName table to perform insertion on
+ * @param overwrite true=update, false=insert
+ */
+ def writeRows(data: DataFrame, tableName: String, overwrite: Boolean) {
+ val schema = data.schema
+ data.foreachPartition(iterator => {
+ val pendingErrors = writeRows(iterator, schema, tableName, overwrite)
+ val errorCount = pendingErrors.getRowErrors.length
+ if (errorCount > 0) {
+ val errors = pendingErrors.getRowErrors.take(5).map(_.getErrorStatus).mkString
+ throw new RuntimeException(
+ s"failed to write $errorCount rows from DataFrame to Kudu; sample errors: $errors")
+ }
+ })
+ }
+
+ /**
+ * Saves partitions of a [[DataFrame]] into Kudu.
+ * @param rows rows to insert or update
+ * @param tableName table to insert or update on
+ */
+ def writeRows(rows: Iterator[Row],
+ schema: StructType,
+ tableName: String,
+ performAsUpdate : Boolean = false): RowErrorsAndOverflowStatus = {
+ val table: KuduTable = syncClient.openTable(tableName)
+ val kuduSchema = table.getSchema
+ val indices: Array[(Int, Int)] = schema.fields.zipWithIndex.map({ case (field, sparkIdx) =>
+ sparkIdx -> table.getSchema.getColumnIndex(field.name)
+ })
+ val session: KuduSession = syncClient.newSession
+ session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND)
+ session.setIgnoreAllDuplicateRows(true)
+ try {
+ for (row <- rows) {
+ val operation = if (performAsUpdate) { table.newUpdate() } else { table.newInsert() }
+ for ((sparkIdx, kuduIdx) <- indices) {
+ if (row.isNullAt(sparkIdx)) {
+ operation.getRow.setNull(kuduIdx)
+ } else schema.fields(sparkIdx).dataType match {
+ case DataTypes.StringType => operation.getRow.addString(kuduIdx, row.getString(sparkIdx))
+ case DataTypes.BinaryType => operation.getRow.addBinary(kuduIdx, row.getAs[Array[Byte]](sparkIdx))
+ case DataTypes.BooleanType => operation.getRow.addBoolean(kuduIdx, row.getBoolean(sparkIdx))
+ case DataTypes.ByteType => operation.getRow.addByte(kuduIdx, row.getByte(sparkIdx))
+ case DataTypes.ShortType => operation.getRow.addShort(kuduIdx, row.getShort(sparkIdx))
+ case DataTypes.IntegerType => operation.getRow.addInt(kuduIdx, row.getInt(sparkIdx))
+ case DataTypes.LongType => operation.getRow.addLong(kuduIdx, row.getLong(sparkIdx))
+ case DataTypes.FloatType => operation.getRow.addFloat(kuduIdx, row.getFloat(sparkIdx))
+ case DataTypes.DoubleType => operation.getRow.addDouble(kuduIdx, row.getDouble(sparkIdx))
+ case DataTypes.TimestampType => operation.getRow.addLong(kuduIdx, KuduRelation.timestampToMicros(row.getTimestamp(sparkIdx)))
+ case t => throw new IllegalArgumentException(s"No support for Spark SQL type $t")
+ }
+ }
+ session.apply(operation)
+ }
+ } finally {
+ session.close()
+ }
+ session.getPendingErrors
+ }
+
+}
+
+private object KuduConnection {
+ private val syncCache = new mutable.HashMap[String, KuduClient]()
+ private val asyncCache = new mutable.HashMap[String, AsyncKuduClient]()
+
+ /**
+ * Set to
+ * [[org.apache.spark.util.ShutdownHookManager.DEFAULT_SHUTDOWN_PRIORITY]].
+ * The client instances are closed through the JVM shutdown hook
+ * mechanism in order to make sure that any unflushed writes are cleaned up
+ * properly. Spark has no shutdown notifications.
+ */
+ private val ShutdownHookPriority = 100
+
+ def getSyncClient(kuduMaster: String): KuduClient = {
+ syncCache.synchronized {
+ if (!syncCache.contains(kuduMaster)) {
+ val syncClient = new KuduClient.KuduClientBuilder(kuduMaster).build()
+ ShutdownHookManager.get().addShutdownHook(new Runnable {
+ override def run() = syncClient.close()
+ }, ShutdownHookPriority)
+ syncCache.put(kuduMaster, syncClient)
+ }
+ return syncCache(kuduMaster)
+ }
+ }
+
+ def getAsyncClient(kuduMaster: String): AsyncKuduClient = {
+ asyncCache.synchronized {
+ if (!asyncCache.contains(kuduMaster)) {
+ val asyncClient = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMaster).build()
+ ShutdownHookManager.get().addShutdownHook(
+ new Runnable {
+ override def run() = asyncClient.close()
+ }, ShutdownHookPriority)
+ asyncCache.put(kuduMaster, asyncClient)
+ }
+ return asyncCache(kuduMaster)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
new file mode 100644
index 0000000..5395d5a
--- /dev/null
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.spark.kudu
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.Row
+import org.apache.spark.{Partition, SparkContext, TaskContext}
+import org.kududb.client._
+import org.kududb.{Type, client}
+
+import scala.collection.JavaConverters._
+
+/**
+ * A Resilient Distributed Dataset backed by a Kudu table.
+ */
+class KuduRDD(val kuduMaster: String,
+ @transient batchSize: Integer,
+ @transient projectedCols: Array[String],
+ @transient predicates: Array[client.KuduPredicate],
+ @transient table: KuduTable,
+ @transient kc: KuduContext,
+ @transient sc: SparkContext) extends RDD[Row](sc, Nil) {
+
+ /**
+ * The [[KuduContext]] for this `KuduRDD`.
+ *
+ * The `KuduContext` manages the Kudu client instances for the `KuduRDD`.
+ * When the `KuduRDD` is first constructed it uses the context passed in as
+ * `kc`. After deserialization, a new `KuduContext` is created as necessary.
+ * The `kc` field should not be used, since it will not be rehydrated after
+ * serialization.
+ */
+ @transient private lazy val kuduContext: KuduContext = {
+ if (kc != null) kc else new KuduContext(kuduMaster)
+ }
+
+ override protected def getPartitions: Array[Partition] = {
+ val builder = kuduContext.syncClient
+ .newScanTokenBuilder(table)
+ .batchSizeBytes(batchSize)
+ .setProjectedColumnNames(projectedCols.toSeq.asJava)
+ .cacheBlocks(true)
+
+ for (predicate <- predicates) {
+ builder.addPredicate(predicate)
+ }
+ val tokens = builder.build().asScala
+ tokens.zipWithIndex.map {
+ case (token, index) =>
+ new KuduPartition(index, token.serialize(),
+ token.getTablet.getReplicas.asScala.map(_.getRpcHost).toArray)
+ }.toArray
+ }
+
+ override def compute(part: Partition, taskContext: TaskContext): Iterator[Row] = {
+ val client: KuduClient = kuduContext.syncClient
+ val partition: KuduPartition = part.asInstanceOf[KuduPartition]
+ val scanner = KuduScanToken.deserializeIntoScanner(partition.scanToken, client)
+ new RowResultIteratorScala(scanner)
+ }
+
+ override def getPreferredLocations(partition: Partition): Seq[String] = {
+ partition.asInstanceOf[KuduPartition].locations
+ }
+}
+
+/**
+ * A Spark SQL [[Partition]] which wraps a [[KuduScanToken]].
+ */
+private[spark] class KuduPartition(val index: Int,
+ val scanToken: Array[Byte],
+ val locations : Array[String]) extends Partition {}
+
+/**
+ * A Spark SQL [[Row]] iterator which wraps a [[KuduScanner]].
+ * @param scanner the wrapped scanner
+ */
+private[spark] class RowResultIteratorScala(private val scanner: KuduScanner) extends Iterator[Row] {
+
+ private var currentIterator: RowResultIterator = null
+
+ override def hasNext: Boolean = {
+ if ((currentIterator != null && !currentIterator.hasNext && scanner.hasMoreRows) ||
+ (scanner.hasMoreRows && currentIterator == null)) {
+ currentIterator = scanner.nextRows()
+ }
+ currentIterator.hasNext
+ }
+
+ override def next(): Row = new KuduRow(currentIterator.next())
+}
+
+/**
+ * A Spark SQL [[Row]] which wraps a Kudu [[RowResult]].
+ * @param rowResult the wrapped row result
+ */
+private[spark] class KuduRow(private val rowResult: RowResult) extends Row {
+ override def length: Int = rowResult.getColumnProjection.getColumnCount
+
+ override def get(i: Int): Any = {
+ if (rowResult.isNull(i)) null
+ else rowResult.getColumnType(i) match {
+ case Type.BOOL => rowResult.getBoolean(i)
+ case Type.INT8 => rowResult.getByte(i)
+ case Type.INT16 => rowResult.getShort(i)
+ case Type.INT32 => rowResult.getInt(i)
+ case Type.INT64 => rowResult.getLong(i)
+ case Type.TIMESTAMP => KuduRelation.microsToTimestamp(rowResult.getLong(i))
+ case Type.FLOAT => rowResult.getFloat(i)
+ case Type.DOUBLE => rowResult.getDouble(i)
+ case Type.STRING => rowResult.getString(i)
+ case Type.BINARY => rowResult.getBinary(i)
+ }
+ }
+
+ override def copy(): Row = Row.fromSeq(Range(0, length).map(get))
+
+ override def toString(): String = rowResult.toString
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
new file mode 100755
index 0000000..4203e31
--- /dev/null
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/package.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.spark
+
+import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter}
+
+package object kudu {
+
+ /**
+ * Adds a method, `kudu`, to DataFrameReader that allows you to read Kudu tables using
+ * the DataFrameReader.
+ */
+ implicit class KuduDataFrameReader(reader: DataFrameReader) {
+ def kudu: DataFrame = reader.format("org.kududb.spark.kudu").load
+ }
+
+ /**
+ * Adds a method, `kudu`, to DataFrameWriter that allows writes to Kudu using
+ * the DataFileWriter
+ */
+ implicit class KuduDataFrameWriter(writer: DataFrameWriter) {
+ def kudu = writer.format("org.kududb.spark.kudu").save
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/DefaultSource.scala
deleted file mode 100644
index 95145d5..0000000
--- a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/DefaultSource.scala
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.kududb.spark.kudu
-
-import java.sql.Timestamp
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.sources._
-import org.apache.spark.sql.types._
-import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
-import org.kududb.Type
-import org.kududb.annotations.InterfaceStability
-import org.kududb.client._
-import org.kududb.client.KuduPredicate.ComparisonOp
-import org.kududb.client.SessionConfiguration.FlushMode
-import org.apache.spark.sql.SaveMode._
-
-import scala.collection.JavaConverters._
-
-/**
- * DefaultSource for integration with Spark's dataframe datasources.
- * This class will produce a relationProvider based on input given to it from spark.
- */
-@InterfaceStability.Unstable
-class DefaultSource extends RelationProvider with CreatableRelationProvider {
-
- val TABLE_KEY = "kudu.table"
- val KUDU_MASTER = "kudu.master"
-
- /**
- * Construct a BaseRelation using the provided context and parameters.
- *
- * @param sqlContext SparkSQL context
- * @param parameters parameters given to us from SparkSQL
- * @return a BaseRelation Object
- */
- override def createRelation(sqlContext: SQLContext,
- parameters: Map[String, String]):
- BaseRelation = {
- val tableName = parameters.getOrElse(TABLE_KEY,
- throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
- s"using key '$TABLE_KEY'"))
- val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
-
- new KuduRelation(tableName, kuduMaster)(sqlContext)
- }
-
- /**
- * Creates a relation and inserts data to specified table.
- *
- * @param sqlContext
- * @param mode Append will not overwrite existing data, Overwrite will perform update, but will
- * not insert data, use upsert on KuduContext if you require both
- * @param parameters Nessisary parameters for kudu.table and kudu.master
- * @param data Dataframe to save into kudu
- * @return returns populated base relation
- */
- override def createRelation(sqlContext: SQLContext, mode: SaveMode,
- parameters: Map[String, String], data: DataFrame): BaseRelation = {
- val tableName = parameters.getOrElse(TABLE_KEY,
- throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
- s"using key '$TABLE_KEY'"))
-
- val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
-
- val kuduRelation = new KuduRelation(tableName, kuduMaster)(sqlContext)
- mode match {
- case Append | Ignore => kuduRelation.insert(data, overwrite = false)
- case Overwrite => kuduRelation.insert(data, overwrite = true)
- case ErrorIfExists =>
- throw new UnsupportedOperationException(
- "ErrorIfExists is currently not supported")
- }
-
- kuduRelation
- }
-}
-
-/**
- * Implementation of Spark BaseRelation.
- *
- * @param tableName Kudu table that we plan to read from
- * @param kuduMaster Kudu master addresses
- * @param sqlContext SparkSQL context
- */
-@InterfaceStability.Unstable
-class KuduRelation(private val tableName: String,
- private val kuduMaster: String)(
- val sqlContext: SQLContext)
-extends BaseRelation
-with PrunedFilteredScan
-with InsertableRelation {
-
- import KuduRelation._
-
- private val context: KuduContext = new KuduContext(kuduMaster)
- private val table: KuduTable = context.syncClient.openTable(tableName)
-
- override def unhandledFilters(filters: Array[Filter]): Array[Filter] =
- filters.filterNot(supportsFilter)
-
- /**
- * Generates a SparkSQL schema object so SparkSQL knows what is being
- * provided by this BaseRelation.
- *
- * @return schema generated from the Kudu table's schema
- */
- override def schema: StructType = {
- val fields: Array[StructField] =
- table.getSchema.getColumns.asScala.map { columnSchema =>
- val sparkType = kuduTypeToSparkType(columnSchema.getType)
- new StructField(columnSchema.getName, sparkType, columnSchema.isNullable)
- }.toArray
-
- new StructType(fields)
- }
-
- /**
- * Build the RDD to scan rows.
- *
- * @param requiredColumns columns that are being requested by the requesting query
- * @param filters filters that are being applied by the requesting query
- * @return RDD will all the results from Kudu
- */
- override def buildScan(requiredColumns: Array[String], filters: Array[Filter]): RDD[Row] = {
- val predicates = filters.flatMap(filterToPredicate)
- new KuduRDD(kuduMaster, 1024 * 1024 * 20, requiredColumns, predicates,
- table, context, sqlContext.sparkContext)
- }
-
- /**
- * Converts a Spark [[Filter]] to a Kudu [[KuduPredicate]].
- *
- * @param filter the filter to convert
- * @return the converted filter
- */
- private def filterToPredicate(filter : Filter) : Array[KuduPredicate] = {
- filter match {
- case EqualTo(column, value) =>
- Array(comparisonPredicate(column, ComparisonOp.EQUAL, value))
- case GreaterThan(column, value) =>
- Array(comparisonPredicate(column, ComparisonOp.GREATER, value))
- case GreaterThanOrEqual(column, value) =>
- Array(comparisonPredicate(column, ComparisonOp.GREATER_EQUAL, value))
- case LessThan(column, value) =>
- Array(comparisonPredicate(column, ComparisonOp.LESS, value))
- case LessThanOrEqual(column, value) =>
- Array(comparisonPredicate(column, ComparisonOp.LESS_EQUAL, value))
- case And(left, right) => filterToPredicate(left) ++ filterToPredicate(right)
- case _ => Array()
- }
- }
-
- /**
- * Creates a new comparison predicate for the column, comparison operator, and comparison value.
- *
- * @param column the column name
- * @param operator the comparison operator
- * @param value the comparison value
- * @return the comparison predicate
- */
- private def comparisonPredicate(column: String,
- operator: ComparisonOp,
- value: Any): KuduPredicate = {
- val columnSchema = table.getSchema.getColumn(column)
- value match {
- case value: Boolean => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Byte => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Short => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Int => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Long => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Timestamp => KuduPredicate.newComparisonPredicate(columnSchema, operator, timestampToMicros(value))
- case value: Float => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Double => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: String => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Array[Byte] => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- }
- }
-
- /**
- * Inserts data into an existing Kudu table.
- * @param data [[DataFrame]] to be inserted into Kudu
- * @param overwrite If True it will update existing records, but will not perform inserts.
- */
- override def insert(data: DataFrame, overwrite: Boolean): Unit = {
- context.writeRows(data, tableName, overwrite)
- }
-}
-
-private[spark] object KuduRelation {
- /**
- * Converts a Kudu [[Type]] to a Spark SQL [[DataType]].
- *
- * @param t the Kudu type
- * @return the corresponding Spark SQL type
- */
- private def kuduTypeToSparkType(t: Type): DataType = t match {
- case Type.BOOL => BooleanType
- case Type.INT8 => ByteType
- case Type.INT16 => ShortType
- case Type.INT32 => IntegerType
- case Type.INT64 => LongType
- case Type.TIMESTAMP => TimestampType
- case Type.FLOAT => FloatType
- case Type.DOUBLE => DoubleType
- case Type.STRING => StringType
- case Type.BINARY => BinaryType
- }
-
- /**
- * Returns `true` if the filter is able to be pushed down to Kudu.
- *
- * @param filter the filter to test
- */
- private def supportsFilter(filter: Filter): Boolean = filter match {
- case EqualTo(_, _)
- | GreaterThan(_, _)
- | GreaterThanOrEqual(_, _)
- | LessThan(_, _)
- | LessThanOrEqual(_, _) => true
- case And(left, right) => supportsFilter(left) && supportsFilter(right)
- case _ => false
- }
-
- /**
- * Converts a [[Timestamp]] to microseconds since the Unix epoch (1970-01-01T00:00:00Z).
- *
- * @param timestamp the timestamp to convert to microseconds
- * @return the microseconds since the Unix epoch
- */
- def timestampToMicros(timestamp: Timestamp): Long = {
- // Number of whole milliseconds since the Unix epoch, in microseconds.
- val millis = timestamp.getTime * 1000
- // Sub millisecond time since the Unix epoch, in microseconds.
- val micros = (timestamp.getNanos % 1000000) / 1000
- if (micros >= 0) {
- millis + micros
- } else {
- millis + 1000000 + micros
- }
- }
-
- /**
- * Converts a microsecond offset from the Unix epoch (1970-01-01T00:00:00Z) to a [[Timestamp]].
- *
- * @param micros the offset in microseconds since the Unix epoch
- * @return the corresponding timestamp
- */
- def microsToTimestamp(micros: Long): Timestamp = {
- var millis = micros / 1000
- var nanos = (micros % 1000000) * 1000
- if (nanos < 0) {
- millis -= 1
- nanos += 1000000000
- }
-
- val timestamp = new Timestamp(millis)
- timestamp.setNanos(nanos.asInstanceOf[Int])
- timestamp
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduContext.scala b/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduContext.scala
deleted file mode 100644
index 167ee13..0000000
--- a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduContext.scala
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.kududb.spark.kudu
-
-import java.util
-import org.apache.hadoop.util.ShutdownHookManager
-import org.apache.spark.SparkContext
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, Row}
-import org.apache.spark.sql.types.{StructType, DataType, DataTypes}
-import org.kududb.{ColumnSchema, Schema, Type}
-import org.kududb.annotations.InterfaceStability
-import org.kududb.client.SessionConfiguration.FlushMode
-import org.kududb.client._
-
-
-import scala.collection.mutable
-
-/**
- * KuduContext is a serializable container for Kudu client connections.
- *
- * If a Kudu client connection is needed as part of a Spark application, a
- * [[KuduContext]] should used as a broadcast variable in the job in order to
- * share connections among the tasks in a JVM.
- */
-@InterfaceStability.Unstable
-class KuduContext(kuduMaster: String) extends Serializable {
-
- @transient lazy val syncClient = {
- KuduConnection.getSyncClient(kuduMaster)
- }
-
- @transient lazy val asyncClient = {
- KuduConnection.getAsyncClient(kuduMaster)
- }
-
- /**
- * Create an RDD from a Kudu table.
- *
- * @param tableName table to read from
- * @param columnProjection list of columns to read. Not specifying this at all
- * (i.e. setting to null) or setting to the special
- * string '*' means to project all columns.
- * @return a new RDD that maps over the given table for the selected columns
- */
- def kuduRDD(sc: SparkContext,
- tableName: String,
- columnProjection: Seq[String] = Nil): RDD[Row] = {
- new KuduRDD(kuduMaster, 1024*1024*20, columnProjection.toArray, Array(),
- syncClient.openTable(tableName), this, sc)
- }
-
- /**
- * Check if kudu table already exists
- * @param tableName tablename to check
- * @return true if table exists, false if table does not exist
- */
- def tableExists(tableName: String): Boolean = syncClient.tableExists(tableName)
-
- /**
- * Delete kudu table
- * @param tableName tablename to delete
- * @return DeleteTableResponse
- */
- def deleteTable(tableName: String): DeleteTableResponse = syncClient.deleteTable(tableName)
-
- /**
- * Creates a kudu table for the given schema. Partitioning can be specified through options.
- * @param tableName table to create
- * @param schema struct schema of table
- * @param keys primary keys of the table
- * @param options replication and partitioning options for the table
- */
- def createTable(tableName: String,
- schema: StructType,
- keys: Seq[String],
- options: CreateTableOptions): KuduTable = {
- val kuduCols = new util.ArrayList[ColumnSchema]()
- // add the key columns first, in the order specified
- for (key <- keys) {
- val f = schema.fields(schema.fieldIndex(key))
- kuduCols.add(new ColumnSchema.ColumnSchemaBuilder(f.name, kuduType(f.dataType)).key(true).build())
- }
- // now add the non-key columns
- for (f <- schema.fields.filter(field=> !keys.contains(field.name))) {
- kuduCols.add(new ColumnSchema.ColumnSchemaBuilder(f.name, kuduType(f.dataType)).nullable(f.nullable).key(false).build())
- }
-
- syncClient.createTable(tableName, new Schema(kuduCols), options)
- }
-
- /** Map Spark SQL type to Kudu type */
- def kuduType(dt: DataType) : Type = dt match {
- case DataTypes.BinaryType => Type.BINARY
- case DataTypes.BooleanType => Type.BOOL
- case DataTypes.StringType => Type.STRING
- case DataTypes.TimestampType => Type.TIMESTAMP
- case DataTypes.ByteType => Type.INT8
- case DataTypes.ShortType => Type.INT16
- case DataTypes.IntegerType => Type.INT32
- case DataTypes.LongType => Type.INT64
- case DataTypes.FloatType => Type.FLOAT
- case DataTypes.DoubleType => Type.DOUBLE
- case _ => throw new IllegalArgumentException(s"No support for Spark SQL type $dt")
- }
-
- /**
- * Inserts or updates rows in kudu from a [[DataFrame]].
- * @param data `DataFrame` to insert/update
- * @param tableName table to perform insertion on
- * @param overwrite true=update, false=insert
- */
- def writeRows(data: DataFrame, tableName: String, overwrite: Boolean) {
- val schema = data.schema
- data.foreachPartition(iterator => {
- val pendingErrors = writeRows(iterator, schema, tableName, overwrite)
- val errorCount = pendingErrors.getRowErrors.length
- if (errorCount > 0) {
- val errors = pendingErrors.getRowErrors.take(5).map(_.getErrorStatus).mkString
- throw new RuntimeException(
- s"failed to write $errorCount rows from DataFrame to Kudu; sample errors: $errors")
- }
- })
- }
-
- /**
- * Saves partitions of a [[DataFrame]] into Kudu.
- * @param rows rows to insert or update
- * @param tableName table to insert or update on
- */
- def writeRows(rows: Iterator[Row],
- schema: StructType,
- tableName: String,
- performAsUpdate : Boolean = false): RowErrorsAndOverflowStatus = {
- val table: KuduTable = syncClient.openTable(tableName)
- val kuduSchema = table.getSchema
- val indices: Array[(Int, Int)] = schema.fields.zipWithIndex.map({ case (field, sparkIdx) =>
- sparkIdx -> table.getSchema.getColumnIndex(field.name)
- })
- val session: KuduSession = syncClient.newSession
- session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND)
- session.setIgnoreAllDuplicateRows(true)
- try {
- for (row <- rows) {
- val operation = if (performAsUpdate) { table.newUpdate() } else { table.newInsert() }
- for ((sparkIdx, kuduIdx) <- indices) {
- if (row.isNullAt(sparkIdx)) {
- operation.getRow.setNull(kuduIdx)
- } else schema.fields(sparkIdx).dataType match {
- case DataTypes.StringType => operation.getRow.addString(kuduIdx, row.getString(sparkIdx))
- case DataTypes.BinaryType => operation.getRow.addBinary(kuduIdx, row.getAs[Array[Byte]](sparkIdx))
- case DataTypes.BooleanType => operation.getRow.addBoolean(kuduIdx, row.getBoolean(sparkIdx))
- case DataTypes.ByteType => operation.getRow.addByte(kuduIdx, row.getByte(sparkIdx))
- case DataTypes.ShortType => operation.getRow.addShort(kuduIdx, row.getShort(sparkIdx))
- case DataTypes.IntegerType => operation.getRow.addInt(kuduIdx, row.getInt(sparkIdx))
- case DataTypes.LongType => operation.getRow.addLong(kuduIdx, row.getLong(sparkIdx))
- case DataTypes.FloatType => operation.getRow.addFloat(kuduIdx, row.getFloat(sparkIdx))
- case DataTypes.DoubleType => operation.getRow.addDouble(kuduIdx, row.getDouble(sparkIdx))
- case DataTypes.TimestampType => operation.getRow.addLong(kuduIdx, KuduRelation.timestampToMicros(row.getTimestamp(sparkIdx)))
- case t => throw new IllegalArgumentException(s"No support for Spark SQL type $t")
- }
- }
- session.apply(operation)
- }
- } finally {
- session.close()
- }
- session.getPendingErrors
- }
-
-}
-
-private object KuduConnection {
- private val syncCache = new mutable.HashMap[String, KuduClient]()
- private val asyncCache = new mutable.HashMap[String, AsyncKuduClient]()
-
- /**
- * Set to
- * [[org.apache.spark.util.ShutdownHookManager.DEFAULT_SHUTDOWN_PRIORITY]].
- * The client instances are closed through the JVM shutdown hook
- * mechanism in order to make sure that any unflushed writes are cleaned up
- * properly. Spark has no shutdown notifications.
- */
- private val ShutdownHookPriority = 100
-
- def getSyncClient(kuduMaster: String): KuduClient = {
- syncCache.synchronized {
- if (!syncCache.contains(kuduMaster)) {
- val syncClient = new KuduClient.KuduClientBuilder(kuduMaster).build()
- ShutdownHookManager.get().addShutdownHook(new Runnable {
- override def run() = syncClient.close()
- }, ShutdownHookPriority)
- syncCache.put(kuduMaster, syncClient)
- }
- return syncCache(kuduMaster)
- }
- }
-
- def getAsyncClient(kuduMaster: String): AsyncKuduClient = {
- asyncCache.synchronized {
- if (!asyncCache.contains(kuduMaster)) {
- val asyncClient = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMaster).build()
- ShutdownHookManager.get().addShutdownHook(
- new Runnable {
- override def run() = asyncClient.close()
- }, ShutdownHookPriority)
- asyncCache.put(kuduMaster, asyncClient)
- }
- return asyncCache(kuduMaster)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduRDD.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduRDD.scala b/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduRDD.scala
deleted file mode 100644
index 5395d5a..0000000
--- a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/KuduRDD.scala
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.spark.kudu
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.Row
-import org.apache.spark.{Partition, SparkContext, TaskContext}
-import org.kududb.client._
-import org.kududb.{Type, client}
-
-import scala.collection.JavaConverters._
-
-/**
- * A Resilient Distributed Dataset backed by a Kudu table.
- */
-class KuduRDD(val kuduMaster: String,
- @transient batchSize: Integer,
- @transient projectedCols: Array[String],
- @transient predicates: Array[client.KuduPredicate],
- @transient table: KuduTable,
- @transient kc: KuduContext,
- @transient sc: SparkContext) extends RDD[Row](sc, Nil) {
-
- /**
- * The [[KuduContext]] for this `KuduRDD`.
- *
- * The `KuduContext` manages the Kudu client instances for the `KuduRDD`.
- * When the `KuduRDD` is first constructed it uses the context passed in as
- * `kc`. After deserialization, a new `KuduContext` is created as necessary.
- * The `kc` field should not be used, since it will not be rehydrated after
- * serialization.
- */
- @transient private lazy val kuduContext: KuduContext = {
- if (kc != null) kc else new KuduContext(kuduMaster)
- }
-
- override protected def getPartitions: Array[Partition] = {
- val builder = kuduContext.syncClient
- .newScanTokenBuilder(table)
- .batchSizeBytes(batchSize)
- .setProjectedColumnNames(projectedCols.toSeq.asJava)
- .cacheBlocks(true)
-
- for (predicate <- predicates) {
- builder.addPredicate(predicate)
- }
- val tokens = builder.build().asScala
- tokens.zipWithIndex.map {
- case (token, index) =>
- new KuduPartition(index, token.serialize(),
- token.getTablet.getReplicas.asScala.map(_.getRpcHost).toArray)
- }.toArray
- }
-
- override def compute(part: Partition, taskContext: TaskContext): Iterator[Row] = {
- val client: KuduClient = kuduContext.syncClient
- val partition: KuduPartition = part.asInstanceOf[KuduPartition]
- val scanner = KuduScanToken.deserializeIntoScanner(partition.scanToken, client)
- new RowResultIteratorScala(scanner)
- }
-
- override def getPreferredLocations(partition: Partition): Seq[String] = {
- partition.asInstanceOf[KuduPartition].locations
- }
-}
-
-/**
- * A Spark SQL [[Partition]] which wraps a [[KuduScanToken]].
- */
-private[spark] class KuduPartition(val index: Int,
- val scanToken: Array[Byte],
- val locations : Array[String]) extends Partition {}
-
-/**
- * A Spark SQL [[Row]] iterator which wraps a [[KuduScanner]].
- * @param scanner the wrapped scanner
- */
-private[spark] class RowResultIteratorScala(private val scanner: KuduScanner) extends Iterator[Row] {
-
- private var currentIterator: RowResultIterator = null
-
- override def hasNext: Boolean = {
- if ((currentIterator != null && !currentIterator.hasNext && scanner.hasMoreRows) ||
- (scanner.hasMoreRows && currentIterator == null)) {
- currentIterator = scanner.nextRows()
- }
- currentIterator.hasNext
- }
-
- override def next(): Row = new KuduRow(currentIterator.next())
-}
-
-/**
- * A Spark SQL [[Row]] which wraps a Kudu [[RowResult]].
- * @param rowResult the wrapped row result
- */
-private[spark] class KuduRow(private val rowResult: RowResult) extends Row {
- override def length: Int = rowResult.getColumnProjection.getColumnCount
-
- override def get(i: Int): Any = {
- if (rowResult.isNull(i)) null
- else rowResult.getColumnType(i) match {
- case Type.BOOL => rowResult.getBoolean(i)
- case Type.INT8 => rowResult.getByte(i)
- case Type.INT16 => rowResult.getShort(i)
- case Type.INT32 => rowResult.getInt(i)
- case Type.INT64 => rowResult.getLong(i)
- case Type.TIMESTAMP => KuduRelation.microsToTimestamp(rowResult.getLong(i))
- case Type.FLOAT => rowResult.getFloat(i)
- case Type.DOUBLE => rowResult.getDouble(i)
- case Type.STRING => rowResult.getString(i)
- case Type.BINARY => rowResult.getBinary(i)
- }
- }
-
- override def copy(): Row = Row.fromSeq(Range(0, length).map(get))
-
- override def toString(): String = rowResult.toString
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/package.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/package.scala b/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/package.scala
deleted file mode 100755
index 4203e31..0000000
--- a/java/kudu-spark/src/main/scala/org/kududb/spark/kudu/package.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.spark
-
-import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter}
-
-package object kudu {
-
- /**
- * Adds a method, `kudu`, to DataFrameReader that allows you to read Kudu tables using
- * the DataFrameReader.
- */
- implicit class KuduDataFrameReader(reader: DataFrameReader) {
- def kudu: DataFrame = reader.format("org.kududb.spark.kudu").load
- }
-
- /**
- * Adds a method, `kudu`, to DataFrameWriter that allows writes to Kudu using
- * the DataFileWriter
- */
- implicit class KuduDataFrameWriter(writer: DataFrameWriter) {
- def kudu = writer.format("org.kududb.spark.kudu").save
- }
-}
[26/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java
new file mode 100644
index 0000000..f1a9075
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java
@@ -0,0 +1,231 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import com.google.common.net.HostAndPort;
+import com.google.protobuf.ByteString;
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.Common;
+import org.kududb.consensus.Metadata;
+import org.kududb.master.Master;
+import org.kududb.util.NetUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Class grouping the callback and the errback for GetMasterRegistration calls
+ * made in getMasterTableLocationsPB.
+ */
+@InterfaceAudience.Private
+final class GetMasterRegistrationReceived {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GetMasterRegistrationReceived.class);
+
+ private final List<HostAndPort> masterAddrs;
+ private final Deferred<Master.GetTableLocationsResponsePB> responseD;
+ private final int numMasters;
+
+ // Used to avoid calling 'responseD' twice.
+ private final AtomicBoolean responseDCalled = new AtomicBoolean(false);
+
+ // Number of responses we've receives: used to tell whether or not we've received
+ // errors/replies from all of the masters, or if there are any
+ // GetMasterRegistrationRequests still pending.
+ private final AtomicInteger countResponsesReceived = new AtomicInteger(0);
+
+ // Exceptions received so far: kept for debugging purposes.
+ // (see: NoLeaderMasterFoundException#create() for how this is used).
+ private final List<Exception> exceptionsReceived =
+ Collections.synchronizedList(new ArrayList<Exception>());
+
+ /**
+ * Creates an object that holds the state needed to retrieve master table's location.
+ * @param masterAddrs Addresses of all master replicas that we want to retrieve the
+ * registration from.
+ * @param responseD Deferred object that will hold the GetTableLocationsResponsePB object for
+ * the master table.
+ */
+ public GetMasterRegistrationReceived(List<HostAndPort> masterAddrs,
+ Deferred<Master.GetTableLocationsResponsePB> responseD) {
+ this.masterAddrs = masterAddrs;
+ this.responseD = responseD;
+ this.numMasters = masterAddrs.size();
+ }
+
+ /**
+ * Creates a callback for a GetMasterRegistrationRequest that was sent to 'hostAndPort'.
+ * @see GetMasterRegistrationCB
+ * @param hostAndPort Host and part for the RPC we're attaching this to. Host and port must
+ * be valid.
+ * @return The callback object that can be added to the RPC request.
+ */
+ public Callback<Void, GetMasterRegistrationResponse> callbackForNode(HostAndPort hostAndPort) {
+ return new GetMasterRegistrationCB(hostAndPort);
+ }
+
+ /**
+ * Creates an errback for a GetMasterRegistrationRequest that was sent to 'hostAndPort'.
+ * @see GetMasterRegistrationErrCB
+ * @param hostAndPort Host and port for the RPC we're attaching this to. Used for debugging
+ * purposes.
+ * @return The errback object that can be added to the RPC request.
+ */
+ public Callback<Void, Exception> errbackForNode(HostAndPort hostAndPort) {
+ return new GetMasterRegistrationErrCB(hostAndPort);
+ }
+
+ /**
+ * Checks if we've already received a response or an exception from every master that
+ * we've sent a GetMasterRegistrationRequest to. If so -- and no leader has been found
+ * (that is, 'responseD' was never called) -- pass a {@link NoLeaderMasterFoundException}
+ * to responseD.
+ */
+ private void incrementCountAndCheckExhausted() {
+ if (countResponsesReceived.incrementAndGet() == numMasters) {
+ if (responseDCalled.compareAndSet(false, true)) {
+ boolean allUnrecoverable = true;
+ for (Exception ex : exceptionsReceived) {
+ if (!(ex instanceof NonRecoverableException)) {
+ allUnrecoverable = false;
+ break;
+ }
+ }
+ String allHosts = NetUtil.hostsAndPortsToString(masterAddrs);
+ // Doing a negative check because allUnrecoverable stays true if there are no exceptions.
+ if (!allUnrecoverable) {
+ String message = "Master config (" + allHosts + ") has no leader.";
+ Exception ex;
+ if (exceptionsReceived.isEmpty()) {
+ LOG.warn("None of the provided masters (" + allHosts + ") is a leader, will retry.");
+ ex = new NoLeaderMasterFoundException(Status.ServiceUnavailable(message));
+ } else {
+ LOG.warn("Unable to find the leader master (" + allHosts + "), will retry");
+ String joinedMsg = message + ". Exceptions received: " +
+ Joiner.on(",").join(
+ Lists.transform(exceptionsReceived, Functions.toStringFunction()));
+ Status statusServiceUnavailable = Status.ServiceUnavailable(joinedMsg);
+ ex = new NoLeaderMasterFoundException(
+ statusServiceUnavailable,
+ exceptionsReceived.get(exceptionsReceived.size() - 1));
+ }
+ responseD.callback(ex);
+ } else {
+ Status statusConfigurationError = Status.ConfigurationError(
+ "Couldn't find a valid master in (" + allHosts +
+ "), exceptions: " + exceptionsReceived);
+ // This will stop retries.
+ responseD.callback(new NonRecoverableException(statusConfigurationError));
+ }
+ }
+ }
+ }
+
+ /**
+ * Callback for each GetMasterRegistrationRequest sent in getMasterTableLocations() above.
+ * If a request (paired to a specific master) returns a reply that indicates it's a leader,
+ * the callback in 'responseD' is invoked with an initialized GetTableLocationResponsePB
+ * object containing the leader's RPC address.
+ * If the master is not a leader, increment 'countResponsesReceived': if the count equals to
+ * the number of masters, pass {@link NoLeaderMasterFoundException} into
+ * 'responseD' if no one else had called 'responseD' before; otherwise, do nothing.
+ */
+ final class GetMasterRegistrationCB implements Callback<Void, GetMasterRegistrationResponse> {
+ private final HostAndPort hostAndPort;
+
+ public GetMasterRegistrationCB(HostAndPort hostAndPort) {
+ this.hostAndPort = hostAndPort;
+ }
+
+ @Override
+ public Void call(GetMasterRegistrationResponse r) throws Exception {
+ Master.TabletLocationsPB.ReplicaPB.Builder replicaBuilder =
+ Master.TabletLocationsPB.ReplicaPB.newBuilder();
+
+ Master.TSInfoPB.Builder tsInfoBuilder = Master.TSInfoPB.newBuilder();
+ tsInfoBuilder.addRpcAddresses(ProtobufHelper.hostAndPortToPB(hostAndPort));
+ tsInfoBuilder.setPermanentUuid(r.getInstanceId().getPermanentUuid());
+ replicaBuilder.setTsInfo(tsInfoBuilder);
+ if (r.getRole().equals(Metadata.RaftPeerPB.Role.LEADER)) {
+ replicaBuilder.setRole(r.getRole());
+ Master.TabletLocationsPB.Builder locationBuilder = Master.TabletLocationsPB.newBuilder();
+ locationBuilder.setPartition(
+ Common.PartitionPB.newBuilder().setPartitionKeyStart(ByteString.EMPTY)
+ .setPartitionKeyEnd(ByteString.EMPTY));
+ locationBuilder.setTabletId(
+ ByteString.copyFromUtf8(AsyncKuduClient.MASTER_TABLE_NAME_PLACEHOLDER));
+ locationBuilder.addReplicas(replicaBuilder);
+ // No one else has called this before us.
+ if (responseDCalled.compareAndSet(false, true)) {
+ responseD.callback(
+ Master.GetTableLocationsResponsePB.newBuilder().addTabletLocations(
+ locationBuilder.build()).build()
+ );
+ } else {
+ LOG.debug("Callback already invoked, discarding response(" + r.toString() + ") from " +
+ hostAndPort.toString());
+ }
+ } else {
+ incrementCountAndCheckExhausted();
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "get master registration for " + hostAndPort.toString();
+ }
+ }
+
+ /**
+ * Errback for each GetMasterRegistrationRequest sent in getMasterTableLocations() above.
+ * Stores each exception in 'exceptionsReceived'. Increments 'countResponseReceived': if
+ * the count is equal to the number of masters and no one else had called 'responseD' before,
+ * pass a {@link NoLeaderMasterFoundException} into 'responseD'; otherwise, do
+ * nothing.
+ */
+ final class GetMasterRegistrationErrCB implements Callback<Void, Exception> {
+ private final HostAndPort hostAndPort;
+
+ public GetMasterRegistrationErrCB(HostAndPort hostAndPort) {
+ this.hostAndPort = hostAndPort;
+ }
+
+ @Override
+ public Void call(Exception e) throws Exception {
+ LOG.warn("Error receiving a response from: " + hostAndPort, e);
+ exceptionsReceived.add(e);
+ incrementCountAndCheckExhausted();
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "get master registration errback for " + hostAndPort.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java
new file mode 100644
index 0000000..bc3d81e
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import static org.kududb.consensus.Metadata.*;
+import static org.kududb.master.Master.*;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * Package-private RPC that can only go to master.
+ */
+@InterfaceAudience.Private
+public class GetMasterRegistrationRequest extends KuduRpc<GetMasterRegistrationResponse> {
+ private static final String GET_MASTER_REGISTRATION = "GetMasterRegistration";
+
+ public GetMasterRegistrationRequest(KuduTable masterTable) {
+ super(masterTable);
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ final GetMasterRegistrationRequestPB.Builder builder =
+ GetMasterRegistrationRequestPB.newBuilder();
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return GET_MASTER_REGISTRATION;
+ }
+
+ @Override
+ Pair<GetMasterRegistrationResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final GetMasterRegistrationResponsePB.Builder respBuilder =
+ GetMasterRegistrationResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), respBuilder);
+ RaftPeerPB.Role role = RaftPeerPB.Role.FOLLOWER;
+ if (!respBuilder.hasError() || respBuilder.getError().getCode() !=
+ MasterErrorPB.Code.CATALOG_MANAGER_NOT_INITIALIZED) {
+ role = respBuilder.getRole();
+ }
+ GetMasterRegistrationResponse response = new GetMasterRegistrationResponse(
+ deadlineTracker.getElapsedMillis(),
+ tsUUID,
+ role,
+ respBuilder.getRegistration(),
+ respBuilder.getInstanceId());
+ return new Pair<GetMasterRegistrationResponse, Object>(
+ response, respBuilder.hasError() ? respBuilder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java
new file mode 100644
index 0000000..292710c
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.WireProtocol;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.consensus.Metadata;
+import org.kududb.master.Master;
+
+/**
+ * Response for {@link GetMasterRegistrationRequest}.
+ */
+@InterfaceAudience.Private
+public class GetMasterRegistrationResponse extends KuduRpcResponse {
+
+ private final Metadata.RaftPeerPB.Role role;
+ private final WireProtocol.ServerRegistrationPB serverRegistration;
+ private final WireProtocol.NodeInstancePB instanceId;
+
+ /**
+ * Describes a response to a {@link GetMasterRegistrationRequest}, built from
+ * {@link Master.GetMasterRegistrationResponsePB}.
+ *
+ * @param role Master's role in the config.
+ * @param serverRegistration server registration (RPC and HTTP addresses) for this master.
+ * @param instanceId Node instance (permanent uuid and
+ */
+ public GetMasterRegistrationResponse(long elapsedMillis, String tsUUID,
+ Metadata.RaftPeerPB.Role role,
+ WireProtocol.ServerRegistrationPB serverRegistration,
+ WireProtocol.NodeInstancePB instanceId) {
+ super(elapsedMillis, tsUUID);
+ this.role = role;
+ this.serverRegistration = serverRegistration;
+ this.instanceId = instanceId;
+ }
+
+ /**
+ * Returns this master's role in the config.
+ *
+ * @see Metadata.RaftPeerPB.Role
+ * @return Node's role in the cluster, or FOLLOWER if the node is not initialized.
+ */
+ public Metadata.RaftPeerPB.Role getRole() {
+ return role;
+ }
+
+ /**
+ * Returns the server registration (list of RPC and HTTP ports) for this master.
+ *
+ * @return The {@link WireProtocol.ServerRegistrationPB} object for this master.
+ */
+ public WireProtocol.ServerRegistrationPB getServerRegistration() {
+ return serverRegistration;
+ }
+
+ /**
+ * The node instance (initial sequence number and permanent uuid) for this master.
+ *
+ * @return The {@link WireProtocol.NodeInstancePB} object for this master.
+ */
+ public WireProtocol.NodeInstancePB getInstanceId() {
+ return instanceId;
+ }
+
+ @Override
+ public String toString() {
+ return "GetMasterRegistrationResponse{" +
+ "role=" + role +
+ ", serverRegistration=" + serverRegistration +
+ ", instanceId=" + instanceId +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java
new file mode 100644
index 0000000..616b523
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.master.Master;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * Package-private RPC that can only go to a master.
+ */
+@InterfaceAudience.Private
+class GetTableLocationsRequest extends KuduRpc<Master.GetTableLocationsResponsePB> {
+
+ private final byte[] startPartitionKey;
+ private final byte[] endKey;
+ private final String tableId;
+
+ GetTableLocationsRequest(KuduTable table, byte[] startPartitionKey,
+ byte[] endPartitionKey, String tableId) {
+ super(table);
+ if (startPartitionKey != null && endPartitionKey != null
+ && Bytes.memcmp(startPartitionKey, endPartitionKey) > 0) {
+ throw new IllegalArgumentException(
+ "The start partition key must be smaller or equal to the end partition key");
+ }
+ this.startPartitionKey = startPartitionKey;
+ this.endKey = endPartitionKey;
+ this.tableId = tableId;
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return "GetTableLocations";
+ }
+
+ @Override
+ Pair<Master.GetTableLocationsResponsePB, Object> deserialize(
+ final CallResponse callResponse, String tsUUID)
+ throws Exception {
+ Master.GetTableLocationsResponsePB.Builder builder = Master.GetTableLocationsResponsePB
+ .newBuilder();
+ readProtobuf(callResponse.getPBMessage(), builder);
+ Master.GetTableLocationsResponsePB resp = builder.build();
+ return new Pair<Master.GetTableLocationsResponsePB, Object>(
+ resp, builder.hasError() ? builder.getError() : null);
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ final Master.GetTableLocationsRequestPB.Builder builder = Master
+ .GetTableLocationsRequestPB.newBuilder();
+ builder.setTable(Master.TableIdentifierPB.newBuilder().
+ setTableId(ByteString.copyFromUtf8(tableId)));
+ if (startPartitionKey != null) {
+ builder.setPartitionKeyStart(ZeroCopyLiteralByteString.wrap(startPartitionKey));
+ }
+ if (endKey != null) {
+ builder.setPartitionKeyEnd(ZeroCopyLiteralByteString.wrap(endKey));
+ }
+ return toChannelBuffer(header, builder.build());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java
new file mode 100644
index 0000000..bb17816
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import static org.kududb.master.Master.*;
+
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * RPC to fetch a table's schema
+ */
+@InterfaceAudience.Private
+public class GetTableSchemaRequest extends KuduRpc<GetTableSchemaResponse> {
+ static final String GET_TABLE_SCHEMA = "GetTableSchema";
+ private final String name;
+
+
+ GetTableSchemaRequest(KuduTable masterTable, String name) {
+ super(masterTable);
+ this.name = name;
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ final GetTableSchemaRequestPB.Builder builder = GetTableSchemaRequestPB.newBuilder();
+ TableIdentifierPB tableID =
+ TableIdentifierPB.newBuilder().setTableName(name).build();
+ builder.setTable(tableID);
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return GET_TABLE_SCHEMA;
+ }
+
+ @Override
+ Pair<GetTableSchemaResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final GetTableSchemaResponsePB.Builder respBuilder = GetTableSchemaResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), respBuilder);
+ Schema schema = ProtobufHelper.pbToSchema(respBuilder.getSchema());
+ GetTableSchemaResponse response = new GetTableSchemaResponse(
+ deadlineTracker.getElapsedMillis(),
+ tsUUID,
+ schema,
+ respBuilder.getTableId().toStringUtf8(),
+ ProtobufHelper.pbToPartitionSchema(respBuilder.getPartitionSchema(), schema),
+ respBuilder.getCreateTableDone());
+ return new Pair<GetTableSchemaResponse, Object>(
+ response, respBuilder.hasError() ? respBuilder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java
new file mode 100644
index 0000000..72ac68e
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class GetTableSchemaResponse extends KuduRpcResponse {
+
+ private final Schema schema;
+ private final PartitionSchema partitionSchema;
+ private final boolean createTableDone;
+ private final String tableId;
+
+ /**
+ * @param ellapsedMillis Time in milliseconds since RPC creation to now
+ * @param schema the table's schema
+ * @param partitionSchema the table's partition schema
+ */
+ GetTableSchemaResponse(long ellapsedMillis,
+ String tsUUID,
+ Schema schema,
+ String tableId,
+ PartitionSchema partitionSchema,
+ boolean createTableDone) {
+ super(ellapsedMillis, tsUUID);
+ this.schema = schema;
+ this.partitionSchema = partitionSchema;
+ this.createTableDone = createTableDone;
+ this.tableId = tableId;
+ }
+
+ /**
+ * Get the table's schema.
+ * @return Table's schema
+ */
+ public Schema getSchema() {
+ return schema;
+ }
+
+ /**
+ * Get the table's partition schema.
+ * @return the table's partition schema
+ */
+ public PartitionSchema getPartitionSchema() {
+ return partitionSchema;
+ }
+
+ /**
+ * Tells if the original CreateTable call has completed and the tablets are ready.
+ * @return true if the table is created, otherwise false
+ */
+ public boolean isCreateTableDone() {
+ return createTableDone;
+ }
+
+ /**
+ * Get the table's unique identifier.
+ * @return the table's tableId
+ */
+ public String getTableId() {
+ return tableId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java
new file mode 100644
index 0000000..08dda52
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Interface implemented by {@link KuduException}s that can tell you which
+ * RPC failed.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface HasFailedRpcException {
+
+ /**
+ * Returns the RPC that caused this exception.
+ */
+ KuduRpc<?> getFailedRpc();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java
new file mode 100644
index 0000000..45240dd
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java
@@ -0,0 +1,83 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.client;
+
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.Message;
+import org.kududb.annotations.InterfaceAudience;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Helper methods for RPCs.
+ */
+@InterfaceAudience.Private
+public class IPCUtil {
+ /**
+ * Write out header, param, and cell block if there is one.
+ * @param dos
+ * @param header
+ * @param param
+ * @return Total number of bytes written.
+ * @throws java.io.IOException
+ */
+ public static int write(final OutputStream dos, final Message header, final Message param)
+ throws IOException {
+ // Must calculate total size and write that first so other side can read it all in in one
+ // swoop. This is dictated by how the server is currently written. Server needs to change
+ // if we are to be able to write without the length prefixing.
+ int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(header, param);
+ return write(dos, header, param, totalSize);
+ }
+
+ private static int write(final OutputStream dos, final Message header, final Message param,
+ final int totalSize)
+ throws IOException {
+ // I confirmed toBytes does same as say DataOutputStream#writeInt.
+ dos.write(toBytes(totalSize));
+ header.writeDelimitedTo(dos);
+ if (param != null) param.writeDelimitedTo(dos);
+ dos.flush();
+ return totalSize;
+ }
+
+ /**
+ * @return Size on the wire when the two messages are written with writeDelimitedTo
+ */
+ public static int getTotalSizeWhenWrittenDelimited(Message ... messages) {
+ int totalSize = 0;
+ for (Message m: messages) {
+ if (m == null) continue;
+ totalSize += m.getSerializedSize();
+ totalSize += CodedOutputStream.computeRawVarint32Size(m.getSerializedSize());
+ }
+ return totalSize;
+ }
+
+ public static byte[] toBytes(int val) {
+ byte [] b = new byte[4];
+ for(int i = 3; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java
new file mode 100644
index 0000000..67b389f
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Represents a single row insert. Instances of this class should not be reused.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Insert extends Operation {
+
+ Insert(KuduTable table) {
+ super(table);
+ }
+
+ @Override
+ ChangeType getChangeType() {
+ return ChangeType.INSERT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java
new file mode 100644
index 0000000..ca161f5
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import static org.kududb.master.Master.*;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * RPC used to check if an alter is running for the specified table
+ */
+@InterfaceAudience.Private
+class IsAlterTableDoneRequest extends KuduRpc<IsAlterTableDoneResponse> {
+
+ static final String IS_ALTER_TABLE_DONE = "IsAlterTableDone";
+ private final String name;
+
+
+ IsAlterTableDoneRequest(KuduTable masterTable, String name) {
+ super(masterTable);
+ this.name = name;
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ final IsAlterTableDoneRequestPB.Builder builder = IsAlterTableDoneRequestPB.newBuilder();
+ TableIdentifierPB tableID =
+ TableIdentifierPB.newBuilder().setTableName(name).build();
+ builder.setTable(tableID);
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return IS_ALTER_TABLE_DONE;
+ }
+
+ @Override
+ Pair<IsAlterTableDoneResponse, Object> deserialize(final CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final IsAlterTableDoneResponsePB.Builder respBuilder = IsAlterTableDoneResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), respBuilder);
+ IsAlterTableDoneResponse resp = new IsAlterTableDoneResponse(deadlineTracker.getElapsedMillis(),
+ tsUUID, respBuilder.getDone());
+ return new Pair<IsAlterTableDoneResponse, Object>(
+ resp, respBuilder.hasError() ? respBuilder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java
new file mode 100644
index 0000000..356c085
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Response to a isAlterTableDone command to use to know if an alter table is currently running on
+ * the specified table.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class IsAlterTableDoneResponse extends KuduRpcResponse {
+
+ private final boolean done;
+
+ IsAlterTableDoneResponse(long elapsedMillis, String tsUUID, boolean done) {
+ super(elapsedMillis, tsUUID);
+ this.done = done;
+ }
+
+ /**
+ * Tells if the table is done being altered or not.
+ * @return whether the table alter is done
+ */
+ public boolean isDone() {
+ return done;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java
new file mode 100644
index 0000000..8e4679c
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.master.Master;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * Package-private RPC that can only go to a master.
+ */
+@InterfaceAudience.Private
+class IsCreateTableDoneRequest extends KuduRpc<Master.IsCreateTableDoneResponsePB> {
+
+ private final String tableId;
+
+ IsCreateTableDoneRequest(KuduTable table, String tableId) {
+ super(table);
+ this.tableId = tableId;
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return "IsCreateTableDone";
+ }
+
+ @Override
+ Pair<Master.IsCreateTableDoneResponsePB, Object> deserialize(
+ final CallResponse callResponse, String tsUUID) throws Exception {
+ Master.IsCreateTableDoneResponsePB.Builder builder = Master.IsCreateTableDoneResponsePB
+ .newBuilder();
+ readProtobuf(callResponse.getPBMessage(), builder);
+ Master.IsCreateTableDoneResponsePB resp = builder.build();
+ return new Pair<Master.IsCreateTableDoneResponsePB, Object>(
+ resp, builder.hasError() ? builder.getError() : null);
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ final Master.IsCreateTableDoneRequestPB.Builder builder = Master
+ .IsCreateTableDoneRequestPB.newBuilder();
+ builder.setTable(Master.TableIdentifierPB.newBuilder().setTableId(
+ ByteString.copyFromUtf8(tableId)));
+ return toChannelBuffer(header, builder.build());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
new file mode 100644
index 0000000..2fbde58
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.primitives.UnsignedLongs;
+import com.sangupta.murmur.Murmur2;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.client.PartitionSchema.HashBucketSchema;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.List;
+
+/**
+ * Utility class for encoding rows into primary and partition keys.
+ */
+@InterfaceAudience.Private
+class KeyEncoder {
+
+ private final ByteArrayOutputStream buf = new ByteArrayOutputStream();
+
+ /**
+ * Encodes the primary key of the row.
+ *
+ * @param row the row to encode
+ * @return the encoded primary key of the row
+ */
+ public byte[] encodePrimaryKey(final PartialRow row) {
+ buf.reset();
+
+ final Schema schema = row.getSchema();
+ for (int columnIdx = 0; columnIdx < schema.getPrimaryKeyColumnCount(); columnIdx++) {
+ final boolean isLast = columnIdx + 1 == schema.getPrimaryKeyColumnCount();
+ encodeColumn(row, columnIdx, isLast);
+ }
+ return extractByteArray();
+ }
+
+ /**
+ * Encodes the provided row into a partition key according to the partition schema.
+ *
+ * @param row the row to encode
+ * @param partitionSchema the partition schema describing the table's partitioning
+ * @return an encoded partition key
+ */
+ public byte[] encodePartitionKey(PartialRow row, PartitionSchema partitionSchema) {
+ buf.reset();
+ if (!partitionSchema.getHashBucketSchemas().isEmpty()) {
+ ByteBuffer bucketBuf = ByteBuffer.allocate(4 * partitionSchema.getHashBucketSchemas().size());
+ bucketBuf.order(ByteOrder.BIG_ENDIAN);
+
+ for (final HashBucketSchema hashBucketSchema : partitionSchema.getHashBucketSchemas()) {
+ encodeColumns(row, hashBucketSchema.getColumnIds());
+ byte[] encodedColumns = extractByteArray();
+ long hash = Murmur2.hash64(encodedColumns,
+ encodedColumns.length,
+ hashBucketSchema.getSeed());
+ int bucket = (int) UnsignedLongs.remainder(hash, hashBucketSchema.getNumBuckets());
+ bucketBuf.putInt(bucket);
+ }
+
+ assert bucketBuf.arrayOffset() == 0;
+ buf.write(bucketBuf.array(), 0, bucketBuf.position());
+ }
+
+ encodeColumns(row, partitionSchema.getRangeSchema().getColumns());
+ return extractByteArray();
+ }
+
+ /**
+ * Encodes a sequence of columns from the row.
+ * @param row the row containing the columns to encode
+ * @param columnIds the IDs of each column to encode
+ */
+ private void encodeColumns(PartialRow row, List<Integer> columnIds) {
+ for (int i = 0; i < columnIds.size(); i++) {
+ boolean isLast = i + 1 == columnIds.size();
+ encodeColumn(row, row.getSchema().getColumnIndex(columnIds.get(i)), isLast);
+ }
+ }
+
+ /**
+ * Encodes a single column of a row.
+ * @param row the row being encoded
+ * @param columnIdx the column index of the column to encode
+ * @param isLast whether the column is the last component of the key
+ */
+ private void encodeColumn(PartialRow row, int columnIdx, boolean isLast) {
+ final Schema schema = row.getSchema();
+ final ColumnSchema column = schema.getColumnByIndex(columnIdx);
+ if (!row.isSet(columnIdx)) {
+ throw new IllegalStateException(String.format("Primary key column %s is not set",
+ column.getName()));
+ }
+ final Type type = column.getType();
+
+ if (type == Type.STRING || type == Type.BINARY) {
+ addBinaryComponent(row.getVarLengthData().get(columnIdx), isLast);
+ } else {
+ addComponent(row.getRowAlloc(),
+ schema.getColumnOffset(columnIdx),
+ type.getSize(),
+ type);
+ }
+ }
+
+ /**
+ * Encodes a byte buffer into the key.
+ * @param value the value to encode
+ * @param isLast whether the value is the final component in the key
+ */
+ private void addBinaryComponent(ByteBuffer value, boolean isLast) {
+ value.reset();
+
+ // TODO find a way to not have to read byte-by-byte that doesn't require extra copies. This is
+ // especially slow now that users can pass direct byte buffers.
+ while (value.hasRemaining()) {
+ byte currentByte = value.get();
+ buf.write(currentByte);
+ if (!isLast && currentByte == 0x00) {
+ // If we're a middle component of a composite key, we need to add a \x00
+ // at the end in order to separate this component from the next one. However,
+ // if we just did that, we'd have issues where a key that actually has
+ // \x00 in it would compare wrong, so we have to instead add \x00\x00, and
+ // encode \x00 as \x00\x01. -- key_encoder.h
+ buf.write(0x01);
+ }
+ }
+
+ if (!isLast) {
+ buf.write(0x00);
+ buf.write(0x00);
+ }
+ }
+
+ /**
+ * Encodes a value of the given type into the key.
+ * @param value the value to encode
+ * @param offset the offset into the {@code value} buffer that the value begins
+ * @param len the length of the value
+ * @param type the type of the value to encode
+ */
+ private void addComponent(byte[] value, int offset, int len, Type type) {
+ switch (type) {
+ case INT8:
+ case INT16:
+ case INT32:
+ case INT64:
+ case TIMESTAMP:
+ // Picking the first byte because big endian.
+ byte lastByte = value[offset + (len - 1)];
+ lastByte = Bytes.xorLeftMostBit(lastByte);
+ buf.write(lastByte);
+ if (len > 1) {
+ for (int i = len - 2; i >= 0; i--) {
+ buf.write(value[offset + i]);
+ }
+ }
+ break;
+ default:
+ throw new IllegalArgumentException(String.format(
+ "The column type %s is not a valid key component type", type));
+ }
+ }
+
+ /**
+ * Returns the encoded key, and resets the key encoder to be used for another key.
+ * @return the encoded key which has been built through calls to {@link #addComponent}
+ */
+ private byte[] extractByteArray() {
+ byte[] bytes = buf.toByteArray();
+ buf.reset();
+ return bytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
new file mode 100644
index 0000000..cfd4662
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
@@ -0,0 +1,415 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.stumbleupon.async.Deferred;
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+/**
+ * A synchronous and thread-safe client for Kudu.
+ * <p>
+ * This class acts as a wrapper around {@link AsyncKuduClient}. The {@link Deferred} objects are
+ * joined against using the default admin operation timeout
+ * (see {@link org.kududb.client.KuduClient.KuduClientBuilder#defaultAdminOperationTimeoutMs(long)} (long)}).
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduClient implements AutoCloseable {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AsyncKuduClient.class);
+
+ private final AsyncKuduClient asyncClient;
+
+ KuduClient(AsyncKuduClient asyncClient) {
+ this.asyncClient = asyncClient;
+ }
+
+ /**
+ * Create a table on the cluster with the specified name, schema, and table configurations.
+ * @param name the table's name
+ * @param schema the table's schema
+ * @param builder a builder containing the table's configurations
+ * @return an object to communicate with the created table
+ * @throws KuduException if anything went wrong
+ */
+ public KuduTable createTable(String name, Schema schema, CreateTableOptions builder)
+ throws KuduException {
+ Deferred<KuduTable> d = asyncClient.createTable(name, schema, builder);
+ return joinAndHandleException(d);
+ }
+
+ /**
+ * Delete a table on the cluster with the specified name.
+ * @param name the table's name
+ * @return an rpc response object
+ * @throws KuduException if anything went wrong
+ */
+ public DeleteTableResponse deleteTable(String name) throws KuduException {
+ Deferred<DeleteTableResponse> d = asyncClient.deleteTable(name);
+ return joinAndHandleException(d);
+ }
+
+ /**
+ * Alter a table on the cluster as specified by the builder.
+ *
+ * When the method returns it only indicates that the master accepted the alter
+ * command, use {@link KuduClient#isAlterTableDone(String)} to know when the alter finishes.
+ * @param name the table's name, if this is a table rename then the old table name must be passed
+ * @param ato the alter table builder
+ * @return an rpc response object
+ * @throws KuduException if anything went wrong
+ */
+ public AlterTableResponse alterTable(String name, AlterTableOptions ato) throws KuduException {
+ Deferred<AlterTableResponse> d = asyncClient.alterTable(name, ato);
+ return joinAndHandleException(d);
+ }
+
+ /**
+ * Helper method that checks and waits until the completion of an alter command.
+ * It will block until the alter command is done or the timeout is reached.
+ * @param name Table's name, if the table was renamed then that name must be checked against
+ * @return a boolean indicating if the table is done being altered
+ * @throws KuduException for any error returned by sending RPCs to the master
+ */
+ public boolean isAlterTableDone(String name) throws KuduException {
+ long totalSleepTime = 0;
+ while (totalSleepTime < getDefaultAdminOperationTimeoutMs()) {
+ long start = System.currentTimeMillis();
+
+ try {
+ Deferred<IsAlterTableDoneResponse> d = asyncClient.isAlterTableDone(name);
+ IsAlterTableDoneResponse response;
+
+ response = d.join(AsyncKuduClient.SLEEP_TIME);
+ if (response.isDone()) {
+ return true;
+ }
+
+ // Count time that was slept and see if we need to wait a little more.
+ long elapsed = System.currentTimeMillis() - start;
+ // Don't oversleep the deadline.
+ if (totalSleepTime + AsyncKuduClient.SLEEP_TIME > getDefaultAdminOperationTimeoutMs()) {
+ return false;
+ }
+ // elapsed can be bigger if we slept about 500ms
+ if (elapsed <= AsyncKuduClient.SLEEP_TIME) {
+ LOG.debug("Alter not done, sleep " + (AsyncKuduClient.SLEEP_TIME - elapsed) +
+ " and slept " + totalSleepTime);
+ Thread.sleep(AsyncKuduClient.SLEEP_TIME - elapsed);
+ totalSleepTime += AsyncKuduClient.SLEEP_TIME;
+ } else {
+ totalSleepTime += elapsed;
+ }
+ } catch (Exception ex) {
+ throw KuduException.transformException(ex);
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get the list of running tablet servers.
+ * @return a list of tablet servers
+ * @throws KuduException if anything went wrong
+ */
+ public ListTabletServersResponse listTabletServers() throws KuduException {
+ Deferred<ListTabletServersResponse> d = asyncClient.listTabletServers();
+ return joinAndHandleException(d);
+ }
+
+ /**
+ * Get the list of all the tables.
+ * @return a list of all the tables
+ * @throws KuduException if anything went wrong
+ */
+ public ListTablesResponse getTablesList() throws KuduException {
+ return getTablesList(null);
+ }
+
+ /**
+ * Get a list of table names. Passing a null filter returns all the tables. When a filter is
+ * specified, it only returns tables that satisfy a substring match.
+ * @param nameFilter an optional table name filter
+ * @return a deferred that contains the list of table names
+ * @throws KuduException if anything went wrong
+ */
+ public ListTablesResponse getTablesList(String nameFilter) throws KuduException {
+ Deferred<ListTablesResponse> d = asyncClient.getTablesList(nameFilter);
+ return joinAndHandleException(d);
+ }
+
+ /**
+ * Test if a table exists.
+ * @param name a non-null table name
+ * @return true if the table exists, else false
+ * @throws KuduException if anything went wrong
+ */
+ public boolean tableExists(String name) throws KuduException {
+ Deferred<Boolean> d = asyncClient.tableExists(name);
+ return joinAndHandleException(d);
+ }
+
+ /**
+ * Open the table with the given name. If the table was just created, this method will block until
+ * all its tablets have also been created.
+ * @param name table to open
+ * @return a KuduTable if the table exists
+ * @throws KuduException if anything went wrong
+ */
+ public KuduTable openTable(final String name) throws KuduException {
+ Deferred<KuduTable> d = asyncClient.openTable(name);
+ return joinAndHandleException(d);
+ }
+
+ /**
+ * Create a new session for interacting with the cluster.
+ * User is responsible for destroying the session object.
+ * This is a fully local operation (no RPCs or blocking).
+ * @return a synchronous wrapper around KuduSession.
+ */
+ public KuduSession newSession() {
+ AsyncKuduSession session = asyncClient.newSession();
+ return new KuduSession(session);
+ }
+
+ /**
+ * Check if statistics collection is enabled for this client.
+ * @return true if it is enabled, else false
+ */
+ public boolean isStatisticsEnabled() {
+ return asyncClient.isStatisticsEnabled();
+ }
+
+ /**
+ * Get the statistics object of this client.
+ *
+ * @return this client's Statistics object
+ * @throws IllegalStateException thrown if statistics collection has been disabled
+ */
+ public Statistics getStatistics() {
+ return asyncClient.getStatistics();
+ }
+
+ /**
+ * Creates a new {@link KuduScanner.KuduScannerBuilder} for a particular table.
+ * @param table the table you intend to scan.
+ * The string is assumed to use the platform's default charset.
+ * @return a new scanner builder for the table
+ */
+ public KuduScanner.KuduScannerBuilder newScannerBuilder(KuduTable table) {
+ return new KuduScanner.KuduScannerBuilder(asyncClient, table);
+ }
+
+ /**
+ * Creates a new {@link KuduScanToken.KuduScanTokenBuilder} for a particular table.
+ * Used for integrations with compute frameworks.
+ * @param table the table you intend to scan
+ * @return a new scan token builder for the table
+ */
+ public KuduScanToken.KuduScanTokenBuilder newScanTokenBuilder(KuduTable table) {
+ return new KuduScanToken.KuduScanTokenBuilder(asyncClient, table);
+ }
+
+ /**
+ * Analogous to {@link #shutdown()}.
+ * @throws KuduException if an error happens while closing the connections
+ */
+ @Override
+ public void close() throws KuduException {
+ try {
+ asyncClient.close();
+ } catch (Exception e) {
+ KuduException.transformException(e);
+ }
+ }
+
+ /**
+ * Performs a graceful shutdown of this instance.
+ * @throws KuduException if anything went wrong
+ */
+ public void shutdown() throws KuduException {
+ Deferred<ArrayList<Void>> d = asyncClient.shutdown();
+ joinAndHandleException(d);
+ }
+
+ /**
+ * Get the timeout used for operations on sessions and scanners.
+ * @return a timeout in milliseconds
+ */
+ public long getDefaultOperationTimeoutMs() {
+ return asyncClient.getDefaultOperationTimeoutMs();
+ }
+
+ /**
+ * Get the timeout used for admin operations.
+ * @return a timeout in milliseconds
+ */
+ public long getDefaultAdminOperationTimeoutMs() {
+ return asyncClient.getDefaultAdminOperationTimeoutMs();
+ }
+
+ // Helper method to handle joining and transforming the Exception we receive.
+ private <R> R joinAndHandleException(Deferred<R> deferred) throws KuduException {
+ try {
+ return deferred.join(getDefaultAdminOperationTimeoutMs());
+ } catch (Exception e) {
+ throw KuduException.transformException(e);
+ }
+ }
+
+ /**
+ * Builder class to use in order to connect to Kudu.
+ * All the parameters beyond those in the constructors are optional.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public final static class KuduClientBuilder {
+ private AsyncKuduClient.AsyncKuduClientBuilder clientBuilder;
+
+ /**
+ * Creates a new builder for a client that will connect to the specified masters.
+ * @param masterAddresses comma-separated list of "host:port" pairs of the masters
+ */
+ public KuduClientBuilder(String masterAddresses) {
+ clientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses);
+ }
+
+ /**
+ * Creates a new builder for a client that will connect to the specified masters.
+ *
+ * <p>Here are some examples of recognized formats:
+ * <ul>
+ * <li>example.com
+ * <li>example.com:80
+ * <li>192.0.2.1
+ * <li>192.0.2.1:80
+ * <li>[2001:db8::1]
+ * <li>[2001:db8::1]:80
+ * <li>2001:db8::1
+ * </ul>
+ *
+ * @param masterAddresses list of master addresses
+ */
+ public KuduClientBuilder(List<String> masterAddresses) {
+ clientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses);
+ }
+
+ /**
+ * Sets the default timeout used for administrative operations (e.g. createTable, deleteTable,
+ * etc).
+ * Optional.
+ * If not provided, defaults to 30s.
+ * A value of 0 disables the timeout.
+ * @param timeoutMs a timeout in milliseconds
+ * @return this builder
+ */
+ public KuduClientBuilder defaultAdminOperationTimeoutMs(long timeoutMs) {
+ clientBuilder.defaultAdminOperationTimeoutMs(timeoutMs);
+ return this;
+ }
+
+ /**
+ * Sets the default timeout used for user operations (using sessions and scanners).
+ * Optional.
+ * If not provided, defaults to 30s.
+ * A value of 0 disables the timeout.
+ * @param timeoutMs a timeout in milliseconds
+ * @return this builder
+ */
+ public KuduClientBuilder defaultOperationTimeoutMs(long timeoutMs) {
+ clientBuilder.defaultOperationTimeoutMs(timeoutMs);
+ return this;
+ }
+
+ /**
+ * Sets the default timeout to use when waiting on data from a socket.
+ * Optional.
+ * If not provided, defaults to 10s.
+ * A value of 0 disables the timeout.
+ * @param timeoutMs a timeout in milliseconds
+ * @return this builder
+ */
+ public KuduClientBuilder defaultSocketReadTimeoutMs(long timeoutMs) {
+ clientBuilder.defaultSocketReadTimeoutMs(timeoutMs);
+ return this;
+ }
+
+ /**
+ * Disable this client's collection of statistics.
+ * Statistics are enabled by default.
+ * @return this builder
+ */
+ public KuduClientBuilder disableStatistics() {
+ clientBuilder.disableStatistics();
+ return this;
+ }
+
+ /**
+ * Set the executors which will be used for the embedded Netty boss and workers.
+ * Optional.
+ * If not provided, uses a simple cached threadpool. If either argument is null,
+ * then such a thread pool will be used in place of that argument.
+ * Note: executor's max thread number must be greater or equal to corresponding
+ * worker count, or netty cannot start enough threads, and client will get stuck.
+ * If not sure, please just use CachedThreadPool.
+ */
+ public KuduClientBuilder nioExecutors(Executor bossExecutor, Executor workerExecutor) {
+ clientBuilder.nioExecutors(bossExecutor, workerExecutor);
+ return this;
+ }
+
+ /**
+ * Set the maximum number of boss threads.
+ * Optional.
+ * If not provided, 1 is used.
+ */
+ public KuduClientBuilder bossCount(int bossCount) {
+ clientBuilder.bossCount(bossCount);
+ return this;
+ }
+
+ /**
+ * Set the maximum number of worker threads.
+ * Optional.
+ * If not provided, (2 * the number of available processors) is used.
+ */
+ public KuduClientBuilder workerCount(int workerCount) {
+ clientBuilder.workerCount(workerCount);
+ return this;
+ }
+
+ /**
+ * Creates a new client that connects to the masters.
+ * Doesn't block and won't throw an exception if the masters don't exist.
+ * @return a new asynchronous Kudu client
+ */
+ public KuduClient build() {
+ AsyncKuduClient client = clientBuilder.build();
+ return new KuduClient(client);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java
new file mode 100644
index 0000000..4bd2eaf
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduException.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import com.stumbleupon.async.DeferredGroupException;
+import com.stumbleupon.async.TimeoutException;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.io.IOException;
+
+/**
+ * The parent class of all exceptions sent by the Kudu client. This is the only exception you will
+ * see if you're using the non-async API, such as {@link KuduSession} instead of
+ * {@link AsyncKuduSession}.
+ *
+ * Each instance of this class has a {@link Status} which gives more information about the error.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+public abstract class KuduException extends IOException {
+
+ private final Status status;
+
+ /**
+ * Constructor.
+ * @param status object containing the reason for the exception
+ * trace.
+ */
+ KuduException(Status status) {
+ super(status.getMessage());
+ this.status = status;
+ }
+
+ /**
+ * Constructor.
+ * @param status object containing the reason for the exception
+ * @param cause The exception that caused this one to be thrown.
+ */
+ KuduException(Status status, Throwable cause) {
+ super(status.getMessage(), cause);
+ this.status = status;
+ }
+
+ /**
+ * Get the Status object for this exception.
+ * @return a status object indicating the reason for the exception
+ */
+ public Status getStatus() {
+ return status;
+ }
+
+ /**
+ * Inspects the given exception and transforms it into a KuduException.
+ * @param e generic exception we want to transform
+ * @return a KuduException that's easier to handle
+ */
+ static KuduException transformException(Exception e) {
+ if (e instanceof KuduException) {
+ return (KuduException) e;
+ } else if (e instanceof DeferredGroupException) {
+ // TODO anything we can do to improve on that kind of exception?
+ } else if (e instanceof TimeoutException) {
+ Status statusTimeout = Status.TimedOut(e.getMessage());
+ return new NonRecoverableException(statusTimeout, e);
+ } else if (e instanceof InterruptedException) {
+ // Need to reset the interrupt flag since we caught it but aren't handling it.
+ Thread.currentThread().interrupt();
+ Status statusAborted = Status.Aborted(e.getMessage());
+ return new NonRecoverableException(statusAborted, e);
+ }
+ Status status = Status.IOError(e.getMessage());
+ return new NonRecoverableException(status, e);
+ }
+}
[30/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITImportCsv.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITImportCsv.java b/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITImportCsv.java
deleted file mode 100644
index d7b8352..0000000
--- a/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITImportCsv.java
+++ /dev/null
@@ -1,123 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce.tools;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.HadoopTestingUtility;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.client.CreateTableOptions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import com.google.common.collect.ImmutableList;
-
-public class ITImportCsv extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- ITImportCsv.class.getName() + "-" + System.currentTimeMillis();
-
- private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
-
- private static Schema schema;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
-
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32)
- .key(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column2_d", Type.DOUBLE)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column3_s", Type.STRING)
- .nullable(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column4_b", Type.BOOL)
- .build());
- schema = new Schema(columns);
-
- createTable(TABLE_NAME, schema,
- new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key")));
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- try {
- BaseKuduTest.tearDownAfterClass();
- } finally {
- HADOOP_UTIL.cleanup();
- }
- }
-
- @Test
- public void test() throws Exception {
- Configuration conf = new Configuration();
- String testHome =
- HADOOP_UTIL.setupAndGetTestDir(ITImportCsv.class.getName(), conf).getAbsolutePath();
-
- // Create a 2 lines input file
- File data = new File(testHome, "data.csv");
- writeCsvFile(data);
-
- StringBuilder sb = new StringBuilder();
- for (ColumnSchema col : schema.getColumns()) {
- sb.append(col.getName());
- sb.append(",");
- }
- sb.deleteCharAt(sb.length() - 1);
- String[] args = new String[] {
- "-D" + CommandLineParser.MASTER_ADDRESSES_KEY + "=" + getMasterAddresses(),
- sb.toString(), TABLE_NAME, data.toString()};
-
- GenericOptionsParser parser = new GenericOptionsParser(conf, args);
- Job job = ImportCsv.createSubmittableJob(parser.getConfiguration(), parser.getRemainingArgs());
- assertTrue("Test job did not end properly", job.waitForCompletion(true));
-
- assertEquals(1, job.getCounters().findCounter(ImportCsv.Counters.BAD_LINES).getValue());
-
- assertEquals(3, countRowsInScan(
- client.newScannerBuilder(openTable(TABLE_NAME)).build()));
- // TODO: should verify the actual returned rows, not just the count!
- }
-
- private void writeCsvFile(File data) throws IOException {
- FileOutputStream fos = new FileOutputStream(data);
- fos.write("1\t3\t2.3\tsome string\ttrue\n".getBytes());
- fos.write("2\t5\t4.5\tsome more\tfalse\n".getBytes());
- fos.write("3\t7\twait this is not a double\tbad row\ttrue\n".getBytes());
- fos.write("4\t9\t10\ttrailing separator isn't bad mkay?\ttrue\t\n".getBytes());
- fos.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITIntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITIntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITIntegrationTestBigLinkedList.java
deleted file mode 100644
index 311c5ee..0000000
--- a/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITIntegrationTestBigLinkedList.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.kududb.mapreduce.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Test;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.HadoopTestingUtility;
-
-public class ITIntegrationTestBigLinkedList extends BaseKuduTest {
-
- private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- try {
- BaseKuduTest.tearDownAfterClass();
- } finally {
- HADOOP_UTIL.cleanup();
- }
- }
-
- @Test
- public void test() throws Exception {
- Configuration conf = new Configuration();
- HADOOP_UTIL.setupAndGetTestDir(
- ITIntegrationTestBigLinkedList.class.getName(),conf).getAbsolutePath();
-
- String[] args = new String[] {
- "-D" + CommandLineParser.MASTER_ADDRESSES_KEY + "=" + getMasterAddresses(),
- "Loop",
- "2", // Two iterations
- "1", // 1 mapper
- "2500", // 2.5k rows to insert
- "1", // 1 tablet
- "/tmp/itbll", // output dir
- "1", // 1 reduce
- "100", // create 100 columns
- "25", // wrap them together after 25 rows
- "0"
- };
- int ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
- Assert.assertEquals(0, ret);
-
- args[2] = "1"; // Just one iteration this time
- args[10] = "5000"; // 2 * 2500 from previous run
- ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
- Assert.assertEquals(0, ret);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITRowCounter.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITRowCounter.java b/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITRowCounter.java
deleted file mode 100644
index 52984bb..0000000
--- a/java/kudu-client-tools/src/test/java/org/kududb/mapreduce/tools/ITRowCounter.java
+++ /dev/null
@@ -1,68 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce.tools;
-
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.HadoopTestingUtility;
-import org.kududb.client.BaseKuduTest;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class ITRowCounter extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- ITRowCounter.class.getName() + "-" + System.currentTimeMillis();
-
- private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- try {
- BaseKuduTest.tearDownAfterClass();
- } finally {
- HADOOP_UTIL.cleanup();
- }
- }
-
- @Test
- public void test() throws Exception {
- Configuration conf = new Configuration();
- HADOOP_UTIL.setupAndGetTestDir(ITRowCounter.class.getName(), conf).getAbsolutePath();
-
- createFourTabletsTableWithNineRows(TABLE_NAME);
-
- String[] args = new String[] {
- "-D" + CommandLineParser.MASTER_ADDRESSES_KEY + "=" + getMasterAddresses(), TABLE_NAME};
- GenericOptionsParser parser = new GenericOptionsParser(conf, args);
- Job job = RowCounter.createSubmittableJob(parser.getConfiguration(), parser.getRemainingArgs());
- assertTrue("Job did not end properly", job.waitForCompletion(true));
-
- assertEquals(9, job.getCounters().findCounter(RowCounter.Counters.ROWS).getValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
new file mode 100644
index 0000000..c3c617d
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
@@ -0,0 +1,301 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb;
+
+import org.kududb.Common.CompressionType;
+import org.kududb.Common.EncodingType;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Represents a Kudu Table column. Use {@link ColumnSchema.ColumnSchemaBuilder} in order to
+ * create columns.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ColumnSchema {
+
+ private final String name;
+ private final Type type;
+ private final boolean key;
+ private final boolean nullable;
+ private final Object defaultValue;
+ private final int desiredBlockSize;
+ private final Encoding encoding;
+ private final CompressionAlgorithm compressionAlgorithm;
+
+ /**
+ * Specifies the encoding of data for a column on disk.
+ * Not all encodings are available for all data types.
+ * Refer to the Kudu documentation for more information on each encoding.
+ */
+ public enum Encoding {
+ UNKNOWN(EncodingType.UNKNOWN_ENCODING),
+ AUTO_ENCODING(EncodingType.AUTO_ENCODING),
+ PLAIN_ENCODING(EncodingType.PLAIN_ENCODING),
+ PREFIX_ENCODING(EncodingType.PREFIX_ENCODING),
+ GROUP_VARINT(EncodingType.GROUP_VARINT),
+ RLE(EncodingType.RLE),
+ DICT_ENCODING(EncodingType.DICT_ENCODING),
+ BIT_SHUFFLE(EncodingType.BIT_SHUFFLE);
+
+ final EncodingType internalPbType;
+
+ Encoding(EncodingType internalPbType) {
+ this.internalPbType = internalPbType;
+ }
+
+ @InterfaceAudience.Private
+ public EncodingType getInternalPbType() {
+ return internalPbType;
+ }
+ };
+
+ /**
+ * Specifies the compression algorithm of data for a column on disk.
+ */
+ public enum CompressionAlgorithm {
+ UNKNOWN(CompressionType.UNKNOWN_COMPRESSION),
+ DEFAULT_COMPRESSION(CompressionType.DEFAULT_COMPRESSION),
+ NO_COMPRESSION(CompressionType.NO_COMPRESSION),
+ SNAPPY(CompressionType.SNAPPY),
+ LZ4(CompressionType.LZ4),
+ ZLIB(CompressionType.ZLIB);
+
+ final CompressionType internalPbType;
+
+ CompressionAlgorithm(CompressionType internalPbType) {
+ this.internalPbType = internalPbType;
+ }
+
+ @InterfaceAudience.Private
+ public CompressionType getInternalPbType() {
+ return internalPbType;
+ }
+ };
+
+ private ColumnSchema(String name, Type type, boolean key, boolean nullable,
+ Object defaultValue, int desiredBlockSize, Encoding encoding,
+ CompressionAlgorithm compressionAlgorithm) {
+ this.name = name;
+ this.type = type;
+ this.key = key;
+ this.nullable = nullable;
+ this.defaultValue = defaultValue;
+ this.desiredBlockSize = desiredBlockSize;
+ this.encoding = encoding;
+ this.compressionAlgorithm = compressionAlgorithm;
+ }
+
+ /**
+ * Get the column's Type
+ * @return the type
+ */
+ public Type getType() {
+ return type;
+ }
+
+ /**
+ * Get the column's name
+ * @return A string representation of the name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Answers if the column part of the key
+ * @return true if the column is part of the key, else false
+ */
+ public boolean isKey() {
+ return key;
+ }
+
+ /**
+ * Answers if the column can be set to null
+ * @return true if it can be set to null, else false
+ */
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ /**
+ * The Java object representation of the default value that's read
+ * @return the default read value
+ */
+ public Object getDefaultValue() {
+ return defaultValue;
+ }
+
+ /**
+ * Gets the desired block size for this column.
+ * If no block size has been explicitly specified for this column,
+ * returns 0 to indicate that the server-side default will be used.
+ *
+ * @return the block size, in bytes, or 0 if none has been configured.
+ */
+ public int getDesiredBlockSize() {
+ return desiredBlockSize;
+ }
+
+ /**
+ * Return the encoding of this column, or null if it is not known.
+ */
+ public Encoding getEncoding() {
+ return encoding;
+ }
+
+ /**
+ * Return the compression algorithm of this column, or null if it is not known.
+ */
+ public CompressionAlgorithm getCompressionAlgorithm() {
+ return compressionAlgorithm;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ColumnSchema that = (ColumnSchema) o;
+
+ if (key != that.key) return false;
+ if (!name.equals(that.name)) return false;
+ if (!type.equals(that.type)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name.hashCode();
+ result = 31 * result + type.hashCode();
+ result = 31 * result + (key ? 1 : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "Column name: " + name + ", type: " + type.getName();
+ }
+
+ /**
+ * Builder for ColumnSchema.
+ */
+ public static class ColumnSchemaBuilder {
+ private final String name;
+ private final Type type;
+ private boolean key = false;
+ private boolean nullable = false;
+ private Object defaultValue = null;
+ private int blockSize = 0;
+ private Encoding encoding = null;
+ private CompressionAlgorithm compressionAlgorithm = null;
+
+ /**
+ * Constructor for the required parameters.
+ * @param name column's name
+ * @param type column's type
+ */
+ public ColumnSchemaBuilder(String name, Type type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ /**
+ * Sets if the column is part of the row key. False by default.
+ * @param key a boolean that indicates if the column is part of the key
+ * @return this instance
+ */
+ public ColumnSchemaBuilder key(boolean key) {
+ this.key = key;
+ return this;
+ }
+
+ /**
+ * Marks the column as allowing null values. False by default.
+ * @param nullable a boolean that indicates if the column allows null values
+ * @return this instance
+ */
+ public ColumnSchemaBuilder nullable(boolean nullable) {
+ this.nullable = nullable;
+ return this;
+ }
+
+ /**
+ * Sets the default value that will be read from the column. Null by default.
+ * @param defaultValue a Java object representation of the default value that's read
+ * @return this instance
+ */
+ public ColumnSchemaBuilder defaultValue(Object defaultValue) {
+ this.defaultValue = defaultValue;
+ return this;
+ }
+
+ /**
+ * Set the desired block size for this column.
+ *
+ * This is the number of bytes of user data packed per block on disk, and
+ * represents the unit of IO when reading this column. Larger values
+ * may improve scan performance, particularly on spinning media. Smaller
+ * values may improve random access performance, particularly for workloads
+ * that have high cache hit rates or operate on fast storage such as SSD.
+ *
+ * Note that the block size specified here corresponds to uncompressed data.
+ * The actual size of the unit read from disk may be smaller if
+ * compression is enabled.
+ *
+ * It's recommended that this not be set any lower than 4096 (4KB) or higher
+ * than 1048576 (1MB).
+ * @param blockSize the desired block size, in bytes
+ * @return this instance
+ * <!-- TODO(KUDU-1107): move the above info to docs -->
+ */
+ public ColumnSchemaBuilder desiredBlockSize(int blockSize) {
+ this.blockSize = blockSize;
+ return this;
+ }
+
+ /**
+ * Set the block encoding for this column. See the documentation for the list
+ * of valid options.
+ */
+ public ColumnSchemaBuilder encoding(Encoding encoding) {
+ this.encoding = encoding;
+ return this;
+ }
+
+ /**
+ * Set the compression algorithm for this column. See the documentation for the list
+ * of valid options.
+ */
+ public ColumnSchemaBuilder compressionAlgorithm(CompressionAlgorithm compressionAlgorithm) {
+ this.compressionAlgorithm = compressionAlgorithm;
+ return this;
+ }
+
+ /**
+ * Builds a {@link ColumnSchema} using the passed parameters.
+ * @return a new {@link ColumnSchema}
+ */
+ public ColumnSchema build() {
+ return new ColumnSchema(name, type,
+ key, nullable, defaultValue,
+ blockSize, encoding, compressionAlgorithm);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/Schema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/Schema.java b/java/kudu-client/src/main/java/org/apache/kudu/Schema.java
new file mode 100644
index 0000000..17e7798
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/Schema.java
@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb;
+
+import com.google.common.collect.ImmutableList;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.Bytes;
+import org.kududb.client.PartialRow;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents table's schema which is essentially a list of columns.
+ * This class offers a few utility methods for querying it.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Schema {
+
+ /**
+ * Mapping of column index to column.
+ */
+ private final List<ColumnSchema> columnsByIndex;
+
+ /**
+ * The primary key columns.
+ */
+ private final List<ColumnSchema> primaryKeyColumns = new ArrayList<>();
+
+ /**
+ * Mapping of column name to index.
+ */
+ private final Map<String, Integer> columnsByName;
+
+ /**
+ * Mapping of column ID to index, or null if the schema does not have assigned column IDs.
+ */
+ private final Map<Integer, Integer> columnsById;
+
+ /**
+ * Mapping of column index to backing byte array offset.
+ */
+ private final int[] columnOffsets;
+
+ private final int varLengthColumnCount;
+ private final int rowSize;
+ private final boolean hasNullableColumns;
+
+ /**
+ * Constructs a schema using the specified columns and does some internal accounting
+ * @param columns the columns in index order
+ * @throws IllegalArgumentException If the key columns aren't specified first
+ *
+ * See {@code ColumnPBsToSchema()} in {@code src/kudu/common/wire_protocol.cc}
+ */
+ public Schema(List<ColumnSchema> columns) {
+ this(columns, null);
+ }
+
+ /**
+ * Constructs a schema using the specified columns and IDs.
+ *
+ * This is not a stable API, prefer using {@link Schema#Schema(List)} to create a new schema.
+ *
+ * @param columns the columns in index order
+ * @param columnIds the column ids of the provided columns, or null
+ * @throws IllegalArgumentException If the primary key columns aren't specified first
+ * @throws IllegalArgumentException If the column ids length does not match the columns length
+ *
+ * See {@code ColumnPBsToSchema()} in {@code src/kudu/common/wire_protocol.cc}
+ */
+ public Schema(List<ColumnSchema> columns, List<Integer> columnIds) {
+ boolean hasColumnIds = columnIds != null;
+ if (hasColumnIds && columns.size() != columnIds.size()) {
+ throw new IllegalArgumentException(
+ "Schema must be constructed with all column IDs, or none.");
+ }
+
+ this.columnsByIndex = ImmutableList.copyOf(columns);
+ int varLenCnt = 0;
+ this.columnOffsets = new int[columns.size()];
+ this.columnsByName = new HashMap<>(columns.size());
+ this.columnsById = hasColumnIds ? new HashMap<Integer, Integer>(columnIds.size()) : null;
+ int offset = 0;
+ boolean hasNulls = false;
+ // pre-compute a few counts and offsets
+ for (int index = 0; index < columns.size(); index++) {
+ final ColumnSchema column = columns.get(index);
+ if (column.isKey()) {
+ primaryKeyColumns.add(column);
+ }
+
+ hasNulls |= column.isNullable();
+ columnOffsets[index] = offset;
+ offset += column.getType().getSize();
+ if (this.columnsByName.put(column.getName(), index) != null) {
+ throw new IllegalArgumentException(
+ String.format("Column names must be unique: %s", columns));
+ }
+ if (column.getType() == Type.STRING || column.getType() == Type.BINARY) {
+ varLenCnt++;
+ }
+
+ if (hasColumnIds) {
+ if (this.columnsById.put(columnIds.get(index), index) != null) {
+ throw new IllegalArgumentException(
+ String.format("Column IDs must be unique: %s", columnIds));
+ }
+ }
+ }
+
+ this.hasNullableColumns = hasNulls;
+ this.varLengthColumnCount = varLenCnt;
+ this.rowSize = getRowSize(this.columnsByIndex);
+ }
+
+ /**
+ * Get the list of columns used to create this schema
+ * @return list of columns
+ */
+ public List<ColumnSchema> getColumns() {
+ return this.columnsByIndex;
+ }
+
+ /**
+ * Get the count of columns with variable length (BINARY/STRING) in
+ * this schema.
+ * @return strings count
+ */
+ public int getVarLengthColumnCount() {
+ return this.varLengthColumnCount;
+ }
+
+ /**
+ * Get the size a row built using this schema would be
+ * @return size in bytes
+ */
+ public int getRowSize() {
+ return this.rowSize;
+ }
+
+ /**
+ * Get the index at which this column can be found in the backing byte array
+ * @param idx column's index
+ * @return column's offset
+ */
+ public int getColumnOffset(int idx) {
+ return this.columnOffsets[idx];
+ }
+
+ /**
+ * Get the index for the provided column name.
+ * @param columnName column to search for
+ * @return an index in the schema
+ */
+ public int getColumnIndex(String columnName) {
+ Integer index = this.columnsByName.get(columnName);
+ if (index == null) {
+ throw new IllegalArgumentException(
+ String.format("Unknown column: %s", columnName));
+ }
+ return index;
+ }
+
+ /**
+ * Get the column at the specified index in the original list
+ * @param idx column's index
+ * @return the column
+ */
+ public ColumnSchema getColumnByIndex(int idx) {
+ return this.columnsByIndex.get(idx);
+ }
+
+ /**
+ * Get the column index of the column with the provided ID.
+ * This method is not part of the stable API.
+ * @param columnId the column id of the column
+ * @return the column index of the column.
+ */
+ public int getColumnIndex(int columnId) {
+ if (!hasColumnIds()) throw new IllegalStateException("Schema does not have Column IDs");
+ Integer index = this.columnsById.get(columnId);
+ if (index == null) throw new IllegalArgumentException(
+ String.format("Unknown column id: %s", columnId));
+ return index;
+ }
+
+ /**
+ * Get the column associated with the specified name
+ * @param columnName column's name
+ * @return the column
+ */
+ public ColumnSchema getColumn(String columnName) {
+ return columnsByIndex.get(getColumnIndex(columnName));
+ }
+
+ /**
+ * Get the count of columns in this schema
+ * @return count of columns
+ */
+ public int getColumnCount() {
+ return this.columnsByIndex.size();
+ }
+
+ /**
+ * Get the count of columns that are part of the primary key.
+ * @return count of primary key columns.
+ */
+ public int getPrimaryKeyColumnCount() {
+ return this.primaryKeyColumns.size();
+ }
+
+ /**
+ * Get the primary key columns.
+ * @return the primary key columns.
+ */
+ public List<ColumnSchema> getPrimaryKeyColumns() {
+ return primaryKeyColumns;
+ }
+
+ /**
+ * Get a schema that only contains the columns which are part of the key
+ * @return new schema with only the keys
+ */
+ public Schema getRowKeyProjection() {
+ return new Schema(primaryKeyColumns);
+ }
+
+ /**
+ * Tells if there's at least one nullable column
+ * @return true if at least one column is nullable, else false.
+ */
+ public boolean hasNullableColumns() {
+ return this.hasNullableColumns;
+ }
+
+ /**
+ * Tells whether this schema includes IDs for columns. A schema created by a client as part of
+ * table creation will not include IDs, but schemas for open tables will include IDs.
+ * This method is not part of the stable API.
+ *
+ * @return whether this schema includes column IDs.
+ */
+ public boolean hasColumnIds() {
+ return columnsById != null;
+ }
+
+ /**
+ * Gives the size in bytes for a single row given the specified schema
+ * @param columns the row's columns
+ * @return row size in bytes
+ */
+ private static int getRowSize(List<ColumnSchema> columns) {
+ int totalSize = 0;
+ boolean hasNullables = false;
+ for (ColumnSchema column : columns) {
+ totalSize += column.getType().getSize();
+ hasNullables |= column.isNullable();
+ }
+ if (hasNullables) {
+ totalSize += Bytes.getBitSetSize(columns.size());
+ }
+ return totalSize;
+ }
+
+ /**
+ * Creates a new partial row for the schema.
+ * @return a new partial row
+ */
+ public PartialRow newPartialRow() {
+ return new PartialRow(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/Type.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/Type.java b/java/kudu-client/src/main/java/org/apache/kudu/Type.java
new file mode 100644
index 0000000..fd23835
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/Type.java
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import com.google.common.primitives.Shorts;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import static org.kududb.Common.DataType;
+
+/**
+ * Describes all the types available to build table schemas.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum Type {
+
+ INT8 (DataType.INT8, "int8"),
+ INT16 (DataType.INT16, "int16"),
+ INT32 (DataType.INT32, "int32"),
+ INT64 (DataType.INT64, "int64"),
+ BINARY (DataType.BINARY, "binary"),
+ STRING (DataType.STRING, "string"),
+ BOOL (DataType.BOOL, "bool"),
+ FLOAT (DataType.FLOAT, "float"),
+ DOUBLE (DataType.DOUBLE, "double"),
+ TIMESTAMP (DataType.TIMESTAMP, "timestamp");
+
+
+ private final DataType dataType;
+ private final String name;
+ private final int size;
+
+ /**
+ * Private constructor used to pre-create the types
+ * @param dataType DataType from the common's pb
+ * @param name string representation of the type
+ */
+ private Type(DataType dataType, String name) {
+ this.dataType = dataType;
+ this.name = name;
+ this.size = getTypeSize(this.dataType);
+ }
+
+ /**
+ * Get the data type from the common's pb
+ * @return A DataType
+ */
+ public DataType getDataType() {
+ return this.dataType;
+ }
+
+ /**
+ * Get the string representation of this type
+ * @return The type's name
+ */
+ public String getName() {
+ return this.name;
+ }
+
+ /**
+ * The size of this type on the wire
+ * @return A size
+ */
+ public int getSize() {
+ return this.size;
+ }
+
+ @Override
+ public String toString() {
+ return "Type: " + this.name + ", size: " + this.size;
+ }
+
+ /**
+ * Gives the size in bytes for a given DataType, as per the pb specification
+ * @param type pb type
+ * @return size in bytes
+ */
+ static int getTypeSize(DataType type) {
+ switch (type) {
+ case STRING:
+ case BINARY: return 8 + 8; // offset then string length
+ case BOOL:
+ case INT8: return 1;
+ case INT16: return Shorts.BYTES;
+ case INT32:
+ case FLOAT: return Ints.BYTES;
+ case INT64:
+ case DOUBLE:
+ case TIMESTAMP: return Longs.BYTES;
+ default: throw new IllegalArgumentException("The provided data type doesn't map" +
+ " to know any known one.");
+ }
+ }
+
+ /**
+ * Convert the pb DataType to a Type
+ * @param type DataType to convert
+ * @return a matching Type
+ */
+ public static Type getTypeForDataType(DataType type) {
+ switch (type) {
+ case STRING: return STRING;
+ case BINARY: return BINARY;
+ case BOOL: return BOOL;
+ case INT8: return INT8;
+ case INT16: return INT16;
+ case INT32: return INT32;
+ case INT64: return INT64;
+ case TIMESTAMP: return TIMESTAMP;
+ case FLOAT: return FLOAT;
+ case DOUBLE: return DOUBLE;
+ default:
+ throw new IllegalArgumentException("The provided data type doesn't map" +
+ " to know any known one: " + type.getDescriptorForType().getFullName());
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java
new file mode 100644
index 0000000..b1b0712
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java
@@ -0,0 +1,338 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import org.kududb.Common;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.tserver.Tserver;
+import org.kududb.util.HybridTimeUtil;
+
+/**
+ * Abstract class to extend in order to create builders for scanners.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class AbstractKuduScannerBuilder
+ <S extends AbstractKuduScannerBuilder<? super S, T>, T> {
+ final AsyncKuduClient client;
+ final KuduTable table;
+
+ /** Map of column name to predicate */
+ final Map<String, KuduPredicate> predicates = new HashMap<>();
+
+ AsyncKuduScanner.ReadMode readMode = AsyncKuduScanner.ReadMode.READ_LATEST;
+ Common.OrderMode orderMode = Common.OrderMode.UNORDERED;
+ int batchSizeBytes = 1024*1024;
+ long limit = Long.MAX_VALUE;
+ boolean prefetching = false;
+ boolean cacheBlocks = true;
+ long htTimestamp = AsyncKuduClient.NO_TIMESTAMP;
+ byte[] lowerBoundPrimaryKey = AsyncKuduClient.EMPTY_ARRAY;
+ byte[] upperBoundPrimaryKey = AsyncKuduClient.EMPTY_ARRAY;
+ byte[] lowerBoundPartitionKey = AsyncKuduClient.EMPTY_ARRAY;
+ byte[] upperBoundPartitionKey = AsyncKuduClient.EMPTY_ARRAY;
+ List<String> projectedColumnNames = null;
+ List<Integer> projectedColumnIndexes = null;
+ long scanRequestTimeout;
+
+ AbstractKuduScannerBuilder(AsyncKuduClient client, KuduTable table) {
+ this.client = client;
+ this.table = table;
+ this.scanRequestTimeout = client.getDefaultOperationTimeoutMs();
+ }
+
+ /**
+ * Sets the read mode, the default is to read the latest values.
+ * @param readMode a read mode for the scanner
+ * @return this instance
+ */
+ public S readMode(AsyncKuduScanner.ReadMode readMode) {
+ this.readMode = readMode;
+ return (S) this;
+ }
+
+ /**
+ * Return scan results in primary key sorted order.
+ *
+ * If the table is hash partitioned, the scan must have an equality predicate
+ * on all hashed columns.
+ *
+ * Package private until proper hash partitioning equality predicate checks
+ * are in place.
+ *
+ * Disabled by default.
+ * @return this instance
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ S sortResultsByPrimaryKey() {
+ orderMode = Common.OrderMode.ORDERED;
+ readMode = AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT;
+ return (S) this;
+ }
+
+ /**
+ * Adds a predicate for a column.
+ * @param predicate predicate for a column to add
+ * @return this instance
+ * @deprecated use {@link #addPredicate(KuduPredicate)}
+ */
+ @Deprecated
+ public S addColumnRangePredicate(ColumnRangePredicate predicate) {
+ return addPredicate(predicate.toKuduPredicate());
+ }
+
+ /**
+ * Adds a list of predicates in their raw format,
+ * as given by {@link ColumnRangePredicate#toByteArray(List)}.
+ * @param predicateBytes predicates to add
+ * @return this instance
+ * @throws IllegalArgumentException thrown when the passed bytes aren't valid
+ * @deprecated use {@link #addPredicate}
+ */
+ @Deprecated
+ public S addColumnRangePredicatesRaw(byte[] predicateBytes) {
+ for (Tserver.ColumnRangePredicatePB pb : ColumnRangePredicate.fromByteArray(predicateBytes)) {
+ addPredicate(ColumnRangePredicate.fromPb(pb).toKuduPredicate());
+ }
+ return (S) this;
+ }
+
+ /**
+ * Adds a predicate to the scan.
+ * @param predicate predicate to add
+ * @return this instance
+ */
+ public S addPredicate(KuduPredicate predicate) {
+ String columnName = predicate.getColumn().getName();
+ KuduPredicate existing = predicates.get(columnName);
+ if (existing == null) {
+ predicates.put(columnName, predicate);
+ } else {
+ predicates.put(columnName, existing.merge(predicate));
+ }
+ return (S) this;
+ }
+
+ /**
+ * Set which columns will be read by the Scanner.
+ * Calling this method after {@link #setProjectedColumnIndexes(List)} will reset the projected
+ * columns to those specified in {@code columnNames}.
+ * @param columnNames the names of columns to read, or 'null' to read all columns
+ * (the default)
+ */
+ public S setProjectedColumnNames(List<String> columnNames) {
+ projectedColumnIndexes = null;
+ if (columnNames != null) {
+ projectedColumnNames = ImmutableList.copyOf(columnNames);
+ } else {
+ projectedColumnNames = null;
+ }
+ return (S) this;
+ }
+
+ /**
+ * Set which columns will be read by the Scanner.
+ * Calling this method after {@link #setProjectedColumnNames(List)} will reset the projected
+ * columns to those specified in {@code columnIndexes}.
+ * @param columnIndexes the indexes of columns to read, or 'null' to read all columns
+ * (the default)
+ */
+ public S setProjectedColumnIndexes(List<Integer> columnIndexes) {
+ projectedColumnNames = null;
+ if (columnIndexes != null) {
+ projectedColumnIndexes = ImmutableList.copyOf(columnIndexes);
+ } else {
+ projectedColumnIndexes = null;
+ }
+ return (S) this;
+ }
+
+ /**
+ * Sets the maximum number of bytes returned by the scanner, on each batch. The default is 1MB.
+ * <p>
+ * Kudu may actually return more than this many bytes because it will not
+ * truncate a rowResult in the middle.
+ * @param batchSizeBytes a strictly positive number of bytes
+ * @return this instance
+ */
+ public S batchSizeBytes(int batchSizeBytes) {
+ this.batchSizeBytes = batchSizeBytes;
+ return (S) this;
+ }
+
+ /**
+ * Sets a limit on the number of rows that will be returned by the scanner. There's no limit
+ * by default.
+ * @param limit a positive long
+ * @return this instance
+ */
+ public S limit(long limit) {
+ this.limit = limit;
+ return (S) this;
+ }
+
+ /**
+ * Enables prefetching of rows for the scanner, i.e. whether to send a request for more data
+ * to the server immediately after we receive a response (instead of waiting for the user
+ * to call {@code nextRows()}). Disabled by default.
+ * NOTE: This is risky until KUDU-1260 is resolved.
+ * @param prefetching a boolean that indicates if the scanner should prefetch rows
+ * @return this instance
+ */
+ public S prefetching(boolean prefetching) {
+ this.prefetching = prefetching;
+ return (S) this;
+ }
+
+ /**
+ * Sets the block caching policy for the scanner. If true, scanned data blocks will be cached
+ * in memory and made available for future scans. Enabled by default.
+ * @param cacheBlocks a boolean that indicates if data blocks should be cached or not
+ * @return this instance
+ */
+ public S cacheBlocks(boolean cacheBlocks) {
+ this.cacheBlocks = cacheBlocks;
+ return (S) this;
+ }
+
+ /**
+ * Sets a previously encoded HT timestamp as a snapshot timestamp, for tests. None is used by
+ * default.
+ * Requires that the ReadMode is READ_AT_SNAPSHOT.
+ * @param htTimestamp a long representing a HybridClock-encoded timestamp
+ * @return this instance
+ * @throws IllegalArgumentException on build(), if the timestamp is less than 0 or if the
+ * read mode was not set to READ_AT_SNAPSHOT
+ */
+ @InterfaceAudience.Private
+ public S snapshotTimestampRaw(long htTimestamp) {
+ this.htTimestamp = htTimestamp;
+ return (S) this;
+ }
+
+ /**
+ * Sets the timestamp the scan must be executed at, in microseconds since the Unix epoch. None is
+ * used by default.
+ * Requires that the ReadMode is READ_AT_SNAPSHOT.
+ * @param timestamp a long representing an instant in microseconds since the unix epoch.
+ * @return this instance
+ * @throws IllegalArgumentException on build(), if the timestamp is less than 0 or if the
+ * read mode was not set to READ_AT_SNAPSHOT
+ */
+ public S snapshotTimestampMicros(long timestamp) {
+ this.htTimestamp = HybridTimeUtil.physicalAndLogicalToHTTimestamp(timestamp, 0);
+ return (S) this;
+ }
+
+ /**
+ * Sets how long each scan request to a server can last.
+ * Defaults to {@link KuduClient#getDefaultOperationTimeoutMs()}.
+ * @param scanRequestTimeout a long representing time in milliseconds
+ * @return this instance
+ */
+ public S scanRequestTimeout(long scanRequestTimeout) {
+ this.scanRequestTimeout = scanRequestTimeout;
+ return (S) this;
+ }
+
+ /**
+ * Add a lower bound (inclusive) primary key for the scan.
+ * If any bound is already added, this bound is intersected with that one.
+ * @param partialRow a partial row with specified key columns
+ * @return this instance
+ */
+ public S lowerBound(PartialRow partialRow) {
+ return lowerBoundRaw(partialRow.encodePrimaryKey());
+ }
+
+ /**
+ * Like lowerBoundPrimaryKey() but the encoded primary key is an opaque byte array obtained elsewhere.
+ * @param startPrimaryKey bytes containing an encoded start key
+ * @return this instance
+ * @deprecated use {@link #lowerBound(PartialRow)}
+ */
+ @Deprecated
+ public S lowerBoundRaw(byte[] startPrimaryKey) {
+ if (lowerBoundPrimaryKey == AsyncKuduClient.EMPTY_ARRAY ||
+ Bytes.memcmp(startPrimaryKey, lowerBoundPrimaryKey) > 0) {
+ this.lowerBoundPrimaryKey = startPrimaryKey;
+ }
+ return (S) this;
+ }
+
+ /**
+ * Add an upper bound (exclusive) primary key for the scan.
+ * If any bound is already added, this bound is intersected with that one.
+ * @param partialRow a partial row with specified key columns
+ * @return this instance
+ */
+ public S exclusiveUpperBound(PartialRow partialRow) {
+ return exclusiveUpperBoundRaw(partialRow.encodePrimaryKey());
+ }
+
+ /**
+ * Like exclusiveUpperBound() but the encoded primary key is an opaque byte array obtained elsewhere.
+ * @param endPrimaryKey bytes containing an encoded end key
+ * @return this instance
+ * @deprecated use {@link #exclusiveUpperBound(PartialRow)}
+ */
+ @Deprecated
+ public S exclusiveUpperBoundRaw(byte[] endPrimaryKey) {
+ if (upperBoundPrimaryKey == AsyncKuduClient.EMPTY_ARRAY ||
+ Bytes.memcmp(endPrimaryKey, upperBoundPrimaryKey) < 0) {
+ this.upperBoundPrimaryKey = endPrimaryKey;
+ }
+ return (S) this;
+ }
+
+ /**
+ * Set an encoded (inclusive) start partition key for the scan.
+ *
+ * @param partitionKey the encoded partition key
+ * @return this instance
+ */
+ @InterfaceAudience.LimitedPrivate("Impala")
+ public S lowerBoundPartitionKeyRaw(byte[] partitionKey) {
+ if (Bytes.memcmp(partitionKey, lowerBoundPartitionKey) > 0) {
+ this.lowerBoundPartitionKey = partitionKey;
+ }
+ return (S) this;
+ }
+
+ /**
+ * Set an encoded (exclusive) end partition key for the scan.
+ *
+ * @param partitionKey the encoded partition key
+ * @return this instance
+ */
+ @InterfaceAudience.LimitedPrivate("Impala")
+ public S exclusiveUpperBoundPartitionKeyRaw(byte[] partitionKey) {
+ if (upperBoundPartitionKey.length == 0 || Bytes.memcmp(partitionKey, upperBoundPartitionKey) < 0) {
+ this.upperBoundPartitionKey = partitionKey;
+ }
+ return (S) this;
+ }
+
+ public abstract T build();
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
new file mode 100644
index 0000000..ecffb5e
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.ColumnSchema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import static org.kududb.master.Master.AlterTableRequestPB;
+
+/**
+ * This builder must be used to alter a table. At least one change must be specified.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class AlterTableOptions {
+
+ AlterTableRequestPB.Builder pb = AlterTableRequestPB.newBuilder();
+
+ /**
+ * Change a table's name.
+ * @param newName new table's name, must be used to check progress
+ * @return this instance
+ */
+ public AlterTableOptions renameTable(String newName) {
+ pb.setNewTableName(newName);
+ return this;
+ }
+
+ /**
+ * Add a new column that's not nullable.
+ * @param name name of the new column
+ * @param type type of the new column
+ * @param defaultVal default value used for the currently existing rows
+ * @return this instance
+ */
+ public AlterTableOptions addColumn(String name, Type type, Object defaultVal) {
+ if (defaultVal == null) {
+ throw new IllegalArgumentException("A new column must have a default value, " +
+ "use addNullableColumn() to add a NULLABLE column");
+ }
+ AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
+ step.setType(AlterTableRequestPB.StepType.ADD_COLUMN);
+ step.setAddColumn(AlterTableRequestPB.AddColumn.newBuilder().setSchema(ProtobufHelper
+ .columnToPb(new ColumnSchema.ColumnSchemaBuilder(name, type)
+ .defaultValue(defaultVal)
+ .build())));
+ return this;
+ }
+
+ /**
+ * Add a new column that's nullable, thus has no default value.
+ * @param name name of the new column
+ * @param type type of the new column
+ * @return this instance
+ */
+ public AlterTableOptions addNullableColumn(String name, Type type) {
+ AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
+ step.setType(AlterTableRequestPB.StepType.ADD_COLUMN);
+ step.setAddColumn(AlterTableRequestPB.AddColumn.newBuilder().setSchema(ProtobufHelper
+ .columnToPb(new ColumnSchema.ColumnSchemaBuilder(name, type)
+ .nullable(true)
+ .build())));
+ return this;
+ }
+
+ /**
+ * Drop a column.
+ * @param name name of the column
+ * @return this instance
+ */
+ public AlterTableOptions dropColumn(String name) {
+ AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
+ step.setType(AlterTableRequestPB.StepType.DROP_COLUMN);
+ step.setDropColumn(AlterTableRequestPB.DropColumn.newBuilder().setName(name));
+ return this;
+ }
+
+ /**
+ * Change the name of a column.
+ * @param oldName old column's name, must exist
+ * @param newName new name to use
+ * @return this instance
+ */
+ public AlterTableOptions renameColumn(String oldName, String newName) {
+ AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
+ step.setType(AlterTableRequestPB.StepType.RENAME_COLUMN);
+ step.setRenameColumn(AlterTableRequestPB.RenameColumn.newBuilder().setOldName(oldName)
+ .setNewName(newName));
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
new file mode 100644
index 0000000..751290c
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import static org.kududb.master.Master.*;
+
+/**
+ * RPC used to alter a table. When it returns it doesn't mean that the table is altered,
+ * a success just means that the master accepted it.
+ */
+@InterfaceAudience.Private
+class AlterTableRequest extends KuduRpc<AlterTableResponse> {
+
+ static final String ALTER_TABLE = "AlterTable";
+ private final String name;
+ private final AlterTableRequestPB.Builder builder;
+
+ AlterTableRequest(KuduTable masterTable, String name, AlterTableOptions ato) {
+ super(masterTable);
+ this.name = name;
+ this.builder = ato.pb;
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ TableIdentifierPB tableID =
+ TableIdentifierPB.newBuilder().setTableName(name).build();
+ this.builder.setTable(tableID);
+ return toChannelBuffer(header, this.builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return ALTER_TABLE;
+ }
+
+ @Override
+ Pair<AlterTableResponse, Object> deserialize(final CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final AlterTableResponsePB.Builder respBuilder = AlterTableResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), respBuilder);
+ AlterTableResponse response = new AlterTableResponse(deadlineTracker.getElapsedMillis(),
+ tsUUID);
+ return new Pair<AlterTableResponse, Object>(
+ response, respBuilder.hasError() ? respBuilder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java
new file mode 100644
index 0000000..7d1d581
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class AlterTableResponse extends KuduRpcResponse {
+
+ /**
+ * @param ellapsedMillis Time in milliseconds since RPC creation to now.
+ */
+ AlterTableResponse(long ellapsedMillis, String tsUUID) {
+ super(ellapsedMillis, tsUUID);
+ }
+}
\ No newline at end of file
[33/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
[java-client] repackage to org.apache.kudu (Part 1)
Move all org.kududb directories to org.apache.kudu
Change-Id: I0b2956974a5869c597b9620d9ee403afa7fed8aa
Reviewed-on: http://gerrit.cloudera.org:8080/3736
Tested-by: Kudu Jenkins
Reviewed-by: Jean-Daniel Cryans <jd...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/5c305689
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/5c305689
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/5c305689
Branch: refs/heads/master
Commit: 5c305689b301140c8e656af16175e9b3c841cd5c
Parents: 88a1cf0
Author: Dan Burkert <da...@cloudera.com>
Authored: Sat Jul 23 11:10:16 2016 -0700
Committer: Jean-Daniel Cryans <jd...@apache.org>
Committed: Mon Jul 25 17:13:53 2016 +0000
----------------------------------------------------------------------
.../kudu/annotations/InterfaceAudience.java | 74 +
.../kudu/annotations/InterfaceStability.java | 66 +
.../ExcludePrivateAnnotationsJDiffDoclet.java | 61 +
...ExcludePrivateAnnotationsStandardDoclet.java | 60 +
.../IncludePublicAnnotationsStandardDoclet.java | 65 +
.../annotations/tools/RootDocProcessor.java | 248 ++
.../annotations/tools/StabilityOptions.java | 71 +
.../kudu/annotations/tools/package-info.java | 22 +
.../kududb/annotations/InterfaceAudience.java | 74 -
.../kududb/annotations/InterfaceStability.java | 66 -
.../ExcludePrivateAnnotationsJDiffDoclet.java | 61 -
...ExcludePrivateAnnotationsStandardDoclet.java | 60 -
.../IncludePublicAnnotationsStandardDoclet.java | 65 -
.../annotations/tools/RootDocProcessor.java | 248 --
.../annotations/tools/StabilityOptions.java | 71 -
.../kududb/annotations/tools/package-info.java | 22 -
.../apache/kudu/mapreduce/tools/CsvParser.java | 162 ++
.../apache/kudu/mapreduce/tools/ImportCsv.java | 116 +
.../kudu/mapreduce/tools/ImportCsvMapper.java | 143 +
.../tools/IntegrationTestBigLinkedList.java | 1662 ++++++++++++
.../apache/kudu/mapreduce/tools/RowCounter.java | 127 +
.../org/kududb/mapreduce/tools/CsvParser.java | 162 --
.../org/kududb/mapreduce/tools/ImportCsv.java | 116 -
.../kududb/mapreduce/tools/ImportCsvMapper.java | 143 -
.../tools/IntegrationTestBigLinkedList.java | 1662 ------------
.../org/kududb/mapreduce/tools/RowCounter.java | 127 -
.../kudu/mapreduce/tools/ITImportCsv.java | 123 +
.../tools/ITIntegrationTestBigLinkedList.java | 69 +
.../kudu/mapreduce/tools/ITRowCounter.java | 68 +
.../org/kududb/mapreduce/tools/ITImportCsv.java | 123 -
.../tools/ITIntegrationTestBigLinkedList.java | 69 -
.../kududb/mapreduce/tools/ITRowCounter.java | 68 -
.../main/java/org/apache/kudu/ColumnSchema.java | 301 +++
.../src/main/java/org/apache/kudu/Schema.java | 291 +++
.../src/main/java/org/apache/kudu/Type.java | 136 +
.../kudu/client/AbstractKuduScannerBuilder.java | 338 +++
.../apache/kudu/client/AlterTableOptions.java | 107 +
.../apache/kudu/client/AlterTableRequest.java | 70 +
.../apache/kudu/client/AlterTableResponse.java | 32 +
.../org/apache/kudu/client/AsyncKuduClient.java | 2437 ++++++++++++++++++
.../apache/kudu/client/AsyncKuduScanner.java | 894 +++++++
.../apache/kudu/client/AsyncKuduSession.java | 856 ++++++
.../main/java/org/apache/kudu/client/Batch.java | 199 ++
.../org/apache/kudu/client/BatchResponse.java | 105 +
.../main/java/org/apache/kudu/client/Bytes.java | 1094 ++++++++
.../org/apache/kudu/client/CallResponse.java | 162 ++
.../kudu/client/ColumnRangePredicate.java | 387 +++
.../apache/kudu/client/CreateTableOptions.java | 172 ++
.../apache/kudu/client/CreateTableRequest.java | 83 +
.../apache/kudu/client/CreateTableResponse.java | 31 +
.../org/apache/kudu/client/DeadlineTracker.java | 157 ++
.../java/org/apache/kudu/client/Delete.java | 41 +
.../apache/kudu/client/DeleteTableRequest.java | 68 +
.../apache/kudu/client/DeleteTableResponse.java | 32 +
.../org/apache/kudu/client/ErrorCollector.java | 83 +
.../kudu/client/ExternalConsistencyMode.java | 42 +
.../client/GetMasterRegistrationReceived.java | 231 ++
.../client/GetMasterRegistrationRequest.java | 74 +
.../client/GetMasterRegistrationResponse.java | 88 +
.../kudu/client/GetTableLocationsRequest.java | 84 +
.../kudu/client/GetTableSchemaRequest.java | 75 +
.../kudu/client/GetTableSchemaResponse.java | 79 +
.../kudu/client/HasFailedRpcException.java | 44 +
.../java/org/apache/kudu/client/IPCUtil.java | 83 +
.../java/org/apache/kudu/client/Insert.java | 37 +
.../kudu/client/IsAlterTableDoneRequest.java | 69 +
.../kudu/client/IsAlterTableDoneResponse.java | 44 +
.../kudu/client/IsCreateTableDoneRequest.java | 66 +
.../java/org/apache/kudu/client/KeyEncoder.java | 193 ++
.../java/org/apache/kudu/client/KuduClient.java | 415 +++
.../org/apache/kudu/client/KuduException.java | 99 +
.../org/apache/kudu/client/KuduPredicate.java | 683 +++++
.../java/org/apache/kudu/client/KuduRpc.java | 363 +++
.../org/apache/kudu/client/KuduRpcResponse.java | 56 +
.../org/apache/kudu/client/KuduScanToken.java | 315 +++
.../org/apache/kudu/client/KuduScanner.java | 149 ++
.../org/apache/kudu/client/KuduSession.java | 190 ++
.../java/org/apache/kudu/client/KuduTable.java | 203 ++
.../apache/kudu/client/ListTablesRequest.java | 73 +
.../apache/kudu/client/ListTablesResponse.java | 42 +
.../kudu/client/ListTabletServersRequest.java | 67 +
.../kudu/client/ListTabletServersResponse.java | 58 +
.../apache/kudu/client/ListTabletsRequest.java | 70 +
.../apache/kudu/client/ListTabletsResponse.java | 40 +
.../org/apache/kudu/client/LocatedTablet.java | 132 +
.../client/NoLeaderMasterFoundException.java | 37 +
.../kudu/client/NonCoveredRangeCache.java | 104 +
.../kudu/client/NonCoveredRangeException.java | 51 +
.../kudu/client/NonRecoverableException.java | 53 +
.../java/org/apache/kudu/client/Operation.java | 345 +++
.../apache/kudu/client/OperationResponse.java | 111 +
.../java/org/apache/kudu/client/PartialRow.java | 626 +++++
.../java/org/apache/kudu/client/Partition.java | 182 ++
.../org/apache/kudu/client/PartitionSchema.java | 142 +
.../kudu/client/PleaseThrottleException.java | 105 +
.../org/apache/kudu/client/ProtobufHelper.java | 253 ++
.../kudu/client/RecoverableException.java | 56 +
.../org/apache/kudu/client/RequestTracker.java | 76 +
.../java/org/apache/kudu/client/RowError.java | 116 +
.../kudu/client/RowErrorsAndOverflowStatus.java | 51 +
.../java/org/apache/kudu/client/RowResult.java | 570 ++++
.../apache/kudu/client/RowResultIterator.java | 132 +
.../org/apache/kudu/client/SecureRpcHelper.java | 285 ++
.../kudu/client/SessionConfiguration.java | 158 ++
.../java/org/apache/kudu/client/Statistics.java | 258 ++
.../java/org/apache/kudu/client/Status.java | 373 +++
.../org/apache/kudu/client/TabletClient.java | 879 +++++++
.../java/org/apache/kudu/client/Update.java | 37 +
.../java/org/apache/kudu/client/Upsert.java | 37 +
.../java/org/apache/kudu/util/AsyncUtil.java | 73 +
.../org/apache/kudu/util/HybridTimeUtil.java | 70 +
.../main/java/org/apache/kudu/util/NetUtil.java | 78 +
.../main/java/org/apache/kudu/util/Pair.java | 57 +
.../main/java/org/apache/kudu/util/Slice.java | 702 +++++
.../main/java/org/apache/kudu/util/Slices.java | 259 ++
.../src/main/java/org/kududb/ColumnSchema.java | 301 ---
.../src/main/java/org/kududb/Schema.java | 291 ---
.../src/main/java/org/kududb/Type.java | 136 -
.../client/AbstractKuduScannerBuilder.java | 338 ---
.../org/kududb/client/AlterTableOptions.java | 107 -
.../org/kududb/client/AlterTableRequest.java | 70 -
.../org/kududb/client/AlterTableResponse.java | 32 -
.../java/org/kududb/client/AsyncKuduClient.java | 2437 ------------------
.../org/kududb/client/AsyncKuduScanner.java | 894 -------
.../org/kududb/client/AsyncKuduSession.java | 856 ------
.../src/main/java/org/kududb/client/Batch.java | 199 --
.../java/org/kududb/client/BatchResponse.java | 105 -
.../src/main/java/org/kududb/client/Bytes.java | 1094 --------
.../java/org/kududb/client/CallResponse.java | 162 --
.../org/kududb/client/ColumnRangePredicate.java | 387 ---
.../org/kududb/client/CreateTableOptions.java | 172 --
.../org/kududb/client/CreateTableRequest.java | 83 -
.../org/kududb/client/CreateTableResponse.java | 31 -
.../java/org/kududb/client/DeadlineTracker.java | 157 --
.../src/main/java/org/kududb/client/Delete.java | 41 -
.../org/kududb/client/DeleteTableRequest.java | 68 -
.../org/kududb/client/DeleteTableResponse.java | 32 -
.../java/org/kududb/client/ErrorCollector.java | 83 -
.../kududb/client/ExternalConsistencyMode.java | 42 -
.../client/GetMasterRegistrationReceived.java | 231 --
.../client/GetMasterRegistrationRequest.java | 74 -
.../client/GetMasterRegistrationResponse.java | 88 -
.../kududb/client/GetTableLocationsRequest.java | 84 -
.../kududb/client/GetTableSchemaRequest.java | 75 -
.../kududb/client/GetTableSchemaResponse.java | 79 -
.../kududb/client/HasFailedRpcException.java | 44 -
.../main/java/org/kududb/client/IPCUtil.java | 83 -
.../src/main/java/org/kududb/client/Insert.java | 37 -
.../kududb/client/IsAlterTableDoneRequest.java | 69 -
.../kududb/client/IsAlterTableDoneResponse.java | 44 -
.../kududb/client/IsCreateTableDoneRequest.java | 66 -
.../main/java/org/kududb/client/KeyEncoder.java | 193 --
.../main/java/org/kududb/client/KuduClient.java | 415 ---
.../java/org/kududb/client/KuduException.java | 99 -
.../java/org/kududb/client/KuduPredicate.java | 683 -----
.../main/java/org/kududb/client/KuduRpc.java | 363 ---
.../java/org/kududb/client/KuduRpcResponse.java | 56 -
.../java/org/kududb/client/KuduScanToken.java | 315 ---
.../java/org/kududb/client/KuduScanner.java | 149 --
.../java/org/kududb/client/KuduSession.java | 190 --
.../main/java/org/kududb/client/KuduTable.java | 203 --
.../org/kududb/client/ListTablesRequest.java | 73 -
.../org/kududb/client/ListTablesResponse.java | 42 -
.../kududb/client/ListTabletServersRequest.java | 67 -
.../client/ListTabletServersResponse.java | 58 -
.../org/kududb/client/ListTabletsRequest.java | 70 -
.../org/kududb/client/ListTabletsResponse.java | 40 -
.../java/org/kududb/client/LocatedTablet.java | 132 -
.../client/NoLeaderMasterFoundException.java | 37 -
.../org/kududb/client/NonCoveredRangeCache.java | 104 -
.../kududb/client/NonCoveredRangeException.java | 51 -
.../kududb/client/NonRecoverableException.java | 53 -
.../main/java/org/kududb/client/Operation.java | 345 ---
.../org/kududb/client/OperationResponse.java | 111 -
.../main/java/org/kududb/client/PartialRow.java | 626 -----
.../main/java/org/kududb/client/Partition.java | 182 --
.../java/org/kududb/client/PartitionSchema.java | 142 -
.../kududb/client/PleaseThrottleException.java | 105 -
.../java/org/kududb/client/ProtobufHelper.java | 253 --
.../org/kududb/client/RecoverableException.java | 56 -
.../java/org/kududb/client/RequestTracker.java | 76 -
.../main/java/org/kududb/client/RowError.java | 116 -
.../client/RowErrorsAndOverflowStatus.java | 51 -
.../main/java/org/kududb/client/RowResult.java | 570 ----
.../org/kududb/client/RowResultIterator.java | 132 -
.../java/org/kududb/client/SecureRpcHelper.java | 285 --
.../org/kududb/client/SessionConfiguration.java | 158 --
.../main/java/org/kududb/client/Statistics.java | 258 --
.../src/main/java/org/kududb/client/Status.java | 373 ---
.../java/org/kududb/client/TabletClient.java | 879 -------
.../src/main/java/org/kududb/client/Update.java | 37 -
.../src/main/java/org/kududb/client/Upsert.java | 37 -
.../main/java/org/kududb/util/AsyncUtil.java | 73 -
.../java/org/kududb/util/HybridTimeUtil.java | 70 -
.../src/main/java/org/kududb/util/NetUtil.java | 78 -
.../src/main/java/org/kududb/util/Pair.java | 57 -
.../src/main/java/org/kududb/util/Slice.java | 702 -----
.../src/main/java/org/kududb/util/Slices.java | 259 --
.../org/apache/kudu/client/BaseKuduTest.java | 438 ++++
.../java/org/apache/kudu/client/ITClient.java | 396 +++
.../kudu/client/ITScannerMultiTablet.java | 129 +
.../org/apache/kudu/client/MiniKuduCluster.java | 474 ++++
.../apache/kudu/client/TestAsyncKuduClient.java | 157 ++
.../kudu/client/TestAsyncKuduSession.java | 514 ++++
.../java/org/apache/kudu/client/TestBitSet.java | 99 +
.../java/org/apache/kudu/client/TestBytes.java | 105 +
.../kudu/client/TestColumnRangePredicate.java | 72 +
.../apache/kudu/client/TestDeadlineTracker.java | 74 +
.../apache/kudu/client/TestErrorCollector.java | 90 +
.../kudu/client/TestFlexiblePartitioning.java | 422 +++
.../org/apache/kudu/client/TestHybridTime.java | 163 ++
.../org/apache/kudu/client/TestKeyEncoding.java | 272 ++
.../org/apache/kudu/client/TestKuduClient.java | 535 ++++
.../apache/kudu/client/TestKuduPredicate.java | 628 +++++
.../org/apache/kudu/client/TestKuduSession.java | 337 +++
.../org/apache/kudu/client/TestKuduTable.java | 301 +++
.../apache/kudu/client/TestLeaderFailover.java | 69 +
.../apache/kudu/client/TestMasterFailover.java | 72 +
.../apache/kudu/client/TestMiniKuduCluster.java | 116 +
.../org/apache/kudu/client/TestOperation.java | 166 ++
.../apache/kudu/client/TestRequestTracker.java | 74 +
.../org/apache/kudu/client/TestRowErrors.java | 98 +
.../org/apache/kudu/client/TestRowResult.java | 129 +
.../apache/kudu/client/TestScanPredicate.java | 609 +++++
.../kudu/client/TestScannerMultiTablet.java | 236 ++
.../org/apache/kudu/client/TestStatistics.java | 74 +
.../java/org/apache/kudu/client/TestStatus.java | 55 +
.../org/apache/kudu/client/TestTestUtils.java | 114 +
.../org/apache/kudu/client/TestTimeouts.java | 68 +
.../java/org/apache/kudu/client/TestUtils.java | 289 +++
.../org/apache/kudu/util/TestAsyncUtil.java | 75 +
.../org/apache/kudu/util/TestMurmurHash.java | 46 +
.../java/org/apache/kudu/util/TestNetUtil.java | 73 +
.../java/org/kududb/client/BaseKuduTest.java | 438 ----
.../test/java/org/kududb/client/ITClient.java | 396 ---
.../org/kududb/client/ITScannerMultiTablet.java | 129 -
.../java/org/kududb/client/MiniKuduCluster.java | 474 ----
.../org/kududb/client/TestAsyncKuduClient.java | 157 --
.../org/kududb/client/TestAsyncKuduSession.java | 514 ----
.../test/java/org/kududb/client/TestBitSet.java | 99 -
.../test/java/org/kududb/client/TestBytes.java | 105 -
.../kududb/client/TestColumnRangePredicate.java | 72 -
.../org/kududb/client/TestDeadlineTracker.java | 74 -
.../org/kududb/client/TestErrorCollector.java | 90 -
.../kududb/client/TestFlexiblePartitioning.java | 422 ---
.../java/org/kududb/client/TestHybridTime.java | 163 --
.../java/org/kududb/client/TestKeyEncoding.java | 272 --
.../java/org/kududb/client/TestKuduClient.java | 535 ----
.../org/kududb/client/TestKuduPredicate.java | 628 -----
.../java/org/kududb/client/TestKuduSession.java | 337 ---
.../java/org/kududb/client/TestKuduTable.java | 301 ---
.../org/kududb/client/TestLeaderFailover.java | 69 -
.../org/kududb/client/TestMasterFailover.java | 72 -
.../org/kududb/client/TestMiniKuduCluster.java | 116 -
.../java/org/kududb/client/TestOperation.java | 166 --
.../org/kududb/client/TestRequestTracker.java | 74 -
.../java/org/kududb/client/TestRowErrors.java | 98 -
.../java/org/kududb/client/TestRowResult.java | 129 -
.../org/kududb/client/TestScanPredicate.java | 609 -----
.../kududb/client/TestScannerMultiTablet.java | 236 --
.../java/org/kududb/client/TestStatistics.java | 74 -
.../test/java/org/kududb/client/TestStatus.java | 55 -
.../java/org/kududb/client/TestTestUtils.java | 114 -
.../java/org/kududb/client/TestTimeouts.java | 68 -
.../test/java/org/kududb/client/TestUtils.java | 289 ---
.../java/org/kududb/util/TestAsyncUtil.java | 75 -
.../java/org/kududb/util/TestMurmurHash.java | 46 -
.../test/java/org/kududb/util/TestNetUtil.java | 73 -
.../kudu/flume/sink/KuduEventProducer.java | 59 +
.../org/apache/kudu/flume/sink/KuduSink.java | 290 +++
.../sink/KuduSinkConfigurationConstants.java | 67 +
.../flume/sink/SimpleKuduEventProducer.java | 84 +
.../kududb/flume/sink/KuduEventProducer.java | 59 -
.../java/org/kududb/flume/sink/KuduSink.java | 290 ---
.../sink/KuduSinkConfigurationConstants.java | 67 -
.../flume/sink/SimpleKuduEventProducer.java | 84 -
.../apache/kudu/flume/sink/KuduSinkTest.java | 246 ++
.../org/kududb/flume/sink/KuduSinkTest.java | 246 --
.../kudu/mapreduce/CommandLineParser.java | 144 ++
.../org/apache/kudu/mapreduce/JarFinder.java | 179 ++
.../kudu/mapreduce/KuduTableInputFormat.java | 444 ++++
.../kudu/mapreduce/KuduTableMapReduceUtil.java | 541 ++++
.../mapreduce/KuduTableOutputCommitter.java | 57 +
.../kudu/mapreduce/KuduTableOutputFormat.java | 215 ++
.../org/apache/kudu/mapreduce/TableReducer.java | 28 +
.../org/kududb/mapreduce/CommandLineParser.java | 144 --
.../java/org/kududb/mapreduce/JarFinder.java | 179 --
.../kududb/mapreduce/KuduTableInputFormat.java | 444 ----
.../mapreduce/KuduTableMapReduceUtil.java | 541 ----
.../mapreduce/KuduTableOutputCommitter.java | 57 -
.../kududb/mapreduce/KuduTableOutputFormat.java | 215 --
.../java/org/kududb/mapreduce/TableReducer.java | 28 -
.../kudu/mapreduce/HadoopTestingUtility.java | 101 +
.../apache/kudu/mapreduce/ITInputFormatJob.java | 129 +
.../kudu/mapreduce/ITKuduTableInputFormat.java | 132 +
.../kudu/mapreduce/ITKuduTableOutputFormat.java | 66 +
.../kudu/mapreduce/ITOutputFormatJob.java | 131 +
.../apache/kudu/mapreduce/TestJarFinder.java | 128 +
.../kududb/mapreduce/HadoopTestingUtility.java | 101 -
.../org/kududb/mapreduce/ITInputFormatJob.java | 129 -
.../mapreduce/ITKuduTableInputFormat.java | 132 -
.../mapreduce/ITKuduTableOutputFormat.java | 66 -
.../org/kududb/mapreduce/ITOutputFormatJob.java | 131 -
.../org/kududb/mapreduce/TestJarFinder.java | 128 -
.../apache/kudu/spark/kudu/DefaultSource.scala | 276 ++
.../apache/kudu/spark/kudu/KuduContext.scala | 227 ++
.../org/apache/kudu/spark/kudu/KuduRDD.scala | 133 +
.../org/apache/kudu/spark/kudu/package.scala | 38 +
.../org/kududb/spark/kudu/DefaultSource.scala | 276 --
.../org/kududb/spark/kudu/KuduContext.scala | 227 --
.../scala/org/kududb/spark/kudu/KuduRDD.scala | 133 -
.../scala/org/kududb/spark/kudu/package.scala | 38 -
.../kudu/spark/kudu/DefaultSourceTest.scala | 266 ++
.../kudu/spark/kudu/KuduContextTest.scala | 35 +
.../apache/kudu/spark/kudu/TestContext.scala | 124 +
.../kududb/spark/kudu/DefaultSourceTest.scala | 266 --
.../org/kududb/spark/kudu/KuduContextTest.scala | 35 -
.../org/kududb/spark/kudu/TestContext.scala | 124 -
318 files changed, 33362 insertions(+), 33362 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java
new file mode 100644
index 0000000..b834947
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java
@@ -0,0 +1,74 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Annotation to inform users of a package, class or method's intended audience.
+ * Currently the audience can be {@link Public}, {@link LimitedPrivate} or
+ * {@link Private}. <br>
+ * All public classes must have InterfaceAudience annotation. <br>
+ * <ul>
+ * <li>Public classes that are not marked with this annotation must be
+ * considered by default as {@link Private}.</li>
+ *
+ * <li>External applications must only use classes that are marked
+ * {@link Public}. Avoid using non public classes as these classes
+ * could be removed or change in incompatible ways.</li>
+ *
+ * <li>Hadoop projects must only use classes that are marked
+ * {@link LimitedPrivate} or {@link Public}</li>
+ *
+ * <li> Methods may have a different annotation that it is more restrictive
+ * compared to the audience classification of the class. Example: A class
+ * might be {@link Public}, but a method may be {@link LimitedPrivate}
+ * </li></ul>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class InterfaceAudience {
+ /**
+ * Intended for use by any project or application.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Public {};
+
+ /**
+ * Intended only for the project(s) specified in the annotation.
+ * For example, "Common", "HDFS", "MapReduce", "ZooKeeper", "HBase".
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface LimitedPrivate {
+ String[] value();
+ };
+
+ /**
+ * Intended for use only within Kudu itself.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Private {};
+
+ private InterfaceAudience() {} // Audience can't exist on its own
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java
new file mode 100644
index 0000000..84950e6
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java
@@ -0,0 +1,66 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+import org.kududb.annotations.InterfaceAudience.LimitedPrivate;
+import org.kududb.annotations.InterfaceAudience.Private;
+import org.kududb.annotations.InterfaceAudience.Public;
+
+/**
+ * Annotation to inform users of how much to rely on a particular package,
+ * class or method not changing over time. Currently the stability can be
+ * {@link Stable}, {@link Evolving} or {@link Unstable}. <br>
+ *
+ * <ul><li>All classes that are annotated with {@link Public} or
+ * {@link LimitedPrivate} must have InterfaceStability annotation. </li>
+ * <li>Classes that are {@link Private} are to be considered unstable unless
+ * a different InterfaceStability annotation states otherwise.</li>
+ * <li>Incompatible changes must not be made to classes marked as stable.</li>
+ * </ul>
+ */
+@Public
+@InterfaceStability.Evolving
+public class InterfaceStability {
+ /**
+ * Can evolve while retaining compatibility for minor release boundaries.;
+ * can break compatibility only at major release (ie. at m.0).
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Stable {};
+
+ /**
+ * Evolving, but can break compatibility at minor release (i.e. m.x)
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Evolving {};
+
+ /**
+ * No guarantee is provided as to reliability or stability across any
+ * level of release granularity.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Unstable {};
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
new file mode 100644
index 0000000..5c0c7b8
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
@@ -0,0 +1,61 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.annotations.tools;
+
+import com.sun.javadoc.DocErrorReporter;
+import com.sun.javadoc.LanguageVersion;
+import com.sun.javadoc.RootDoc;
+
+import jdiff.JDiff;
+import org.kududb.annotations.InterfaceAudience;
+
+/**
+ * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
+ * for excluding elements that are annotated with
+ * {@link InterfaceAudience.Private} or
+ * {@link InterfaceAudience.LimitedPrivate}.
+ * It delegates to the JDiff Doclet, and takes the same options.
+ */
+public class ExcludePrivateAnnotationsJDiffDoclet {
+
+ public static LanguageVersion languageVersion() {
+ return LanguageVersion.JAVA_1_5;
+ }
+
+ public static boolean start(RootDoc root) {
+ System.out.println(
+ ExcludePrivateAnnotationsJDiffDoclet.class.getSimpleName());
+ return JDiff.start(RootDocProcessor.process(root));
+ }
+
+ public static int optionLength(String option) {
+ Integer length = StabilityOptions.optionLength(option);
+ if (length != null) {
+ return length;
+ }
+ return JDiff.optionLength(option);
+ }
+
+ public static boolean validOptions(String[][] options,
+ DocErrorReporter reporter) {
+ StabilityOptions.validOptions(options, reporter);
+ String[][] filteredOptions = StabilityOptions.filterOptions(options);
+ return JDiff.validOptions(filteredOptions, reporter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
new file mode 100644
index 0000000..af8b088
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
@@ -0,0 +1,60 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.annotations.tools;
+
+import com.sun.javadoc.DocErrorReporter;
+import com.sun.javadoc.LanguageVersion;
+import com.sun.javadoc.RootDoc;
+import com.sun.tools.doclets.standard.Standard;
+import org.kududb.annotations.InterfaceAudience;
+
+/**
+ * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
+ * for excluding elements that are annotated with
+ * {@link InterfaceAudience.Private} or
+ * {@link InterfaceAudience.LimitedPrivate}.
+ * It delegates to the Standard Doclet, and takes the same options.
+ */
+public class ExcludePrivateAnnotationsStandardDoclet {
+
+ public static LanguageVersion languageVersion() {
+ return LanguageVersion.JAVA_1_5;
+ }
+
+ public static boolean start(RootDoc root) {
+ System.out.println(
+ ExcludePrivateAnnotationsStandardDoclet.class.getSimpleName());
+ return Standard.start(RootDocProcessor.process(root));
+ }
+
+ public static int optionLength(String option) {
+ Integer length = StabilityOptions.optionLength(option);
+ if (length != null) {
+ return length;
+ }
+ return Standard.optionLength(option);
+ }
+
+ public static boolean validOptions(String[][] options,
+ DocErrorReporter reporter) {
+ StabilityOptions.validOptions(options, reporter);
+ String[][] filteredOptions = StabilityOptions.filterOptions(options);
+ return Standard.validOptions(filteredOptions, reporter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
new file mode 100644
index 0000000..b5a67b2
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
@@ -0,0 +1,65 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.annotations.tools;
+
+import com.sun.javadoc.DocErrorReporter;
+import com.sun.javadoc.LanguageVersion;
+import com.sun.javadoc.RootDoc;
+import com.sun.tools.doclets.standard.Standard;
+import org.kududb.annotations.InterfaceAudience;
+
+/**
+ * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
+ * that only includes class-level elements that are annotated with
+ * {@link InterfaceAudience.Public}.
+ * Class-level elements with no annotation are excluded.
+ * In addition, all elements that are annotated with
+ * {@link InterfaceAudience.Private} or
+ * {@link InterfaceAudience.LimitedPrivate}
+ * are also excluded.
+ * It delegates to the Standard Doclet, and takes the same options.
+ */
+public class IncludePublicAnnotationsStandardDoclet {
+
+ public static LanguageVersion languageVersion() {
+ return LanguageVersion.JAVA_1_5;
+ }
+
+ public static boolean start(RootDoc root) {
+ System.out.println(
+ IncludePublicAnnotationsStandardDoclet.class.getSimpleName());
+ RootDocProcessor.treatUnannotatedClassesAsPrivate = true;
+ return Standard.start(RootDocProcessor.process(root));
+ }
+
+ public static int optionLength(String option) {
+ Integer length = StabilityOptions.optionLength(option);
+ if (length != null) {
+ return length;
+ }
+ return Standard.optionLength(option);
+ }
+
+ public static boolean validOptions(String[][] options,
+ DocErrorReporter reporter) {
+ StabilityOptions.validOptions(options, reporter);
+ String[][] filteredOptions = StabilityOptions.filterOptions(options);
+ return Standard.validOptions(filteredOptions, reporter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java
new file mode 100644
index 0000000..c4f19fb
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java
@@ -0,0 +1,248 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.annotations.tools;
+
+import com.sun.javadoc.AnnotationDesc;
+import com.sun.javadoc.AnnotationTypeDoc;
+import com.sun.javadoc.ClassDoc;
+import com.sun.javadoc.ConstructorDoc;
+import com.sun.javadoc.Doc;
+import com.sun.javadoc.FieldDoc;
+import com.sun.javadoc.MethodDoc;
+import com.sun.javadoc.PackageDoc;
+import com.sun.javadoc.ProgramElementDoc;
+import com.sun.javadoc.RootDoc;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Process the {@link RootDoc} by substituting with (nested) proxy objects that
+ * exclude elements with Private or LimitedPrivate annotations.
+ * <p>
+ * Based on code from http://www.sixlegs.com/blog/java/exclude-javadoc-tag.html.
+ */
+class RootDocProcessor {
+
+ static String stability = StabilityOptions.UNSTABLE_OPTION;
+ static boolean treatUnannotatedClassesAsPrivate = false;
+
+ public static RootDoc process(RootDoc root) {
+ return (RootDoc) process(root, RootDoc.class);
+ }
+
+ private static Object process(Object obj, Class<?> type) {
+ if (obj == null) {
+ return null;
+ }
+ Class<?> cls = obj.getClass();
+ if (cls.getName().startsWith("com.sun.")) {
+ return getProxy(obj);
+ } else if (obj instanceof Object[]) {
+ Class<?> componentType = type.isArray() ? type.getComponentType()
+ : cls.getComponentType();
+ Object[] array = (Object[]) obj;
+ Object[] newArray = (Object[]) Array.newInstance(componentType,
+ array.length);
+ for (int i = 0; i < array.length; ++i) {
+ newArray[i] = process(array[i], componentType);
+ }
+ return newArray;
+ }
+ return obj;
+ }
+
+ private static Map<Object, Object> proxies =
+ new WeakHashMap<Object, Object>();
+
+ private static Object getProxy(Object obj) {
+ Object proxy = proxies.get(obj);
+ if (proxy == null) {
+ proxy = Proxy.newProxyInstance(obj.getClass().getClassLoader(),
+ obj.getClass().getInterfaces(), new ExcludeHandler(obj));
+ proxies.put(obj, proxy);
+ }
+ return proxy;
+ }
+
+ private static class ExcludeHandler implements InvocationHandler {
+ private Object target;
+
+ public ExcludeHandler(Object target) {
+ this.target = target;
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args)
+ throws Throwable {
+ String methodName = method.getName();
+ if (target instanceof Doc) {
+ if (methodName.equals("isIncluded")) {
+ Doc doc = (Doc) target;
+ return !exclude(doc) && doc.isIncluded();
+ }
+ if (target instanceof RootDoc) {
+ if (methodName.equals("classes")) {
+ return filter(((RootDoc) target).classes(), ClassDoc.class);
+ } else if (methodName.equals("specifiedClasses")) {
+ return filter(((RootDoc) target).specifiedClasses(), ClassDoc.class);
+ } else if (methodName.equals("specifiedPackages")) {
+ return filter(((RootDoc) target).specifiedPackages(), PackageDoc.class);
+ }
+ } else if (target instanceof ClassDoc) {
+ if (isFiltered(args)) {
+ if (methodName.equals("methods")) {
+ return filter(((ClassDoc) target).methods(true), MethodDoc.class);
+ } else if (methodName.equals("fields")) {
+ return filter(((ClassDoc) target).fields(true), FieldDoc.class);
+ } else if (methodName.equals("innerClasses")) {
+ return filter(((ClassDoc) target).innerClasses(true),
+ ClassDoc.class);
+ } else if (methodName.equals("constructors")) {
+ return filter(((ClassDoc) target).constructors(true),
+ ConstructorDoc.class);
+ }
+ }
+ } else if (target instanceof PackageDoc) {
+ if (methodName.equals("allClasses")) {
+ if (isFiltered(args)) {
+ return filter(((PackageDoc) target).allClasses(true),
+ ClassDoc.class);
+ } else {
+ return filter(((PackageDoc) target).allClasses(), ClassDoc.class);
+ }
+ } else if (methodName.equals("annotationTypes")) {
+ return filter(((PackageDoc) target).annotationTypes(),
+ AnnotationTypeDoc.class);
+ } else if (methodName.equals("enums")) {
+ return filter(((PackageDoc) target).enums(),
+ ClassDoc.class);
+ } else if (methodName.equals("errors")) {
+ return filter(((PackageDoc) target).errors(),
+ ClassDoc.class);
+ } else if (methodName.equals("exceptions")) {
+ return filter(((PackageDoc) target).exceptions(),
+ ClassDoc.class);
+ } else if (methodName.equals("interfaces")) {
+ return filter(((PackageDoc) target).interfaces(),
+ ClassDoc.class);
+ } else if (methodName.equals("ordinaryClasses")) {
+ return filter(((PackageDoc) target).ordinaryClasses(),
+ ClassDoc.class);
+ }
+ }
+ }
+
+ if (args != null) {
+ if (methodName.equals("compareTo") || methodName.equals("equals")
+ || methodName.equals("overrides")
+ || methodName.equals("subclassOf")) {
+ args[0] = unwrap(args[0]);
+ }
+ }
+ try {
+ return process(method.invoke(target, args), method.getReturnType());
+ } catch (InvocationTargetException e) {
+ throw e.getTargetException();
+ }
+ }
+
+ private static boolean exclude(Doc doc) {
+ AnnotationDesc[] annotations = null;
+ if (doc instanceof ProgramElementDoc) {
+ annotations = ((ProgramElementDoc) doc).annotations();
+ } else if (doc instanceof PackageDoc) {
+ annotations = ((PackageDoc) doc).annotations();
+ }
+ if (annotations != null) {
+ for (AnnotationDesc annotation : annotations) {
+ String qualifiedTypeName = annotation.annotationType().qualifiedTypeName();
+ if (qualifiedTypeName.equals(
+ InterfaceAudience.Private.class.getCanonicalName())
+ || qualifiedTypeName.equals(
+ InterfaceAudience.LimitedPrivate.class.getCanonicalName())) {
+ return true;
+ }
+ if (stability.equals(StabilityOptions.EVOLVING_OPTION)) {
+ if (qualifiedTypeName.equals(
+ InterfaceStability.Unstable.class.getCanonicalName())) {
+ return true;
+ }
+ }
+ if (stability.equals(StabilityOptions.STABLE_OPTION)) {
+ if (qualifiedTypeName.equals(
+ InterfaceStability.Unstable.class.getCanonicalName())
+ || qualifiedTypeName.equals(
+ InterfaceStability.Evolving.class.getCanonicalName())) {
+ return true;
+ }
+ }
+ }
+ for (AnnotationDesc annotation : annotations) {
+ String qualifiedTypeName =
+ annotation.annotationType().qualifiedTypeName();
+ if (qualifiedTypeName.equals(
+ InterfaceAudience.Public.class.getCanonicalName())) {
+ return false;
+ }
+ }
+ }
+ if (treatUnannotatedClassesAsPrivate) {
+ return doc.isClass() || doc.isInterface() || doc.isAnnotationType();
+ }
+ return false;
+ }
+
+ private static Object[] filter(Doc[] array, Class<?> componentType) {
+ if (array == null || array.length == 0) {
+ return array;
+ }
+ List<Object> list = new ArrayList<Object>(array.length);
+ for (Doc entry : array) {
+ if (!exclude(entry)) {
+ list.add(process(entry, componentType));
+ }
+ }
+ return list.toArray((Object[]) Array.newInstance(componentType, list
+ .size()));
+ }
+
+ private Object unwrap(Object proxy) {
+ if (proxy instanceof Proxy)
+ return ((ExcludeHandler) Proxy.getInvocationHandler(proxy)).target;
+ return proxy;
+ }
+
+ private boolean isFiltered(Object[] args) {
+ return args != null && Boolean.TRUE.equals(args[0]);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java
new file mode 100644
index 0000000..d5cf5e1
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java
@@ -0,0 +1,71 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.annotations.tools;
+
+import com.sun.javadoc.DocErrorReporter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+class StabilityOptions {
+ public static final String STABLE_OPTION = "-stable";
+ public static final String EVOLVING_OPTION = "-evolving";
+ public static final String UNSTABLE_OPTION = "-unstable";
+
+ public static Integer optionLength(String option) {
+ String opt = option.toLowerCase(Locale.ENGLISH);
+ if (opt.equals(UNSTABLE_OPTION)) return 1;
+ if (opt.equals(EVOLVING_OPTION)) return 1;
+ if (opt.equals(STABLE_OPTION)) return 1;
+ return null;
+ }
+
+ public static void validOptions(String[][] options,
+ DocErrorReporter reporter) {
+ for (int i = 0; i < options.length; i++) {
+ String opt = options[i][0].toLowerCase(Locale.ENGLISH);
+ if (opt.equals(UNSTABLE_OPTION)) {
+ RootDocProcessor.stability = UNSTABLE_OPTION;
+ } else if (opt.equals(EVOLVING_OPTION)) {
+ RootDocProcessor.stability = EVOLVING_OPTION;
+ } else if (opt.equals(STABLE_OPTION)) {
+ RootDocProcessor.stability = STABLE_OPTION;
+ }
+ }
+ }
+
+ public static String[][] filterOptions(String[][] options) {
+ List<String[]> optionsList = new ArrayList<String[]>();
+ for (int i = 0; i < options.length; i++) {
+ if (!options[i][0].equalsIgnoreCase(UNSTABLE_OPTION)
+ && !options[i][0].equalsIgnoreCase(EVOLVING_OPTION)
+ && !options[i][0].equalsIgnoreCase(STABLE_OPTION)) {
+ optionsList.add(options[i]);
+ }
+ }
+ String[][] filteredOptions = new String[optionsList.size()][];
+ int i = 0;
+ for (String[] option : optionsList) {
+ filteredOptions[i++] = option;
+ }
+ return filteredOptions;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java
new file mode 100644
index 0000000..ec0103b
--- /dev/null
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java
@@ -0,0 +1,22 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+package org.kududb.annotations.tools;
+
+import org.kududb.annotations.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceAudience.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceAudience.java b/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceAudience.java
deleted file mode 100644
index b834947..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceAudience.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.annotations;
-
-import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-
-/**
- * Annotation to inform users of a package, class or method's intended audience.
- * Currently the audience can be {@link Public}, {@link LimitedPrivate} or
- * {@link Private}. <br>
- * All public classes must have InterfaceAudience annotation. <br>
- * <ul>
- * <li>Public classes that are not marked with this annotation must be
- * considered by default as {@link Private}.</li>
- *
- * <li>External applications must only use classes that are marked
- * {@link Public}. Avoid using non public classes as these classes
- * could be removed or change in incompatible ways.</li>
- *
- * <li>Hadoop projects must only use classes that are marked
- * {@link LimitedPrivate} or {@link Public}</li>
- *
- * <li> Methods may have a different annotation that it is more restrictive
- * compared to the audience classification of the class. Example: A class
- * might be {@link Public}, but a method may be {@link LimitedPrivate}
- * </li></ul>
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class InterfaceAudience {
- /**
- * Intended for use by any project or application.
- */
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface Public {};
-
- /**
- * Intended only for the project(s) specified in the annotation.
- * For example, "Common", "HDFS", "MapReduce", "ZooKeeper", "HBase".
- */
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface LimitedPrivate {
- String[] value();
- };
-
- /**
- * Intended for use only within Kudu itself.
- */
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface Private {};
-
- private InterfaceAudience() {} // Audience can't exist on its own
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceStability.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceStability.java b/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceStability.java
deleted file mode 100644
index 84950e6..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/InterfaceStability.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.annotations;
-
-import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-
-import org.kududb.annotations.InterfaceAudience.LimitedPrivate;
-import org.kududb.annotations.InterfaceAudience.Private;
-import org.kududb.annotations.InterfaceAudience.Public;
-
-/**
- * Annotation to inform users of how much to rely on a particular package,
- * class or method not changing over time. Currently the stability can be
- * {@link Stable}, {@link Evolving} or {@link Unstable}. <br>
- *
- * <ul><li>All classes that are annotated with {@link Public} or
- * {@link LimitedPrivate} must have InterfaceStability annotation. </li>
- * <li>Classes that are {@link Private} are to be considered unstable unless
- * a different InterfaceStability annotation states otherwise.</li>
- * <li>Incompatible changes must not be made to classes marked as stable.</li>
- * </ul>
- */
-@Public
-@InterfaceStability.Evolving
-public class InterfaceStability {
- /**
- * Can evolve while retaining compatibility for minor release boundaries.;
- * can break compatibility only at major release (ie. at m.0).
- */
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface Stable {};
-
- /**
- * Evolving, but can break compatibility at minor release (i.e. m.x)
- */
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface Evolving {};
-
- /**
- * No guarantee is provided as to reliability or stability across any
- * level of release granularity.
- */
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface Unstable {};
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java b/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
deleted file mode 100644
index 5c0c7b8..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.annotations.tools;
-
-import com.sun.javadoc.DocErrorReporter;
-import com.sun.javadoc.LanguageVersion;
-import com.sun.javadoc.RootDoc;
-
-import jdiff.JDiff;
-import org.kududb.annotations.InterfaceAudience;
-
-/**
- * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
- * for excluding elements that are annotated with
- * {@link InterfaceAudience.Private} or
- * {@link InterfaceAudience.LimitedPrivate}.
- * It delegates to the JDiff Doclet, and takes the same options.
- */
-public class ExcludePrivateAnnotationsJDiffDoclet {
-
- public static LanguageVersion languageVersion() {
- return LanguageVersion.JAVA_1_5;
- }
-
- public static boolean start(RootDoc root) {
- System.out.println(
- ExcludePrivateAnnotationsJDiffDoclet.class.getSimpleName());
- return JDiff.start(RootDocProcessor.process(root));
- }
-
- public static int optionLength(String option) {
- Integer length = StabilityOptions.optionLength(option);
- if (length != null) {
- return length;
- }
- return JDiff.optionLength(option);
- }
-
- public static boolean validOptions(String[][] options,
- DocErrorReporter reporter) {
- StabilityOptions.validOptions(options, reporter);
- String[][] filteredOptions = StabilityOptions.filterOptions(options);
- return JDiff.validOptions(filteredOptions, reporter);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
deleted file mode 100644
index af8b088..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.annotations.tools;
-
-import com.sun.javadoc.DocErrorReporter;
-import com.sun.javadoc.LanguageVersion;
-import com.sun.javadoc.RootDoc;
-import com.sun.tools.doclets.standard.Standard;
-import org.kududb.annotations.InterfaceAudience;
-
-/**
- * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
- * for excluding elements that are annotated with
- * {@link InterfaceAudience.Private} or
- * {@link InterfaceAudience.LimitedPrivate}.
- * It delegates to the Standard Doclet, and takes the same options.
- */
-public class ExcludePrivateAnnotationsStandardDoclet {
-
- public static LanguageVersion languageVersion() {
- return LanguageVersion.JAVA_1_5;
- }
-
- public static boolean start(RootDoc root) {
- System.out.println(
- ExcludePrivateAnnotationsStandardDoclet.class.getSimpleName());
- return Standard.start(RootDocProcessor.process(root));
- }
-
- public static int optionLength(String option) {
- Integer length = StabilityOptions.optionLength(option);
- if (length != null) {
- return length;
- }
- return Standard.optionLength(option);
- }
-
- public static boolean validOptions(String[][] options,
- DocErrorReporter reporter) {
- StabilityOptions.validOptions(options, reporter);
- String[][] filteredOptions = StabilityOptions.filterOptions(options);
- return Standard.validOptions(filteredOptions, reporter);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/IncludePublicAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/kududb/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
deleted file mode 100644
index b5a67b2..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.annotations.tools;
-
-import com.sun.javadoc.DocErrorReporter;
-import com.sun.javadoc.LanguageVersion;
-import com.sun.javadoc.RootDoc;
-import com.sun.tools.doclets.standard.Standard;
-import org.kududb.annotations.InterfaceAudience;
-
-/**
- * A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
- * that only includes class-level elements that are annotated with
- * {@link InterfaceAudience.Public}.
- * Class-level elements with no annotation are excluded.
- * In addition, all elements that are annotated with
- * {@link InterfaceAudience.Private} or
- * {@link InterfaceAudience.LimitedPrivate}
- * are also excluded.
- * It delegates to the Standard Doclet, and takes the same options.
- */
-public class IncludePublicAnnotationsStandardDoclet {
-
- public static LanguageVersion languageVersion() {
- return LanguageVersion.JAVA_1_5;
- }
-
- public static boolean start(RootDoc root) {
- System.out.println(
- IncludePublicAnnotationsStandardDoclet.class.getSimpleName());
- RootDocProcessor.treatUnannotatedClassesAsPrivate = true;
- return Standard.start(RootDocProcessor.process(root));
- }
-
- public static int optionLength(String option) {
- Integer length = StabilityOptions.optionLength(option);
- if (length != null) {
- return length;
- }
- return Standard.optionLength(option);
- }
-
- public static boolean validOptions(String[][] options,
- DocErrorReporter reporter) {
- StabilityOptions.validOptions(options, reporter);
- String[][] filteredOptions = StabilityOptions.filterOptions(options);
- return Standard.validOptions(filteredOptions, reporter);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/tools/RootDocProcessor.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/RootDocProcessor.java b/java/interface-annotations/src/main/java/org/kududb/annotations/tools/RootDocProcessor.java
deleted file mode 100644
index c4f19fb..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/RootDocProcessor.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.annotations.tools;
-
-import com.sun.javadoc.AnnotationDesc;
-import com.sun.javadoc.AnnotationTypeDoc;
-import com.sun.javadoc.ClassDoc;
-import com.sun.javadoc.ConstructorDoc;
-import com.sun.javadoc.Doc;
-import com.sun.javadoc.FieldDoc;
-import com.sun.javadoc.MethodDoc;
-import com.sun.javadoc.PackageDoc;
-import com.sun.javadoc.ProgramElementDoc;
-import com.sun.javadoc.RootDoc;
-
-import java.lang.reflect.Array;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.WeakHashMap;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Process the {@link RootDoc} by substituting with (nested) proxy objects that
- * exclude elements with Private or LimitedPrivate annotations.
- * <p>
- * Based on code from http://www.sixlegs.com/blog/java/exclude-javadoc-tag.html.
- */
-class RootDocProcessor {
-
- static String stability = StabilityOptions.UNSTABLE_OPTION;
- static boolean treatUnannotatedClassesAsPrivate = false;
-
- public static RootDoc process(RootDoc root) {
- return (RootDoc) process(root, RootDoc.class);
- }
-
- private static Object process(Object obj, Class<?> type) {
- if (obj == null) {
- return null;
- }
- Class<?> cls = obj.getClass();
- if (cls.getName().startsWith("com.sun.")) {
- return getProxy(obj);
- } else if (obj instanceof Object[]) {
- Class<?> componentType = type.isArray() ? type.getComponentType()
- : cls.getComponentType();
- Object[] array = (Object[]) obj;
- Object[] newArray = (Object[]) Array.newInstance(componentType,
- array.length);
- for (int i = 0; i < array.length; ++i) {
- newArray[i] = process(array[i], componentType);
- }
- return newArray;
- }
- return obj;
- }
-
- private static Map<Object, Object> proxies =
- new WeakHashMap<Object, Object>();
-
- private static Object getProxy(Object obj) {
- Object proxy = proxies.get(obj);
- if (proxy == null) {
- proxy = Proxy.newProxyInstance(obj.getClass().getClassLoader(),
- obj.getClass().getInterfaces(), new ExcludeHandler(obj));
- proxies.put(obj, proxy);
- }
- return proxy;
- }
-
- private static class ExcludeHandler implements InvocationHandler {
- private Object target;
-
- public ExcludeHandler(Object target) {
- this.target = target;
- }
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args)
- throws Throwable {
- String methodName = method.getName();
- if (target instanceof Doc) {
- if (methodName.equals("isIncluded")) {
- Doc doc = (Doc) target;
- return !exclude(doc) && doc.isIncluded();
- }
- if (target instanceof RootDoc) {
- if (methodName.equals("classes")) {
- return filter(((RootDoc) target).classes(), ClassDoc.class);
- } else if (methodName.equals("specifiedClasses")) {
- return filter(((RootDoc) target).specifiedClasses(), ClassDoc.class);
- } else if (methodName.equals("specifiedPackages")) {
- return filter(((RootDoc) target).specifiedPackages(), PackageDoc.class);
- }
- } else if (target instanceof ClassDoc) {
- if (isFiltered(args)) {
- if (methodName.equals("methods")) {
- return filter(((ClassDoc) target).methods(true), MethodDoc.class);
- } else if (methodName.equals("fields")) {
- return filter(((ClassDoc) target).fields(true), FieldDoc.class);
- } else if (methodName.equals("innerClasses")) {
- return filter(((ClassDoc) target).innerClasses(true),
- ClassDoc.class);
- } else if (methodName.equals("constructors")) {
- return filter(((ClassDoc) target).constructors(true),
- ConstructorDoc.class);
- }
- }
- } else if (target instanceof PackageDoc) {
- if (methodName.equals("allClasses")) {
- if (isFiltered(args)) {
- return filter(((PackageDoc) target).allClasses(true),
- ClassDoc.class);
- } else {
- return filter(((PackageDoc) target).allClasses(), ClassDoc.class);
- }
- } else if (methodName.equals("annotationTypes")) {
- return filter(((PackageDoc) target).annotationTypes(),
- AnnotationTypeDoc.class);
- } else if (methodName.equals("enums")) {
- return filter(((PackageDoc) target).enums(),
- ClassDoc.class);
- } else if (methodName.equals("errors")) {
- return filter(((PackageDoc) target).errors(),
- ClassDoc.class);
- } else if (methodName.equals("exceptions")) {
- return filter(((PackageDoc) target).exceptions(),
- ClassDoc.class);
- } else if (methodName.equals("interfaces")) {
- return filter(((PackageDoc) target).interfaces(),
- ClassDoc.class);
- } else if (methodName.equals("ordinaryClasses")) {
- return filter(((PackageDoc) target).ordinaryClasses(),
- ClassDoc.class);
- }
- }
- }
-
- if (args != null) {
- if (methodName.equals("compareTo") || methodName.equals("equals")
- || methodName.equals("overrides")
- || methodName.equals("subclassOf")) {
- args[0] = unwrap(args[0]);
- }
- }
- try {
- return process(method.invoke(target, args), method.getReturnType());
- } catch (InvocationTargetException e) {
- throw e.getTargetException();
- }
- }
-
- private static boolean exclude(Doc doc) {
- AnnotationDesc[] annotations = null;
- if (doc instanceof ProgramElementDoc) {
- annotations = ((ProgramElementDoc) doc).annotations();
- } else if (doc instanceof PackageDoc) {
- annotations = ((PackageDoc) doc).annotations();
- }
- if (annotations != null) {
- for (AnnotationDesc annotation : annotations) {
- String qualifiedTypeName = annotation.annotationType().qualifiedTypeName();
- if (qualifiedTypeName.equals(
- InterfaceAudience.Private.class.getCanonicalName())
- || qualifiedTypeName.equals(
- InterfaceAudience.LimitedPrivate.class.getCanonicalName())) {
- return true;
- }
- if (stability.equals(StabilityOptions.EVOLVING_OPTION)) {
- if (qualifiedTypeName.equals(
- InterfaceStability.Unstable.class.getCanonicalName())) {
- return true;
- }
- }
- if (stability.equals(StabilityOptions.STABLE_OPTION)) {
- if (qualifiedTypeName.equals(
- InterfaceStability.Unstable.class.getCanonicalName())
- || qualifiedTypeName.equals(
- InterfaceStability.Evolving.class.getCanonicalName())) {
- return true;
- }
- }
- }
- for (AnnotationDesc annotation : annotations) {
- String qualifiedTypeName =
- annotation.annotationType().qualifiedTypeName();
- if (qualifiedTypeName.equals(
- InterfaceAudience.Public.class.getCanonicalName())) {
- return false;
- }
- }
- }
- if (treatUnannotatedClassesAsPrivate) {
- return doc.isClass() || doc.isInterface() || doc.isAnnotationType();
- }
- return false;
- }
-
- private static Object[] filter(Doc[] array, Class<?> componentType) {
- if (array == null || array.length == 0) {
- return array;
- }
- List<Object> list = new ArrayList<Object>(array.length);
- for (Doc entry : array) {
- if (!exclude(entry)) {
- list.add(process(entry, componentType));
- }
- }
- return list.toArray((Object[]) Array.newInstance(componentType, list
- .size()));
- }
-
- private Object unwrap(Object proxy) {
- if (proxy instanceof Proxy)
- return ((ExcludeHandler) Proxy.getInvocationHandler(proxy)).target;
- return proxy;
- }
-
- private boolean isFiltered(Object[] args) {
- return args != null && Boolean.TRUE.equals(args[0]);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/tools/StabilityOptions.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/StabilityOptions.java b/java/interface-annotations/src/main/java/org/kududb/annotations/tools/StabilityOptions.java
deleted file mode 100644
index d5cf5e1..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/StabilityOptions.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.annotations.tools;
-
-import com.sun.javadoc.DocErrorReporter;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
-class StabilityOptions {
- public static final String STABLE_OPTION = "-stable";
- public static final String EVOLVING_OPTION = "-evolving";
- public static final String UNSTABLE_OPTION = "-unstable";
-
- public static Integer optionLength(String option) {
- String opt = option.toLowerCase(Locale.ENGLISH);
- if (opt.equals(UNSTABLE_OPTION)) return 1;
- if (opt.equals(EVOLVING_OPTION)) return 1;
- if (opt.equals(STABLE_OPTION)) return 1;
- return null;
- }
-
- public static void validOptions(String[][] options,
- DocErrorReporter reporter) {
- for (int i = 0; i < options.length; i++) {
- String opt = options[i][0].toLowerCase(Locale.ENGLISH);
- if (opt.equals(UNSTABLE_OPTION)) {
- RootDocProcessor.stability = UNSTABLE_OPTION;
- } else if (opt.equals(EVOLVING_OPTION)) {
- RootDocProcessor.stability = EVOLVING_OPTION;
- } else if (opt.equals(STABLE_OPTION)) {
- RootDocProcessor.stability = STABLE_OPTION;
- }
- }
- }
-
- public static String[][] filterOptions(String[][] options) {
- List<String[]> optionsList = new ArrayList<String[]>();
- for (int i = 0; i < options.length; i++) {
- if (!options[i][0].equalsIgnoreCase(UNSTABLE_OPTION)
- && !options[i][0].equalsIgnoreCase(EVOLVING_OPTION)
- && !options[i][0].equalsIgnoreCase(STABLE_OPTION)) {
- optionsList.add(options[i]);
- }
- }
- String[][] filteredOptions = new String[optionsList.size()][];
- int i = 0;
- for (String[] option : optionsList) {
- filteredOptions[i++] = option;
- }
- return filteredOptions;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/interface-annotations/src/main/java/org/kududb/annotations/tools/package-info.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/package-info.java b/java/interface-annotations/src/main/java/org/kududb/annotations/tools/package-info.java
deleted file mode 100644
index ec0103b..0000000
--- a/java/interface-annotations/src/main/java/org/kududb/annotations/tools/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-@InterfaceAudience.Private
-package org.kududb.annotations.tools;
-
-import org.kududb.annotations.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java
new file mode 100644
index 0000000..945b82c
--- /dev/null
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java
@@ -0,0 +1,162 @@
+/**
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.mapreduce.tools;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.Bytes;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Column-separated values parser that gives access to the different columns inside each line of
+ * data.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class CsvParser {
+
+ private final byte separatorByte;
+
+ private final int maxColumnCount;
+
+ private final List<String> columnNames;
+
+ /**
+ * @param columnsSpecification the list of columns to parse out, comma separated.
+ * @param separatorStr The 1 byte separator.
+ */
+ public CsvParser(String columnsSpecification, String separatorStr) {
+ // Configure separator
+ byte[] separator = Bytes.fromString(separatorStr);
+ Preconditions.checkArgument(separator.length == 1, "CsvParser only supports single-byte " +
+ "separators");
+ separatorByte = separator[0];
+
+ // Configure columns
+ columnNames = Lists.newArrayList(Splitter.on(',').trimResults().split(columnsSpecification));
+
+ maxColumnCount = columnNames.size();
+ }
+
+ /**
+ * Creates a ParsedLine of a line of data.
+ * @param lineBytes Whole line as a byte array.
+ * @param length How long the line really is in the byte array
+ * @return A parsed line of CSV.
+ * @throws BadCsvLineException
+ */
+ public ParsedLine parse(byte[] lineBytes, int length) throws BadCsvLineException {
+ // Enumerate separator offsets
+ List<Integer> tabOffsets = new ArrayList<Integer>(maxColumnCount);
+ for (int i = 0; i < length; i++) {
+ if (lineBytes[i] == separatorByte) {
+ tabOffsets.add(i);
+ }
+ }
+ if (tabOffsets.isEmpty()) {
+ throw new BadCsvLineException("No delimiter");
+ }
+
+ // trailing separator shouldn't count as a column
+ if (lineBytes[length - 1] != separatorByte) {
+ tabOffsets.add(length);
+ }
+
+ if (tabOffsets.size() > maxColumnCount) {
+ throw new BadCsvLineException("Excessive columns");
+ }
+
+ if (tabOffsets.size() < maxColumnCount) {
+ throw new BadCsvLineException("Not enough columns");
+ }
+
+ return new ParsedLine(tabOffsets, lineBytes);
+ }
+
+ /**
+ * Helper class that knows where the columns are situated in the line.
+ */
+ class ParsedLine {
+ private final List<Integer> tabOffsets;
+ private final byte[] lineBytes;
+
+ ParsedLine(List<Integer> tabOffsets, byte[] lineBytes) {
+ this.tabOffsets = tabOffsets;
+ this.lineBytes = lineBytes;
+ }
+
+ /**
+ * Get the position for the given column.
+ * @param idx Column to lookup.
+ * @return Offset in the line.
+ */
+ public int getColumnOffset(int idx) {
+ if (idx > 0) {
+ return tabOffsets.get(idx - 1) + 1;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Get how many bytes the given column occupies.
+ * @param idx Column to lookup.
+ * @return Column's length.
+ */
+ public int getColumnLength(int idx) {
+ return tabOffsets.get(idx) - getColumnOffset(idx);
+ }
+
+ /**
+ * Get the number of columns in this file.
+ * @return Number of columns.
+ */
+ public int getColumnCount() {
+ return tabOffsets.size();
+ }
+
+ /**
+ * Get the bytes originally given for this line.
+ * @return Original byte array.
+ */
+ public byte[] getLineBytes() {
+ return lineBytes;
+ }
+
+ /**
+ * Get the given column's name.
+ * @param idx Column to lookup.
+ * @return Column's name.
+ */
+ public String getColumnName(int idx) {
+ return columnNames.get(idx);
+ }
+ }
+
+ /**
+ * Exception used when the CsvParser is unable to parse a line.
+ */
+ @SuppressWarnings("serial")
+ public static class BadCsvLineException extends Exception {
+ public BadCsvLineException(String err) {
+ super(err);
+ }
+ }
+}
[03/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/JarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/JarFinder.java b/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/JarFinder.java
deleted file mode 100644
index 57593db..0000000
--- a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/JarFinder.java
+++ /dev/null
@@ -1,179 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import com.google.common.base.Preconditions;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.text.MessageFormat;
-import java.util.Enumeration;
-import java.util.jar.JarFile;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-/**
- * Finds the Jar for a class. If the class is in a directory in the
- * classpath, it creates a Jar on the fly with the contents of the directory
- * and returns the path to that Jar. If a Jar is created, it is created in
- * the system temporary directory.
- *
- * This file was forked from hbase/branches/master@4ce6f48.
- */
-public class JarFinder {
-
- private static void copyToZipStream(File file, ZipEntry entry,
- ZipOutputStream zos) throws IOException {
- InputStream is = new FileInputStream(file);
- try {
- zos.putNextEntry(entry);
- byte[] arr = new byte[4096];
- int read = is.read(arr);
- while (read > -1) {
- zos.write(arr, 0, read);
- read = is.read(arr);
- }
- } finally {
- try {
- is.close();
- } finally {
- zos.closeEntry();
- }
- }
- }
-
- public static void jarDir(File dir, String relativePath, ZipOutputStream zos)
- throws IOException {
- Preconditions.checkNotNull(relativePath, "relativePath");
- Preconditions.checkNotNull(zos, "zos");
-
- // by JAR spec, if there is a manifest, it must be the first entry in the
- // ZIP.
- File manifestFile = new File(dir, JarFile.MANIFEST_NAME);
- ZipEntry manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
- if (!manifestFile.exists()) {
- zos.putNextEntry(manifestEntry);
- new Manifest().write(new BufferedOutputStream(zos));
- zos.closeEntry();
- } else {
- copyToZipStream(manifestFile, manifestEntry, zos);
- }
- zos.closeEntry();
- zipDir(dir, relativePath, zos, true);
- zos.close();
- }
-
- private static void zipDir(File dir, String relativePath, ZipOutputStream zos,
- boolean start) throws IOException {
- String[] dirList = dir.list();
- for (String aDirList : dirList) {
- File f = new File(dir, aDirList);
- if (!f.isHidden()) {
- if (f.isDirectory()) {
- if (!start) {
- ZipEntry dirEntry = new ZipEntry(relativePath + f.getName() + "/");
- zos.putNextEntry(dirEntry);
- zos.closeEntry();
- }
- String filePath = f.getPath();
- File file = new File(filePath);
- zipDir(file, relativePath + f.getName() + "/", zos, false);
- }
- else {
- String path = relativePath + f.getName();
- if (!path.equals(JarFile.MANIFEST_NAME)) {
- ZipEntry anEntry = new ZipEntry(path);
- copyToZipStream(f, anEntry, zos);
- }
- }
- }
- }
- }
-
- private static void createJar(File dir, File jarFile) throws IOException {
- Preconditions.checkNotNull(dir, "dir");
- Preconditions.checkNotNull(jarFile, "jarFile");
- File jarDir = jarFile.getParentFile();
- if (!jarDir.exists()) {
- if (!jarDir.mkdirs()) {
- throw new IOException(MessageFormat.format("could not create dir [{0}]",
- jarDir));
- }
- }
- JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile));
- jarDir(dir, "", zos);
- }
-
- /**
- * Returns the full path to the Jar containing the class. It always returns a
- * JAR.
- *
- * @param klass class.
- *
- * @return path to the Jar containing the class.
- */
- public static String getJar(Class klass) {
- Preconditions.checkNotNull(klass, "klass");
- ClassLoader loader = klass.getClassLoader();
- if (loader != null) {
- String class_file = klass.getName().replaceAll("\\.", "/") + ".class";
- try {
- for (Enumeration itr = loader.getResources(class_file);
- itr.hasMoreElements(); ) {
- URL url = (URL) itr.nextElement();
- String path = url.getPath();
- if (path.startsWith("file:")) {
- path = path.substring("file:".length());
- }
- path = URLDecoder.decode(path, "UTF-8");
- if ("jar".equals(url.getProtocol())) {
- path = URLDecoder.decode(path, "UTF-8");
- return path.replaceAll("!.*$", "");
- }
- else if ("file".equals(url.getProtocol())) {
- String klassName = klass.getName();
- klassName = klassName.replace(".", "/") + ".class";
- path = path.substring(0, path.length() - klassName.length());
- File baseDir = new File(path);
- File testDir = new File(System.getProperty("test.build.dir", "target/test-dir"));
- testDir = testDir.getAbsoluteFile();
- if (!testDir.exists()) {
- testDir.mkdirs();
- }
- File tempJar = File.createTempFile("hadoop-", "", testDir);
- tempJar = new File(tempJar.getAbsolutePath() + ".jar");
- tempJar.deleteOnExit();
- createJar(baseDir, tempJar);
- return tempJar.getAbsolutePath();
- }
- }
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableInputFormat.java b/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableInputFormat.java
deleted file mode 100644
index 25235cb..0000000
--- a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableInputFormat.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.mapreduce;
-
-import com.google.common.base.Objects;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import com.google.common.primitives.UnsignedBytes;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import javax.naming.NamingException;
-
-import org.apache.commons.net.util.Base64;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.net.DNS;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
-import org.kududb.client.Bytes;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduPredicate;
-import org.kududb.client.KuduScanner;
-import org.kududb.client.KuduTable;
-import org.kududb.client.LocatedTablet;
-import org.kududb.client.RowResult;
-import org.kududb.client.RowResultIterator;
-import org.kududb.client.KuduScanToken;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * This input format generates one split per tablet and the only location for each split is that
- * tablet's leader.
- * </p>
- *
- * <p>
- * Hadoop doesn't have the concept of "closing" the input format so in order to release the
- * resources we assume that once either {@link #getSplits(org.apache.hadoop.mapreduce.JobContext)}
- * or {@link KuduTableInputFormat.TableRecordReader#close()} have been called that
- * the object won't be used again and the AsyncKuduClient is shut down.
- * </p>
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduTableInputFormat extends InputFormat<NullWritable, RowResult>
- implements Configurable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KuduTableInputFormat.class);
-
- /** Job parameter that specifies the input table. */
- static final String INPUT_TABLE_KEY = "kudu.mapreduce.input.table";
-
- /** Job parameter that specifies if the scanner should cache blocks or not (default: false). */
- static final String SCAN_CACHE_BLOCKS = "kudu.mapreduce.input.scan.cache.blocks";
-
- /** Job parameter that specifies where the masters are. */
- static final String MASTER_ADDRESSES_KEY = "kudu.mapreduce.master.address";
-
- /** Job parameter that specifies how long we wait for operations to complete (default: 10s). */
- static final String OPERATION_TIMEOUT_MS_KEY = "kudu.mapreduce.operation.timeout.ms";
-
- /** Job parameter that specifies the address for the name server. */
- static final String NAME_SERVER_KEY = "kudu.mapreduce.name.server";
-
- /** Job parameter that specifies the encoded column predicates (may be empty). */
- static final String ENCODED_PREDICATES_KEY =
- "kudu.mapreduce.encoded.predicates";
-
- /**
- * Job parameter that specifies the column projection as a comma-separated list of column names.
- *
- * Not specifying this at all (i.e. setting to null) or setting to the special string
- * '*' means to project all columns.
- *
- * Specifying the empty string means to project no columns (i.e just count the rows).
- */
- static final String COLUMN_PROJECTION_KEY = "kudu.mapreduce.column.projection";
-
- /**
- * The reverse DNS lookup cache mapping: address from Kudu => hostname for Hadoop. This cache is
- * used in order to not do DNS lookups multiple times for each tablet server.
- */
- private final Map<String, String> reverseDNSCacheMap = new HashMap<>();
-
- private Configuration conf;
- private KuduClient client;
- private KuduTable table;
- private long operationTimeoutMs;
- private String nameServer;
- private boolean cacheBlocks;
- private List<String> projectedCols;
- private List<KuduPredicate> predicates;
-
- @Override
- public List<InputSplit> getSplits(JobContext jobContext)
- throws IOException, InterruptedException {
- try {
- if (table == null) {
- throw new IOException("No table was provided");
- }
-
- KuduScanToken.KuduScanTokenBuilder tokenBuilder = client.newScanTokenBuilder(table)
- .setProjectedColumnNames(projectedCols)
- .cacheBlocks(cacheBlocks)
- .setTimeout(operationTimeoutMs);
- for (KuduPredicate predicate : predicates) {
- tokenBuilder.addPredicate(predicate);
- }
- List<KuduScanToken> tokens = tokenBuilder.build();
-
- List<InputSplit> splits = new ArrayList<>(tokens.size());
- for (KuduScanToken token : tokens) {
- List<String> locations = new ArrayList<>(token.getTablet().getReplicas().size());
- for (LocatedTablet.Replica replica : token.getTablet().getReplicas()) {
- locations.add(reverseDNS(replica.getRpcHost(), replica.getRpcPort()));
- }
- splits.add(new TableSplit(token, locations.toArray(new String[locations.size()])));
- }
- return splits;
- } finally {
- shutdownClient();
- }
- }
-
- private void shutdownClient() throws IOException {
- try {
- client.shutdown();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- /**
- * This method might seem alien, but we do this in order to resolve the hostnames the same way
- * Hadoop does. This ensures we get locality if Kudu is running along MR/YARN.
- * @param host hostname we got from the master
- * @param port port we got from the master
- * @return reverse DNS'd address
- */
- private String reverseDNS(String host, Integer port) {
- String location = this.reverseDNSCacheMap.get(host);
- if (location != null) {
- return location;
- }
- // The below InetSocketAddress creation does a name resolution.
- InetSocketAddress isa = new InetSocketAddress(host, port);
- if (isa.isUnresolved()) {
- LOG.warn("Failed address resolve for: " + isa);
- }
- InetAddress tabletInetAddress = isa.getAddress();
- try {
- location = domainNamePointerToHostName(
- DNS.reverseDns(tabletInetAddress, this.nameServer));
- this.reverseDNSCacheMap.put(host, location);
- } catch (NamingException e) {
- LOG.warn("Cannot resolve the host name for " + tabletInetAddress + " because of " + e);
- location = host;
- }
- return location;
- }
-
- @Override
- public RecordReader<NullWritable, RowResult> createRecordReader(InputSplit inputSplit,
- TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
- return new TableRecordReader();
- }
-
- @Override
- public void setConf(Configuration entries) {
- this.conf = new Configuration(entries);
-
- String tableName = conf.get(INPUT_TABLE_KEY);
- String masterAddresses = conf.get(MASTER_ADDRESSES_KEY);
- this.operationTimeoutMs = conf.getLong(OPERATION_TIMEOUT_MS_KEY,
- AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS);
- this.nameServer = conf.get(NAME_SERVER_KEY);
- this.cacheBlocks = conf.getBoolean(SCAN_CACHE_BLOCKS, false);
-
- this.client = new KuduClient.KuduClientBuilder(masterAddresses)
- .defaultOperationTimeoutMs(operationTimeoutMs)
- .build();
- try {
- this.table = client.openTable(tableName);
- } catch (Exception ex) {
- throw new RuntimeException("Could not obtain the table from the master, " +
- "is the master running and is this table created? tablename=" + tableName + " and " +
- "master address= " + masterAddresses, ex);
- }
-
- String projectionConfig = conf.get(COLUMN_PROJECTION_KEY);
- if (projectionConfig == null || projectionConfig.equals("*")) {
- this.projectedCols = null; // project the whole table
- } else if ("".equals(projectionConfig)) {
- this.projectedCols = new ArrayList<>();
- } else {
- this.projectedCols = Lists.newArrayList(Splitter.on(',').split(projectionConfig));
-
- // Verify that the column names are valid -- better to fail with an exception
- // before we submit the job.
- Schema tableSchema = table.getSchema();
- for (String columnName : projectedCols) {
- if (tableSchema.getColumn(columnName) == null) {
- throw new IllegalArgumentException("Unknown column " + columnName);
- }
- }
- }
-
- this.predicates = new ArrayList<>();
- try {
- InputStream is =
- new ByteArrayInputStream(Base64.decodeBase64(conf.get(ENCODED_PREDICATES_KEY, "")));
- while (is.available() > 0) {
- this.predicates.add(KuduPredicate.fromPB(table.getSchema(),
- Common.ColumnPredicatePB.parseDelimitedFrom(is)));
- }
- } catch (IOException e) {
- throw new RuntimeException("unable to deserialize predicates from the configuration", e);
- }
- }
-
- /**
- * Given a PTR string generated via reverse DNS lookup, return everything
- * except the trailing period. Example for host.example.com., return
- * host.example.com
- * @param dnPtr a domain name pointer (PTR) string.
- * @return Sanitized hostname with last period stripped off.
- *
- */
- private static String domainNamePointerToHostName(String dnPtr) {
- if (dnPtr == null)
- return null;
- return dnPtr.endsWith(".") ? dnPtr.substring(0, dnPtr.length() - 1) : dnPtr;
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- static class TableSplit extends InputSplit implements Writable, Comparable<TableSplit> {
-
- /** The scan token that the split will use to scan the Kudu table. */
- private byte[] scanToken;
-
- /** The start partition key of the scan. Used for sorting splits. */
- private byte[] partitionKey;
-
- /** Tablet server locations which host the tablet to be scanned. */
- private String[] locations;
-
- public TableSplit() { } // Writable
-
- public TableSplit(KuduScanToken token, String[] locations) throws IOException {
- this.scanToken = token.serialize();
- this.partitionKey = token.getTablet().getPartition().getPartitionKeyStart();
- this.locations = locations;
- }
-
- public byte[] getScanToken() {
- return scanToken;
- }
-
- public byte[] getPartitionKey() {
- return partitionKey;
- }
-
- @Override
- public long getLength() throws IOException, InterruptedException {
- // TODO Guesstimate a size
- return 0;
- }
-
- @Override
- public String[] getLocations() throws IOException, InterruptedException {
- return locations;
- }
-
- @Override
- public int compareTo(TableSplit other) {
- return UnsignedBytes.lexicographicalComparator().compare(partitionKey, other.partitionKey);
- }
-
- @Override
- public void write(DataOutput dataOutput) throws IOException {
- Bytes.writeByteArray(dataOutput, scanToken);
- Bytes.writeByteArray(dataOutput, partitionKey);
- dataOutput.writeInt(locations.length);
- for (String location : locations) {
- byte[] str = Bytes.fromString(location);
- Bytes.writeByteArray(dataOutput, str);
- }
- }
-
- @Override
- public void readFields(DataInput dataInput) throws IOException {
- scanToken = Bytes.readByteArray(dataInput);
- partitionKey = Bytes.readByteArray(dataInput);
- locations = new String[dataInput.readInt()];
- for (int i = 0; i < locations.length; i++) {
- byte[] str = Bytes.readByteArray(dataInput);
- locations[i] = Bytes.getString(str);
- }
- }
-
- @Override
- public int hashCode() {
- // We currently just care about the partition key since we're within the same table.
- return Arrays.hashCode(partitionKey);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TableSplit that = (TableSplit) o;
-
- return this.compareTo(that) == 0;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("partitionKey", Bytes.pretty(partitionKey))
- .add("locations", Arrays.toString(locations))
- .toString();
- }
- }
-
- class TableRecordReader extends RecordReader<NullWritable, RowResult> {
-
- private final NullWritable currentKey = NullWritable.get();
- private RowResult currentValue;
- private RowResultIterator iterator;
- private KuduScanner scanner;
- private TableSplit split;
-
- @Override
- public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
- if (!(inputSplit instanceof TableSplit)) {
- throw new IllegalArgumentException("TableSplit is the only accepted input split");
- }
-
- split = (TableSplit) inputSplit;
-
- try {
- scanner = KuduScanToken.deserializeIntoScanner(split.getScanToken(), client);
- } catch (Exception e) {
- throw new IOException(e);
- }
-
- // Calling this now to set iterator.
- tryRefreshIterator();
- }
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- if (!iterator.hasNext()) {
- tryRefreshIterator();
- if (!iterator.hasNext()) {
- // Means we still have the same iterator, we're done
- return false;
- }
- }
- currentValue = iterator.next();
- return true;
- }
-
- /**
- * If the scanner has more rows, get a new iterator else don't do anything.
- * @throws IOException
- */
- private void tryRefreshIterator() throws IOException {
- if (!scanner.hasMoreRows()) {
- return;
- }
- try {
- iterator = scanner.nextRows();
- } catch (Exception e) {
- throw new IOException("Couldn't get scan data", e);
- }
- }
-
- @Override
- public NullWritable getCurrentKey() throws IOException, InterruptedException {
- return currentKey;
- }
-
- @Override
- public RowResult getCurrentValue() throws IOException, InterruptedException {
- return currentValue;
- }
-
- @Override
- public float getProgress() throws IOException, InterruptedException {
- // TODO Guesstimate progress
- return 0;
- }
-
- @Override
- public void close() throws IOException {
- try {
- scanner.close();
- } catch (Exception e) {
- throw new IOException(e);
- }
- shutdownClient();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableMapReduceUtil.java b/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableMapReduceUtil.java
deleted file mode 100644
index 0b919d9..0000000
--- a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableMapReduceUtil.java
+++ /dev/null
@@ -1,541 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.mapreduce;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.net.util.Base64;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.TaskInputOutputContext;
-import org.apache.hadoop.util.StringUtils;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
-import org.kududb.client.ColumnRangePredicate;
-import org.kududb.client.KuduPredicate;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
-
-/**
- * Utility class to setup MR jobs that use Kudu as an input and/or output.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduTableMapReduceUtil {
- // Mostly lifted from HBase's TableMapReduceUtil
-
- private static final Log LOG = LogFactory.getLog(KuduTableMapReduceUtil.class);
-
- /**
- * Doesn't need instantiation
- */
- private KuduTableMapReduceUtil() { }
-
-
- /**
- * Base class for MR I/O formats, contains the common configurations.
- */
- private static abstract class AbstractMapReduceConfigurator<S> {
- protected final Job job;
- protected final String table;
-
- protected boolean addDependencies = true;
-
- /**
- * Constructor for the required fields to configure.
- * @param job a job to configure
- * @param table a string that contains the name of the table to read from
- */
- private AbstractMapReduceConfigurator(Job job, String table) {
- this.job = job;
- this.table = table;
- }
-
- /**
- * Sets whether this job should add Kudu's dependencies to the distributed cache. Turned on
- * by default.
- * @param addDependencies a boolean that says if we should add the dependencies
- * @return this instance
- */
- @SuppressWarnings("unchecked")
- public S addDependencies(boolean addDependencies) {
- this.addDependencies = addDependencies;
- return (S) this;
- }
-
- /**
- * Configures the job using the passed parameters.
- * @throws IOException If addDependencies is enabled and a problem is encountered reading
- * files on the filesystem
- */
- public abstract void configure() throws IOException;
- }
-
- /**
- * Builder-like class that sets up the required configurations and classes to write to Kudu.
- * <p>
- * Use either child classes when configuring the table output format.
- */
- private static abstract class AbstractTableOutputFormatConfigurator
- <S extends AbstractTableOutputFormatConfigurator<? super S>>
- extends AbstractMapReduceConfigurator<S> {
-
- protected String masterAddresses;
- protected long operationTimeoutMs = AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
-
- /**
- * {@inheritDoc}
- */
- private AbstractTableOutputFormatConfigurator(Job job, String table) {
- super(job, table);
- }
-
- /**
- * {@inheritDoc}
- */
- public void configure() throws IOException {
- job.setOutputFormatClass(KuduTableOutputFormat.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(Operation.class);
-
- Configuration conf = job.getConfiguration();
- conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, masterAddresses);
- conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, table);
- conf.setLong(KuduTableOutputFormat.OPERATION_TIMEOUT_MS_KEY, operationTimeoutMs);
- if (addDependencies) {
- addDependencyJars(job);
- }
- }
- }
-
- /**
- * Builder-like class that sets up the required configurations and classes to read from Kudu.
- * By default, block caching is disabled.
- * <p>
- * Use either child classes when configuring the table input format.
- */
- private static abstract class AbstractTableInputFormatConfigurator
- <S extends AbstractTableInputFormatConfigurator<? super S>>
- extends AbstractMapReduceConfigurator<S> {
-
- protected String masterAddresses;
- protected long operationTimeoutMs = AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
- protected final String columnProjection;
- protected boolean cacheBlocks;
- protected List<KuduPredicate> predicates = new ArrayList<>();
-
- /**
- * Constructor for the required fields to configure.
- * @param job a job to configure
- * @param table a string that contains the name of the table to read from
- * @param columnProjection a string containing a comma-separated list of columns to read.
- * It can be null in which case we read empty rows
- */
- private AbstractTableInputFormatConfigurator(Job job, String table, String columnProjection) {
- super(job, table);
- this.columnProjection = columnProjection;
- }
-
- /**
- * Sets the block caching configuration for the scanners. Turned off by default.
- * @param cacheBlocks whether the job should use scanners that cache blocks.
- * @return this instance
- */
- public S cacheBlocks(boolean cacheBlocks) {
- this.cacheBlocks = cacheBlocks;
- return (S) this;
- }
-
- /**
- * Configures the job with all the passed parameters.
- * @throws IOException If addDependencies is enabled and a problem is encountered reading
- * files on the filesystem
- */
- public void configure() throws IOException {
- job.setInputFormatClass(KuduTableInputFormat.class);
-
- Configuration conf = job.getConfiguration();
-
- conf.set(KuduTableInputFormat.MASTER_ADDRESSES_KEY, masterAddresses);
- conf.set(KuduTableInputFormat.INPUT_TABLE_KEY, table);
- conf.setLong(KuduTableInputFormat.OPERATION_TIMEOUT_MS_KEY, operationTimeoutMs);
- conf.setBoolean(KuduTableInputFormat.SCAN_CACHE_BLOCKS, cacheBlocks);
-
- if (columnProjection != null) {
- conf.set(KuduTableInputFormat.COLUMN_PROJECTION_KEY, columnProjection);
- }
-
- conf.set(KuduTableInputFormat.ENCODED_PREDICATES_KEY, base64EncodePredicates(predicates));
-
- if (addDependencies) {
- addDependencyJars(job);
- }
- }
- }
-
- /**
- * Returns the provided predicates as a Base64 encoded string.
- * @param predicates the predicates to encode
- * @return the encoded predicates
- */
- static String base64EncodePredicates(List<KuduPredicate> predicates) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- for (KuduPredicate predicate : predicates) {
- predicate.toPB().writeDelimitedTo(baos);
- }
- return Base64.encodeBase64String(baos.toByteArray());
- }
-
-
- /**
- * Table output format configurator to use to specify the parameters directly.
- */
- public static class TableOutputFormatConfigurator
- extends AbstractTableOutputFormatConfigurator<TableOutputFormatConfigurator> {
-
- /**
- * Constructor for the required fields to configure.
- * @param job a job to configure
- * @param table a string that contains the name of the table to read from
- * @param masterAddresses a comma-separated list of masters' hosts and ports
- */
- public TableOutputFormatConfigurator(Job job, String table, String masterAddresses) {
- super(job, table);
- this.masterAddresses = masterAddresses;
- }
-
- /**
- * Sets the timeout for all the operations. The default is 10 seconds.
- * @param operationTimeoutMs a long that represents the timeout for operations to complete,
- * must be a positive value or 0
- * @return this instance
- * @throws IllegalArgumentException if the operation timeout is lower than 0
- */
- public TableOutputFormatConfigurator operationTimeoutMs(long operationTimeoutMs) {
- if (operationTimeoutMs < 0) {
- throw new IllegalArgumentException("The operation timeout must be => 0, " +
- "passed value is: " + operationTimeoutMs);
- }
- this.operationTimeoutMs = operationTimeoutMs;
- return this;
- }
- }
-
- /**
- * Table output format that uses a {@link CommandLineParser} in order to set the
- * master config and the operation timeout.
- */
- public static class TableOutputFormatConfiguratorWithCommandLineParser extends
- AbstractTableOutputFormatConfigurator<TableOutputFormatConfiguratorWithCommandLineParser> {
-
- /**
- * {@inheritDoc}
- */
- public TableOutputFormatConfiguratorWithCommandLineParser(Job job, String table) {
- super(job, table);
- CommandLineParser parser = new CommandLineParser(job.getConfiguration());
- this.masterAddresses = parser.getMasterAddresses();
- this.operationTimeoutMs = parser.getOperationTimeoutMs();
- }
- }
-
- /**
- * Table input format configurator to use to specify the parameters directly.
- */
- public static class TableInputFormatConfigurator
- extends AbstractTableInputFormatConfigurator<TableInputFormatConfigurator> {
-
- /**
- * Constructor for the required fields to configure.
- * @param job a job to configure
- * @param table a string that contains the name of the table to read from
- * @param columnProjection a string containing a comma-separated list of columns to read.
- * It can be null in which case we read empty rows
- * @param masterAddresses a comma-separated list of masters' hosts and ports
- */
- public TableInputFormatConfigurator(Job job, String table, String columnProjection,
- String masterAddresses) {
- super(job, table, columnProjection);
- this.masterAddresses = masterAddresses;
- }
-
- /**
- * Sets the timeout for all the operations. The default is 10 seconds.
- * @param operationTimeoutMs a long that represents the timeout for operations to complete,
- * must be a positive value or 0
- * @return this instance
- * @throws IllegalArgumentException if the operation timeout is lower than 0
- */
- public TableInputFormatConfigurator operationTimeoutMs(long operationTimeoutMs) {
- if (operationTimeoutMs < 0) {
- throw new IllegalArgumentException("The operation timeout must be => 0, " +
- "passed value is: " + operationTimeoutMs);
- }
- this.operationTimeoutMs = operationTimeoutMs;
- return this;
- }
-
- /**
- * Adds a new predicate that will be pushed down to all the tablets.
- * @param predicate a predicate to add
- * @return this instance
- * @deprecated use {@link #addPredicate}
- */
- @Deprecated
- public TableInputFormatConfigurator addColumnRangePredicate(ColumnRangePredicate predicate) {
- return addPredicate(predicate.toKuduPredicate());
- }
-
- /**
- * Adds a new predicate that will be pushed down to all the tablets.
- * @param predicate a predicate to add
- * @return this instance
- */
- public TableInputFormatConfigurator addPredicate(KuduPredicate predicate) {
- this.predicates.add(predicate);
- return this;
- }
- }
-
- /**
- * Table input format that uses a {@link CommandLineParser} in order to set the
- * master config and the operation timeout.
- * This version cannot set column range predicates.
- */
- public static class TableInputFormatConfiguratorWithCommandLineParser extends
- AbstractTableInputFormatConfigurator<TableInputFormatConfiguratorWithCommandLineParser> {
-
- /**
- * {@inheritDoc}
- */
- public TableInputFormatConfiguratorWithCommandLineParser(Job job,
- String table,
- String columnProjection) {
- super(job, table, columnProjection);
- CommandLineParser parser = new CommandLineParser(job.getConfiguration());
- this.masterAddresses = parser.getMasterAddresses();
- this.operationTimeoutMs = parser.getOperationTimeoutMs();
- }
- }
-
- /**
- * Use this method when setting up a task to get access to the KuduTable in order to create
- * Inserts, Updates, and Deletes.
- * @param context Map context
- * @return The kudu table object as setup by the output format
- */
- @SuppressWarnings("rawtypes")
- public static KuduTable getTableFromContext(TaskInputOutputContext context) {
- String multitonKey = context.getConfiguration().get(KuduTableOutputFormat.MULTITON_KEY);
- return KuduTableOutputFormat.getKuduTable(multitonKey);
- }
-
- /**
- * Add the Kudu dependency jars as well as jars for any of the configured
- * job classes to the job configuration, so that JobClient will ship them
- * to the cluster and add them to the DistributedCache.
- */
- public static void addDependencyJars(Job job) throws IOException {
- addKuduDependencyJars(job.getConfiguration());
- try {
- addDependencyJars(job.getConfiguration(),
- // when making changes here, consider also mapred.TableMapReduceUtil
- // pull job classes
- job.getMapOutputKeyClass(),
- job.getMapOutputValueClass(),
- job.getInputFormatClass(),
- job.getOutputKeyClass(),
- job.getOutputValueClass(),
- job.getOutputFormatClass(),
- job.getPartitionerClass(),
- job.getCombinerClass());
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Add the jars containing the given classes to the job's configuration
- * such that JobClient will ship them to the cluster and add them to
- * the DistributedCache.
- */
- public static void addDependencyJars(Configuration conf,
- Class<?>... classes) throws IOException {
-
- FileSystem localFs = FileSystem.getLocal(conf);
- Set<String> jars = new HashSet<String>();
- // Add jars that are already in the tmpjars variable
- jars.addAll(conf.getStringCollection("tmpjars"));
-
- // add jars as we find them to a map of contents jar name so that we can avoid
- // creating new jars for classes that have already been packaged.
- Map<String, String> packagedClasses = new HashMap<String, String>();
-
- // Add jars containing the specified classes
- for (Class<?> clazz : classes) {
- if (clazz == null) continue;
-
- Path path = findOrCreateJar(clazz, localFs, packagedClasses);
- if (path == null) {
- LOG.warn("Could not find jar for class " + clazz +
- " in order to ship it to the cluster.");
- continue;
- }
- if (!localFs.exists(path)) {
- LOG.warn("Could not validate jar file " + path + " for class "
- + clazz);
- continue;
- }
- jars.add(path.toString());
- }
- if (jars.isEmpty()) return;
-
- conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
- }
-
- /**
- * Add Kudu and its dependencies (only) to the job configuration.
- * <p>
- * This is intended as a low-level API, facilitating code reuse between this
- * class and its mapred counterpart. It also of use to external tools that
- * need to build a MapReduce job that interacts with Kudu but want
- * fine-grained control over the jars shipped to the cluster.
- * </p>
- * @param conf The Configuration object to extend with dependencies.
- * @see KuduTableMapReduceUtil
- * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
- */
- public static void addKuduDependencyJars(Configuration conf) throws IOException {
- addDependencyJars(conf,
- // explicitly pull a class from each module
- Operation.class, // kudu-client
- KuduTableMapReduceUtil.class, // kudu-mapreduce
- // pull necessary dependencies
- com.stumbleupon.async.Deferred.class);
- }
-
- /**
- * Finds the Jar for a class or creates it if it doesn't exist. If the class
- * is in a directory in the classpath, it creates a Jar on the fly with the
- * contents of the directory and returns the path to that Jar. If a Jar is
- * created, it is created in the system temporary directory. Otherwise,
- * returns an existing jar that contains a class of the same name. Maintains
- * a mapping from jar contents to the tmp jar created.
- * @param my_class the class to find.
- * @param fs the FileSystem with which to qualify the returned path.
- * @param packagedClasses a map of class name to path.
- * @return a jar file that contains the class.
- * @throws IOException
- */
- @SuppressWarnings("deprecation")
- private static Path findOrCreateJar(Class<?> my_class, FileSystem fs,
- Map<String, String> packagedClasses)
- throws IOException {
- // attempt to locate an existing jar for the class.
- String jar = findContainingJar(my_class, packagedClasses);
- if (null == jar || jar.isEmpty()) {
- jar = JarFinder.getJar(my_class);
- updateMap(jar, packagedClasses);
- }
-
- if (null == jar || jar.isEmpty()) {
- return null;
- }
-
- LOG.debug(String.format("For class %s, using jar %s", my_class.getName(), jar));
- return new Path(jar).makeQualified(fs);
- }
-
- /**
- * Find a jar that contains a class of the same name, if any. It will return
- * a jar file, even if that is not the first thing on the class path that
- * has a class with the same name. Looks first on the classpath and then in
- * the <code>packagedClasses</code> map.
- * @param my_class the class to find.
- * @return a jar file that contains the class, or null.
- * @throws IOException
- */
- private static String findContainingJar(Class<?> my_class, Map<String, String> packagedClasses)
- throws IOException {
- ClassLoader loader = my_class.getClassLoader();
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
-
- // first search the classpath
- for (Enumeration<URL> itr = loader.getResources(class_file); itr.hasMoreElements();) {
- URL url = itr.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- // URLDecoder is a misnamed class, since it actually decodes
- // x-www-form-urlencoded MIME type rather than actual
- // URL encoding (which the file path has). Therefore it would
- // decode +s to ' 's which is incorrect (spaces are actually
- // either unencoded or encoded as "%20"). Replace +s first, so
- // that they are kept sacred during the decoding process.
- toReturn = toReturn.replaceAll("\\+", "%2B");
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- return toReturn.replaceAll("!.*$", "");
- }
- }
-
- // now look in any jars we've packaged using JarFinder. Returns null when
- // no jar is found.
- return packagedClasses.get(class_file);
- }
-
- /**
- * Add entries to <code>packagedClasses</code> corresponding to class files
- * contained in <code>jar</code>.
- * @param jar The jar who's content to list.
- * @param packagedClasses map[class -> jar]
- */
- private static void updateMap(String jar, Map<String, String> packagedClasses) throws IOException {
- if (null == jar || jar.isEmpty()) {
- return;
- }
- ZipFile zip = null;
- try {
- zip = new ZipFile(jar);
- for (Enumeration<? extends ZipEntry> iter = zip.entries(); iter.hasMoreElements();) {
- ZipEntry entry = iter.nextElement();
- if (entry.getName().endsWith("class")) {
- packagedClasses.put(entry.getName(), jar);
- }
- }
- } finally {
- if (null != zip) zip.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputCommitter.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputCommitter.java b/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputCommitter.java
deleted file mode 100644
index 8af750b..0000000
--- a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputCommitter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.io.IOException;
-
-/**
- * Small committer class that does not do anything.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduTableOutputCommitter extends OutputCommitter {
- @Override
- public void setupJob(JobContext jobContext) throws IOException {
-
- }
-
- @Override
- public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException {
-
- }
-
- @Override
- public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException {
- return false;
- }
-
- @Override
- public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException {
-
- }
-
- @Override
- public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputFormat.java b/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputFormat.java
deleted file mode 100644
index e80b73f..0000000
--- a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/KuduTableOutputFormat.java
+++ /dev/null
@@ -1,215 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.*;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * <p>
- * Use {@link
- * KuduTableMapReduceUtil.TableOutputFormatConfigurator}
- * to correctly setup this output format, then {@link
- * KuduTableMapReduceUtil#getTableFromContext(org.apache.hadoop.mapreduce.TaskInputOutputContext)}
- * to get a KuduTable.
- * </p>
- *
- * <p>
- * Hadoop doesn't have the concept of "closing" the output format so in order to release the
- * resources we assume that once either
- * {@link #checkOutputSpecs(org.apache.hadoop.mapreduce.JobContext)}
- * or {@link TableRecordWriter#close(org.apache.hadoop.mapreduce.TaskAttemptContext)}
- * have been called that the object won't be used again and the KuduClient is shut down.
- * </p>
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduTableOutputFormat extends OutputFormat<NullWritable,Operation>
- implements Configurable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KuduTableOutputFormat.class);
-
- /** Job parameter that specifies the output table. */
- static final String OUTPUT_TABLE_KEY = "kudu.mapreduce.output.table";
-
- /** Job parameter that specifies where the masters are */
- static final String MASTER_ADDRESSES_KEY = "kudu.mapreduce.master.addresses";
-
- /** Job parameter that specifies how long we wait for operations to complete */
- static final String OPERATION_TIMEOUT_MS_KEY = "kudu.mapreduce.operation.timeout.ms";
-
- /** Number of rows that are buffered before flushing to the tablet server */
- static final String BUFFER_ROW_COUNT_KEY = "kudu.mapreduce.buffer.row.count";
-
- /**
- * Job parameter that specifies which key is to be used to reach the KuduTableOutputFormat
- * belonging to the caller
- */
- static final String MULTITON_KEY = "kudu.mapreduce.multitonkey";
-
- /**
- * This multiton is used so that the tasks using this output format/record writer can find
- * their KuduTable without having a direct dependency on this class,
- * with the additional complexity that the output format cannot be shared between threads.
- */
- private static final ConcurrentHashMap<String, KuduTableOutputFormat> MULTITON = new
- ConcurrentHashMap<String, KuduTableOutputFormat>();
-
- /**
- * This counter helps indicate which task log to look at since rows that weren't applied will
- * increment this counter.
- */
- public enum Counters { ROWS_WITH_ERRORS }
-
- private Configuration conf = null;
-
- private KuduClient client;
- private KuduTable table;
- private KuduSession session;
- private long operationTimeoutMs;
-
- @Override
- public void setConf(Configuration entries) {
- this.conf = new Configuration(entries);
-
- String masterAddress = this.conf.get(MASTER_ADDRESSES_KEY);
- String tableName = this.conf.get(OUTPUT_TABLE_KEY);
- this.operationTimeoutMs = this.conf.getLong(OPERATION_TIMEOUT_MS_KEY,
- AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS);
- int bufferSpace = this.conf.getInt(BUFFER_ROW_COUNT_KEY, 1000);
-
- this.client = new KuduClient.KuduClientBuilder(masterAddress)
- .defaultOperationTimeoutMs(operationTimeoutMs)
- .build();
- try {
- this.table = client.openTable(tableName);
- } catch (Exception ex) {
- throw new RuntimeException("Could not obtain the table from the master, " +
- "is the master running and is this table created? tablename=" + tableName + " and " +
- "master address= " + masterAddress, ex);
- }
- this.session = client.newSession();
- this.session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
- this.session.setMutationBufferSpace(bufferSpace);
- this.session.setIgnoreAllDuplicateRows(true);
- String multitonKey = String.valueOf(Thread.currentThread().getId());
- assert(MULTITON.get(multitonKey) == null);
- MULTITON.put(multitonKey, this);
- entries.set(MULTITON_KEY, multitonKey);
- }
-
- private void shutdownClient() throws IOException {
- try {
- client.shutdown();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- public static KuduTable getKuduTable(String multitonKey) {
- return MULTITON.get(multitonKey).getKuduTable();
- }
-
- private KuduTable getKuduTable() {
- return this.table;
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public RecordWriter<NullWritable, Operation> getRecordWriter(TaskAttemptContext taskAttemptContext)
- throws IOException, InterruptedException {
- return new TableRecordWriter(this.session);
- }
-
- @Override
- public void checkOutputSpecs(JobContext jobContext) throws IOException, InterruptedException {
- shutdownClient();
- }
-
- @Override
- public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) throws
- IOException, InterruptedException {
- return new KuduTableOutputCommitter();
- }
-
- protected class TableRecordWriter extends RecordWriter<NullWritable, Operation> {
-
- private final AtomicLong rowsWithErrors = new AtomicLong();
- private final KuduSession session;
-
- public TableRecordWriter(KuduSession session) {
- this.session = session;
- }
-
- @Override
- public void write(NullWritable key, Operation operation)
- throws IOException, InterruptedException {
- try {
- session.apply(operation);
- } catch (Exception e) {
- throw new IOException("Encountered an error while writing", e);
- }
- }
-
- @Override
- public void close(TaskAttemptContext taskAttemptContext) throws IOException,
- InterruptedException {
- try {
- processRowErrors(session.close());
- shutdownClient();
- } catch (Exception e) {
- throw new IOException("Encountered an error while closing this task", e);
- } finally {
- if (taskAttemptContext != null) {
- // This is the only place where we have access to the context in the record writer,
- // so set the counter here.
- taskAttemptContext.getCounter(Counters.ROWS_WITH_ERRORS).setValue(rowsWithErrors.get());
- }
- }
- }
-
- private void processRowErrors(List<OperationResponse> responses) {
- List<RowError> errors = OperationResponse.collectErrors(responses);
- if (!errors.isEmpty()) {
- int rowErrorsCount = errors.size();
- rowsWithErrors.addAndGet(rowErrorsCount);
- LOG.warn("Got per errors for " + rowErrorsCount + " rows, " +
- "the first one being " + errors.get(0).getStatus());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/TableReducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/TableReducer.java b/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/TableReducer.java
deleted file mode 100644
index 7cf3ada..0000000
--- a/java/kudu-mapreduce/src/main/java/org/kududb/mapreduce/TableReducer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Operation;
-import org.apache.hadoop.mapreduce.Reducer;
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public abstract class TableReducer<KEYIN, VALUEIN, KEYOUT>
- extends Reducer<KEYIN, VALUEIN, KEYOUT, Operation> {
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
new file mode 100644
index 0000000..1e2cb41
--- /dev/null
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/HadoopTestingUtility.java
@@ -0,0 +1,101 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kududb.mapreduce;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * This class is analog to HBaseTestingUtility except that we only need it for the MR tests.
+ */
+public class HadoopTestingUtility {
+
+ private static final Log LOG = LogFactory.getLog(HadoopTestingUtility.class);
+
+ private File testDir;
+
+ private Configuration conf = new Configuration();
+
+ /**
+ * System property key to get base test directory value
+ */
+ public static final String BASE_TEST_DIRECTORY_KEY =
+ "test.build.data.basedirectory";
+
+ /**
+ * Default base directory for test output.
+ */
+ private static final String DEFAULT_BASE_TEST_DIRECTORY = "target/mr-data";
+
+ public Configuration getConfiguration() {
+ return this.conf;
+ }
+
+ /**
+ * Sets up a temporary directory for the test to run in. Call cleanup() at the end of your
+ * tests to remove it.
+ * @param testName Will be used to build a part of the directory name for the test
+ * @return Where the test is homed
+ */
+ public File setupAndGetTestDir(String testName, Configuration conf) {
+ if (this.testDir != null) {
+ return this.testDir;
+ }
+ Path testPath = new Path(getBaseTestDir(), testName + System.currentTimeMillis());
+ this.testDir = new File(testPath.toString()).getAbsoluteFile();
+ this.testDir.mkdirs();
+ // Set this property so when mapreduce jobs run, they will use this as their home dir.
+ System.setProperty("test.build.dir", this.testDir.toString());
+ System.setProperty("hadoop.home.dir", this.testDir.toString());
+ conf.set("hadoop.tmp.dir", this.testDir.toString() + "/mapred");
+
+ LOG.info("Test configured to write to " + this.testDir);
+ return this.testDir;
+ }
+
+ private Path getBaseTestDir() {
+ String pathName = System.getProperty(BASE_TEST_DIRECTORY_KEY, DEFAULT_BASE_TEST_DIRECTORY);
+ return new Path(pathName);
+ }
+
+ public void cleanup() throws IOException {
+ FileSystem.closeAll();
+ if (this.testDir != null) {
+ delete(this.testDir);
+ }
+ }
+
+ private void delete(File dir) throws IOException {
+ if (dir == null || !dir.exists()) {
+ return;
+ }
+ try {
+ FileUtils.deleteDirectory(dir);
+ } catch (IOException ex) {
+ LOG.warn("Failed to delete " + dir.getAbsolutePath());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
new file mode 100644
index 0000000..3d04043
--- /dev/null
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kududb.client.BaseKuduTest;
+import org.kududb.client.KuduPredicate;
+import org.kududb.client.RowResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class ITInputFormatJob extends BaseKuduTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ITInputFormatJob.class);
+
+ private static final String TABLE_NAME =
+ ITInputFormatJob.class.getName() + "-" + System.currentTimeMillis();
+
+ private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
+
+ /** Counter enumeration to count the actual rows. */
+ private static enum Counters { ROWS }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ try {
+ BaseKuduTest.tearDownAfterClass();
+ } finally {
+ HADOOP_UTIL.cleanup();
+ }
+ }
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void test() throws Exception {
+
+ createFourTabletsTableWithNineRows(TABLE_NAME);
+
+ Configuration conf = new Configuration();
+ HADOOP_UTIL.setupAndGetTestDir(ITInputFormatJob.class.getName(), conf).getAbsolutePath();
+
+ createAndTestJob(conf, new ArrayList<KuduPredicate>(), 9);
+
+ KuduPredicate pred1 = KuduPredicate.newComparisonPredicate(
+ basicSchema.getColumnByIndex(0), KuduPredicate.ComparisonOp.GREATER_EQUAL, 20);
+ createAndTestJob(conf, Lists.newArrayList(pred1), 6);
+
+ KuduPredicate pred2 = KuduPredicate.newComparisonPredicate(
+ basicSchema.getColumnByIndex(2), KuduPredicate.ComparisonOp.LESS_EQUAL, 1);
+ createAndTestJob(conf, Lists.newArrayList(pred1, pred2), 2);
+ }
+
+ private void createAndTestJob(Configuration conf,
+ List<KuduPredicate> predicates, int expectedCount)
+ throws Exception {
+ String jobName = ITInputFormatJob.class.getName();
+ Job job = new Job(conf, jobName);
+
+ Class<TestMapperTableInput> mapperClass = TestMapperTableInput.class;
+ job.setJarByClass(mapperClass);
+ job.setMapperClass(mapperClass);
+ job.setNumReduceTasks(0);
+ job.setOutputFormatClass(NullOutputFormat.class);
+ KuduTableMapReduceUtil.TableInputFormatConfigurator configurator =
+ new KuduTableMapReduceUtil.TableInputFormatConfigurator(
+ job,
+ TABLE_NAME,
+ "*",
+ getMasterAddresses())
+ .operationTimeoutMs(DEFAULT_SLEEP)
+ .addDependencies(false)
+ .cacheBlocks(false);
+ for (KuduPredicate predicate : predicates) {
+ configurator.addPredicate(predicate);
+ }
+ configurator.configure();
+
+ assertTrue("Test job did not end properly", job.waitForCompletion(true));
+
+ assertEquals(expectedCount, job.getCounters().findCounter(Counters.ROWS).getValue());
+ }
+
+ /**
+ * Simple row counter and printer
+ */
+ static class TestMapperTableInput extends
+ Mapper<NullWritable, RowResult, NullWritable, NullWritable> {
+
+ @Override
+ protected void map(NullWritable key, RowResult value, Context context) throws IOException,
+ InterruptedException {
+ context.getCounter(Counters.ROWS).increment(1);
+ LOG.info(value.toStringLongFormat()); // useful to visual debugging
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
new file mode 100644
index 0000000..ff4d81a
--- /dev/null
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.kududb.Schema;
+import org.kududb.client.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class ITKuduTableInputFormat extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ ITKuduTableInputFormat.class.getName() + "-" + System.currentTimeMillis();
+
+ @Test
+ public void test() throws Exception {
+ createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
+
+ KuduTable table = openTable(TABLE_NAME);
+ Schema schema = getBasicSchema();
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addInt(0, 1);
+ row.addInt(1, 2);
+ row.addInt(2, 3);
+ row.addString(3, "a string");
+ row.addBoolean(4, true);
+ AsyncKuduSession session = client.newSession();
+ session.apply(insert).join(DEFAULT_SLEEP);
+ session.close().join(DEFAULT_SLEEP);
+
+ // Test getting all the columns back
+ RecordReader<NullWritable, RowResult> reader = createRecordReader("*", null);
+ assertTrue(reader.nextKeyValue());
+ assertEquals(5, reader.getCurrentValue().getColumnProjection().getColumnCount());
+ assertFalse(reader.nextKeyValue());
+
+ // Test getting two columns back
+ reader = createRecordReader(schema.getColumnByIndex(3).getName() + "," +
+ schema.getColumnByIndex(2).getName(), null);
+ assertTrue(reader.nextKeyValue());
+ assertEquals(2, reader.getCurrentValue().getColumnProjection().getColumnCount());
+ assertEquals("a string", reader.getCurrentValue().getString(0));
+ assertEquals(3, reader.getCurrentValue().getInt(1));
+ try {
+ reader.getCurrentValue().getString(2);
+ fail("Should only be getting 2 columns back");
+ } catch (IndexOutOfBoundsException e) {
+ // expected
+ }
+
+ // Test getting one column back
+ reader = createRecordReader(schema.getColumnByIndex(1).getName(), null);
+ assertTrue(reader.nextKeyValue());
+ assertEquals(1, reader.getCurrentValue().getColumnProjection().getColumnCount());
+ assertEquals(2, reader.getCurrentValue().getInt(0));
+ try {
+ reader.getCurrentValue().getString(1);
+ fail("Should only be getting 1 column back");
+ } catch (IndexOutOfBoundsException e) {
+ // expected
+ }
+
+ // Test getting empty rows back
+ reader = createRecordReader("", null);
+ assertTrue(reader.nextKeyValue());
+ assertEquals(0, reader.getCurrentValue().getColumnProjection().getColumnCount());
+ assertFalse(reader.nextKeyValue());
+
+ // Test getting an unknown table, will not work
+ try {
+ createRecordReader("unknown", null);
+ fail("Should not be able to scan a column that doesn't exist");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ // Test using a predicate that filters the row out.
+ KuduPredicate pred1 = KuduPredicate.newComparisonPredicate(
+ schema.getColumnByIndex(1), KuduPredicate.ComparisonOp.GREATER_EQUAL, 3);
+ reader = createRecordReader("*", Lists.newArrayList(pred1));
+ assertFalse(reader.nextKeyValue());
+ }
+
+ private RecordReader<NullWritable, RowResult> createRecordReader(String columnProjection,
+ List<KuduPredicate> predicates) throws IOException, InterruptedException {
+ KuduTableInputFormat input = new KuduTableInputFormat();
+ Configuration conf = new Configuration();
+ conf.set(KuduTableInputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
+ conf.set(KuduTableInputFormat.INPUT_TABLE_KEY, TABLE_NAME);
+ if (columnProjection != null) {
+ conf.set(KuduTableInputFormat.COLUMN_PROJECTION_KEY, columnProjection);
+ }
+ if (predicates != null) {
+ String encodedPredicates = KuduTableMapReduceUtil.base64EncodePredicates(predicates);
+ conf.set(KuduTableInputFormat.ENCODED_PREDICATES_KEY, encodedPredicates);
+ }
+ input.setConf(conf);
+ List<InputSplit> splits = input.getSplits(null);
+
+ // We need to re-create the input format to reconnect the client.
+ input = new KuduTableInputFormat();
+ input.setConf(conf);
+ RecordReader<NullWritable, RowResult> reader = input.createRecordReader(null, null);
+ reader.initialize(Iterables.getOnlyElement(splits), null);
+ return reader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
new file mode 100644
index 0000000..86452ed
--- /dev/null
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import org.kududb.client.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ITKuduTableOutputFormat extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ ITKuduTableOutputFormat.class.getName() + "-" + System.currentTimeMillis();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ }
+
+ @Test
+ public void test() throws Exception {
+ createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
+
+ KuduTableOutputFormat output = new KuduTableOutputFormat();
+ Configuration conf = new Configuration();
+ conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
+ conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, TABLE_NAME);
+ output.setConf(conf);
+
+ String multitonKey = conf.get(KuduTableOutputFormat.MULTITON_KEY);
+ KuduTable table = KuduTableOutputFormat.getKuduTable(multitonKey);
+ assertNotNull(table);
+
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addInt(0, 1);
+ row.addInt(1, 2);
+ row.addInt(2, 3);
+ row.addString(3, "a string");
+ row.addBoolean(4, true);
+
+ RecordWriter<NullWritable, Operation> rw = output.getRecordWriter(null);
+ rw.write(NullWritable.get(), insert);
+ rw.close(null);
+ AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
+ assertEquals(1, countRowsInScan(builder.build()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
new file mode 100644
index 0000000..dff2400
--- /dev/null
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import org.kududb.client.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+
+public class ITOutputFormatJob extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ ITOutputFormatJob.class.getName() + "-" + System.currentTimeMillis();
+
+ private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ try {
+ BaseKuduTest.tearDownAfterClass();
+ } finally {
+ HADOOP_UTIL.cleanup();
+ }
+ }
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void test() throws Exception {
+ Configuration conf = new Configuration();
+ String testHome =
+ HADOOP_UTIL.setupAndGetTestDir(ITOutputFormatJob.class.getName(), conf).getAbsolutePath();
+ String jobName = ITOutputFormatJob.class.getName();
+ Job job = new Job(conf, jobName);
+
+
+ // Create a 2 lines input file
+ File data = new File(testHome, "data.txt");
+ writeDataFile(data);
+ FileInputFormat.setInputPaths(job, data.toString());
+
+ // Configure the job to map the file and write to kudu, without reducers
+ Class<TestMapperTableOutput> mapperClass = TestMapperTableOutput.class;
+ job.setJarByClass(mapperClass);
+ job.setMapperClass(mapperClass);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setNumReduceTasks(0);
+ new KuduTableMapReduceUtil.TableOutputFormatConfigurator(
+ job,
+ TABLE_NAME,
+ getMasterAddresses())
+ .operationTimeoutMs(DEFAULT_SLEEP)
+ .addDependencies(false)
+ .configure();
+
+ assertTrue("Test job did not end properly", job.waitForCompletion(true));
+
+ // Make sure the data's there
+ KuduTable table = openTable(TABLE_NAME);
+ AsyncKuduScanner.AsyncKuduScannerBuilder builder =
+ client.newScannerBuilder(table);
+ assertEquals(2, countRowsInScan(builder.build()));
+ }
+
+ /**
+ * Simple Mapper that writes one row per line, the key is the line number and the STRING column
+ * is the data from that line
+ */
+ static class TestMapperTableOutput extends
+ Mapper<LongWritable, Text, NullWritable, Operation> {
+
+ private KuduTable table;
+ @Override
+ protected void map(LongWritable key, Text value, Context context) throws IOException,
+ InterruptedException {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addInt(0, (int) key.get());
+ row.addInt(1, 1);
+ row.addInt(2, 2);
+ row.addString(3, value.toString());
+ row.addBoolean(4, true);
+ context.write(NullWritable.get(), insert);
+ }
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ table = KuduTableMapReduceUtil.getTableFromContext(context);
+ }
+ }
+
+ private void writeDataFile(File data) throws IOException {
+ FileOutputStream fos = new FileOutputStream(data);
+ fos.write("VALUE1\nVALUE2\n".getBytes());
+ fos.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
----------------------------------------------------------------------
diff --git a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
new file mode 100644
index 0000000..3801a0c
--- /dev/null
+++ b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/TestJarFinder.java
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce;
+
+import org.apache.commons.logging.LogFactory;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.text.MessageFormat;
+import java.util.Properties;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+/**
+ * This file was forked from hbase/branches/master@4ce6f48.
+ */
+public class TestJarFinder {
+
+ @Test
+ public void testJar() throws Exception {
+
+ // Picking a class that is for sure in a JAR in the classpath
+ String jar = JarFinder.getJar(LogFactory.class);
+ Assert.assertTrue(new File(jar).exists());
+ }
+
+ private static void delete(File file) throws IOException {
+ if (file.getAbsolutePath().length() < 5) {
+ throw new IllegalArgumentException(
+ MessageFormat.format("Path [{0}] is too short, not deleting",
+ file.getAbsolutePath()));
+ }
+ if (file.exists()) {
+ if (file.isDirectory()) {
+ File[] children = file.listFiles();
+ if (children != null) {
+ for (File child : children) {
+ delete(child);
+ }
+ }
+ }
+ if (!file.delete()) {
+ throw new RuntimeException(
+ MessageFormat.format("Could not delete path [{0}]",
+ file.getAbsolutePath()));
+ }
+ }
+ }
+
+ @Test
+ public void testExpandedClasspath() throws Exception {
+ // Picking a class that is for sure in a directory in the classpath
+ // In this case, the JAR is created on the fly
+ String jar = JarFinder.getJar(TestJarFinder.class);
+ Assert.assertTrue(new File(jar).exists());
+ }
+
+ @Test
+ public void testExistingManifest() throws Exception {
+ File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+ TestJarFinder.class.getName() + "-testExistingManifest");
+ delete(dir);
+ dir.mkdirs();
+
+ File metaInfDir = new File(dir, "META-INF");
+ metaInfDir.mkdirs();
+ File manifestFile = new File(metaInfDir, "MANIFEST.MF");
+ Manifest manifest = new Manifest();
+ OutputStream os = new FileOutputStream(manifestFile);
+ manifest.write(os);
+ os.close();
+
+ File propsFile = new File(dir, "props.properties");
+ Writer writer = new FileWriter(propsFile);
+ new Properties().store(writer, "");
+ writer.close();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ JarOutputStream zos = new JarOutputStream(baos);
+ JarFinder.jarDir(dir, "", zos);
+ JarInputStream jis =
+ new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
+ Assert.assertNotNull(jis.getManifest());
+ jis.close();
+ }
+
+ @Test
+ public void testNoManifest() throws Exception {
+ File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+ TestJarFinder.class.getName() + "-testNoManifest");
+ delete(dir);
+ dir.mkdirs();
+ File propsFile = new File(dir, "props.properties");
+ Writer writer = new FileWriter(propsFile);
+ new Properties().store(writer, "");
+ writer.close();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ JarOutputStream zos = new JarOutputStream(baos);
+ JarFinder.jarDir(dir, "", zos);
+ JarInputStream jis =
+ new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
+ Assert.assertNotNull(jis.getManifest());
+ jis.close();
+ }
+}
\ No newline at end of file
[32/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java
new file mode 100644
index 0000000..f407c5c
--- /dev/null
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java
@@ -0,0 +1,116 @@
+/**
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.mapreduce.tools;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.mapreduce.CommandLineParser;
+import org.kududb.mapreduce.KuduTableMapReduceUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import java.io.IOException;
+
+/**
+ * Map-only job that reads CSV files and inserts them into a single Kudu table.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class ImportCsv extends Configured implements Tool {
+
+ public static enum Counters { BAD_LINES };
+
+ static final String NAME = "importcsv";
+ static final String DEFAULT_SEPARATOR = "\t";
+ static final String SEPARATOR_CONF_KEY = "importcsv.separator";
+ static final String JOB_NAME_CONF_KEY = "importcsv.job.name";
+ static final String SKIP_LINES_CONF_KEY = "importcsv.skip.bad.lines";
+ static final String COLUMNS_NAMES_KEY = "importcsv.column.names";
+
+ /**
+ * Sets up the actual job.
+ *
+ * @param conf The current configuration.
+ * @param args The command line parameters.
+ * @return The newly created job.
+ * @throws java.io.IOException When setting up the job fails.
+ */
+ @SuppressWarnings("deprecation")
+ public static Job createSubmittableJob(Configuration conf, String[] args)
+ throws IOException, ClassNotFoundException {
+
+ Class<ImportCsvMapper> mapperClass = ImportCsvMapper.class;
+ conf.set(COLUMNS_NAMES_KEY, args[0]);
+ String tableName = args[1];
+ Path inputDir = new Path(args[2]);
+
+ String jobName = conf.get(JOB_NAME_CONF_KEY, NAME + "_" + tableName);
+ Job job = new Job(conf, jobName);
+ job.setJarByClass(mapperClass);
+ FileInputFormat.setInputPaths(job, inputDir);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setMapperClass(mapperClass);
+ job.setNumReduceTasks(0);
+ new KuduTableMapReduceUtil.TableOutputFormatConfiguratorWithCommandLineParser(
+ job,
+ tableName)
+ .configure();
+ return job;
+ }
+
+ /*
+ * @param errorMsg Error message. Can be null.
+ */
+ private static void usage(final String errorMsg) {
+ if (errorMsg != null && errorMsg.length() > 0) {
+ System.err.println("ERROR: " + errorMsg);
+ }
+ String usage =
+ "Usage: " + NAME + " <colAa,colB,colC> <table.name> <input.dir>\n\n" +
+ "Imports the given input directory of CSV data into the specified table.\n" +
+ "\n" +
+ "The column names of the CSV data must be specified in the form of " +
+ "comma-separated column names.\n" +
+ "Other options that may be specified with -D include:\n" +
+ " -D" + SKIP_LINES_CONF_KEY + "=false - fail if encountering an invalid line\n" +
+ " '-D" + SEPARATOR_CONF_KEY + "=|' - eg separate on pipes instead of tabs\n" +
+ " -D" + JOB_NAME_CONF_KEY + "=jobName - use the specified mapreduce job name for the" +
+ " import.\n" +
+ CommandLineParser.getHelpSnippet();
+
+ System.err.println(usage);
+ }
+
+ @Override
+ public int run(String[] otherArgs) throws Exception {
+ if (otherArgs.length < 3) {
+ usage("Wrong number of arguments: " + otherArgs.length);
+ return -1;
+ }
+ Job job = createSubmittableJob(getConf(), otherArgs);
+ return job.waitForCompletion(true) ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int status = ToolRunner.run(new ImportCsv(), args);
+ System.exit(status);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java
new file mode 100644
index 0000000..21f43b5
--- /dev/null
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java
@@ -0,0 +1,143 @@
+/**
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.mapreduce.tools;
+
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.*;
+import org.kududb.mapreduce.KuduTableMapReduceUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import java.io.IOException;
+
+/**
+ * Mapper that ingests CSV lines and turns them into Kudu Inserts.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ImportCsvMapper extends Mapper<LongWritable, Text, NullWritable, Operation> {
+
+ private static final NullWritable NULL_KEY = NullWritable.get();
+
+ /** Column seperator */
+ private String separator;
+
+ /** Should skip bad lines */
+ private boolean skipBadLines;
+ private Counter badLineCount;
+
+ private CsvParser parser;
+
+ private KuduTable table;
+ private Schema schema;
+
+ /**
+ * Handles initializing this class with objects specific to it (i.e., the parser).
+ */
+ @Override
+ protected void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+
+ this.separator = conf.get(ImportCsv.SEPARATOR_CONF_KEY);
+ if (this.separator == null) {
+ this.separator = ImportCsv.DEFAULT_SEPARATOR;
+ }
+
+ this.skipBadLines = conf.getBoolean(ImportCsv.SKIP_LINES_CONF_KEY, true);
+ this.badLineCount = context.getCounter(ImportCsv.Counters.BAD_LINES);
+
+ this.parser = new CsvParser(conf.get(ImportCsv.COLUMNS_NAMES_KEY), this.separator);
+
+ this.table = KuduTableMapReduceUtil.getTableFromContext(context);
+ this.schema = this.table.getSchema();
+ }
+
+ /**
+ * Convert a line of CSV text into a Kudu Insert
+ */
+ @Override
+ public void map(LongWritable offset, Text value,
+ Context context)
+ throws IOException {
+ byte[] lineBytes = value.getBytes();
+
+ try {
+ CsvParser.ParsedLine parsed = this.parser.parse(lineBytes, value.getLength());
+
+ Insert insert = this.table.newInsert();
+ PartialRow row = insert.getRow();
+ for (int i = 0; i < parsed.getColumnCount(); i++) {
+ String colName = parsed.getColumnName(i);
+ ColumnSchema col = this.schema.getColumn(colName);
+ String colValue = Bytes.getString(parsed.getLineBytes(), parsed.getColumnOffset(i),
+ parsed.getColumnLength(i));
+ switch (col.getType()) {
+ case BOOL:
+ row.addBoolean(colName, Boolean.parseBoolean(colValue));
+ break;
+ case INT8:
+ row.addByte(colName, Byte.parseByte(colValue));
+ break;
+ case INT16:
+ row.addShort(colName, Short.parseShort(colValue));
+ break;
+ case INT32:
+ row.addInt(colName, Integer.parseInt(colValue));
+ break;
+ case INT64:
+ row.addLong(colName, Long.parseLong(colValue));
+ break;
+ case STRING:
+ row.addString(colName, colValue);
+ break;
+ case FLOAT:
+ row.addFloat(colName, Float.parseFloat(colValue));
+ break;
+ case DOUBLE:
+ row.addDouble(colName, Double.parseDouble(colValue));
+ break;
+ default:
+ throw new IllegalArgumentException("Type " + col.getType() + " not recognized");
+ }
+ }
+ context.write(NULL_KEY, insert);
+ } catch (CsvParser.BadCsvLineException badLine) {
+ if (this.skipBadLines) {
+ System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage());
+ this.badLineCount.increment(1);
+ return;
+ } else {
+ throw new IOException("Failing task because of a bad line", badLine);
+ }
+ } catch (IllegalArgumentException e) {
+ if (this.skipBadLines) {
+ System.err.println("Bad line at offset: " + offset.get() + ":\n" + e.getMessage());
+ this.badLineCount.increment(1);
+ return;
+ } else {
+ throw new IOException("Failing task because of an illegal argument", e);
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Failing task since it was interrupted", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
new file mode 100644
index 0000000..549e117
--- /dev/null
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
@@ -0,0 +1,1662 @@
+/**
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.mapreduce.tools;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.*;
+import org.kududb.mapreduce.CommandLineParser;
+import org.kududb.mapreduce.KuduTableMapReduceUtil;
+import org.kududb.util.Pair;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * <p>
+ * This is an integration test borrowed from goraci, written by Keith Turner,
+ * which is in turn inspired by the Accumulo test called continous ingest (ci).
+ * The original source code can be found here:
+ * </p>
+ * <ul>
+ * <li>
+ * <a href="https://github.com/keith-turner/goraci">https://github.com/keith-turner/goraci</a>
+ * </li>
+ * <li>
+ * <a href="https://github.com/enis/goraci/">https://github.com/enis/goraci/</a>
+ * </li>
+ * </ul>
+ *
+ * <p>
+ * Apache Accumulo has a simple test suite that verifies that data is not
+ * lost at scale. This test suite is called continuous ingest. This test runs
+ * many ingest clients that continually create linked lists containing 25
+ * million nodes. At some point the clients are stopped and a map reduce job is
+ * run to ensure no linked list has a hole. A hole indicates data was lost.
+ * </p>
+ *
+ * <p>
+ * The nodes in the linked list are random. This causes each linked list to
+ * spread across the table. Therefore if one part of a table loses data, then it
+ * will be detected by references in another part of the table.
+ * </p>
+ *
+ * <h3>
+ * THE ANATOMY OF THE TEST
+ * </h3>
+ *
+ * <p>
+ * Below is rough sketch of how data is written. For specific details look at
+ * the Generator code.
+ * </p>
+ * <ol>
+ * <li>
+ * Write out 1 million nodes
+ * </li>
+ * <li>
+ * Flush the client
+ * </li>
+ * <li>
+ * Write out 1 million that reference previous million
+ * </li>
+ * <li>
+ * If this is the 25th set of 1 million nodes, then update 1st set of million to point to last
+ * </li>
+ * <li>
+ * Goto 1
+ * </li>
+ * </ol>
+ *
+ * <p>
+ * The key is that nodes only reference flushed nodes. Therefore a node should
+ * never reference a missing node, even if the ingest client is killed at any
+ * point in time.
+ * </p>
+ *
+ * <p>
+ * When running this test suite w/ Accumulo there is a script running in
+ * parallel called the Agitator that randomly and continuously kills server
+ * processes. The outcome was that many data loss bugs were found in Accumulo
+ * by doing this. This test suite can also help find bugs that impact uptime
+ * and stability when run for days or weeks.
+ * </p>
+ *
+ * <p>
+ * This test suite consists the following:
+ * </p>
+ * <ul>
+ * <li>
+ * A few Java programs
+ * </li>
+ * <li>
+ * A little helper script to run the java programs
+ * </li>
+ * <li>
+ * A maven script to build it.
+ * </li>
+ * </ul>
+ *
+ * <p>
+ * When generating data, its best to have each map task generate a multiple of
+ * 25 million. The reason for this is that circular linked list are generated
+ * every 25M. Not generating a multiple in 25M will result in some nodes in the
+ * linked list not having references. The loss of an unreferenced node can not
+ * be detected.
+ * </p>
+ *
+ * <h3>
+ * Below is a description of the Java programs
+ * </h3>
+ *
+ * <ul>
+ * <li>
+ * Generator - A map only job that generates data. As stated previously,
+ * its best to generate data in multiples of 25M.
+ * </li>
+ * <li>
+ * Verify - A map reduce job that looks for holes. Look at the counts after running. REFERENCED and
+ * UNREFERENCED are ok, any UNDEFINED counts are bad. Do not run at the same
+ * time as the Generator.
+ * </li>
+ * <li>
+ * Print - A standalone program that prints nodes in the linked list
+ * </li>
+ * <li>
+ * Delete - Disabled. A standalone program that deletes a single node
+ * </li>
+ * <li>
+ * Walker - Disabled. A standalong program that start following a linked list and emits timing
+ * info.
+ * </li>
+ * </ul>
+ *
+ * <h3>
+ * KUDU-SPECIFIC CHANGES
+ * </h3>
+ *
+ * <ul>
+ * <li>
+ * The 16 bytes row key is divided into two 8 byte long since we don't have a "bytes" type in
+ * Kudu. Note that the C++ client can store bytes directly in string columns. Using longs
+ * enables us to pretty print human readable keys than can then be passed back just as easily.
+ * </li>
+ * <li>
+ * The table can be pre-split when running the Generator. The row keys' first component will be
+ * spread over the Long.MIN_VALUE - Long.MAX_VALUE keyspace.
+ * </li>
+ * <li>
+ * The Walker and Deleter progams were disabled to save some time but they can be re-enabled then
+ * ported to Kudu without too much effort.
+ * </li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class IntegrationTestBigLinkedList extends Configured implements Tool {
+ private static final byte[] NO_KEY = new byte[1];
+
+ protected static final String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
+
+ protected static final String DEFAULT_TABLE_NAME = "IntegrationTestBigLinkedList";
+
+ protected static final String HEADS_TABLE_NAME_KEY = "IntegrationTestBigLinkedList.heads_table";
+
+ protected static final String DEFAULT_HEADS_TABLE_NAME = "IntegrationTestBigLinkedListHeads";
+
+ /** Row key, two times 8 bytes. */
+ private static final String COLUMN_KEY_ONE = "key1";
+ private static final String COLUMN_KEY_TWO = "key2";
+
+ /** Link to the id of the prev node in the linked list, two times 8 bytes. */
+ private static final String COLUMN_PREV_ONE = "prev1";
+ private static final String COLUMN_PREV_TWO = "prev2";
+
+ /** identifier of the mapred task that generated this row. */
+ private static final String COLUMN_CLIENT = "client";
+
+ /** the id of the row within the same client. */
+ private static final String COLUMN_ROW_ID = "row_id";
+
+ /** The number of times this row was updated. */
+ private static final String COLUMN_UPDATE_COUNT = "update_count";
+
+ /** How many rows to write per map task. This has to be a multiple of 25M. */
+ private static final String GENERATOR_NUM_ROWS_PER_MAP_KEY
+ = "IntegrationTestBigLinkedList.generator.num_rows";
+
+ private static final String GENERATOR_NUM_MAPPERS_KEY
+ = "IntegrationTestBigLinkedList.generator.map.tasks";
+
+ private static final String GENERATOR_WIDTH_KEY
+ = "IntegrationTestBigLinkedList.generator.width";
+
+ private static final String GENERATOR_WRAP_KEY
+ = "IntegrationTestBigLinkedList.generator.wrap";
+
+ private static final int WIDTH_DEFAULT = 1000000;
+ private static final int WRAP_DEFAULT = 25;
+ private static final int ROWKEY_LENGTH = 16;
+
+ private String toRun;
+ private String[] otherArgs;
+
+ static class CINode {
+ String key;
+ String prev;
+ String client;
+ long rowId;
+ int updateCount;
+ }
+
+ static Schema getTableSchema() {
+ List<ColumnSchema> columns = new ArrayList<ColumnSchema>(7);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_ONE, Type.INT64)
+ .key(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_TWO, Type.INT64)
+ .key(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_PREV_ONE, Type.INT64)
+ .nullable(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_PREV_TWO, Type.INT64)
+ .nullable(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_ROW_ID, Type.INT64)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_CLIENT, Type.STRING)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_UPDATE_COUNT, Type.INT32)
+ .build());
+ return new Schema(columns);
+ }
+
+ static Schema getHeadsTableSchema() {
+ List<ColumnSchema> columns = new ArrayList<ColumnSchema>(2);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_ONE, Type.INT64)
+ .key(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_TWO, Type.INT64)
+ .key(true)
+ .build());
+ return new Schema(columns);
+ }
+
+ /**
+ * A Map only job that generates random linked list and stores them.
+ */
+ static class Generator extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(Generator.class);
+
+ static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> {
+ static class GeneratorInputSplit extends InputSplit implements Writable {
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ return 1;
+ }
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return new String[0];
+ }
+ @Override
+ public void readFields(DataInput arg0) throws IOException {
+ }
+ @Override
+ public void write(DataOutput arg0) throws IOException {
+ }
+ }
+
+ static class GeneratorRecordReader extends RecordReader<BytesWritable,NullWritable> {
+ private long count;
+ private long numNodes;
+ private Random rand;
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public BytesWritable getCurrentKey() throws IOException, InterruptedException {
+ byte[] bytes = new byte[ROWKEY_LENGTH];
+ rand.nextBytes(bytes);
+ return new BytesWritable(bytes);
+ }
+
+ @Override
+ public NullWritable getCurrentValue() throws IOException, InterruptedException {
+ return NullWritable.get();
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return (float)(count / (double)numNodes);
+ }
+
+ @Override
+ public void initialize(InputSplit arg0, TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
+ // Use SecureRandom to avoid issue described in HBASE-13382.
+ rand = new SecureRandom();
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ return count++ < numNodes;
+ }
+
+ }
+
+ @Override
+ public RecordReader<BytesWritable,NullWritable> createRecordReader(
+ InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
+ GeneratorRecordReader rr = new GeneratorRecordReader();
+ rr.initialize(split, context);
+ return rr;
+ }
+
+ @Override
+ public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
+ int numMappers = job.getConfiguration().getInt(GENERATOR_NUM_MAPPERS_KEY, 1);
+
+ ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numMappers);
+
+ for (int i = 0; i < numMappers; i++) {
+ splits.add(new GeneratorInputSplit());
+ }
+
+ return splits;
+ }
+ }
+
+ /** Ensure output files from prev-job go to map inputs for current job */
+ static class OneFilePerMapperSFIF<K, V> extends SequenceFileInputFormat<K, V> {
+ @Override
+ protected boolean isSplitable(JobContext context, Path filename) {
+ return false;
+ }
+ }
+
+ /**
+ * Some ASCII art time:
+ * [ . . . ] represents one batch of random longs of length WIDTH
+ *
+ * _________________________
+ * | ______ |
+ * | | ||
+ * __+_________________+_____ ||
+ * v v v |||
+ * first = [ . . . . . . . . . . . ] |||
+ * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ |||
+ * | | | | | | | | | | | |||
+ * prev = [ . . . . . . . . . . . ] |||
+ * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ |||
+ * | | | | | | | | | | | |||
+ * current = [ . . . . . . . . . . . ] |||
+ * |||
+ * ... |||
+ * |||
+ * last = [ . . . . . . . . . . . ] |||
+ * | | | | | | | | | | |-----|||
+ * | |--------||
+ * |___________________________|
+ */
+ static class GeneratorMapper
+ extends Mapper<BytesWritable, NullWritable, NullWritable, NullWritable> {
+
+ private byte[][] first = null;
+ private byte[][] prev = null;
+ private byte[][] current = null;
+ private String id;
+ private long rowId = 0;
+ private int i;
+ private KuduClient client;
+ private KuduTable table;
+ private KuduSession session;
+ private KuduTable headsTable;
+ private long numNodes;
+ private long wrap;
+ private int width;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ id = "Job: " + context.getJobID() + " Task: " + context.getTaskAttemptID();
+ Configuration conf = context.getConfiguration();
+ CommandLineParser parser = new CommandLineParser(conf);
+ client = parser.getClient();
+ try {
+ table = client.openTable(getTableName(conf));
+ headsTable = client.openTable(getHeadsTable(conf));
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ session = client.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ session.setMutationBufferSpace(WIDTH_DEFAULT);
+ session.setIgnoreAllDuplicateRows(true);
+
+ this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT);
+ current = new byte[this.width][];
+ int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT);
+ this.wrap = (long)wrapMultiplier * width;
+ this.numNodes = context.getConfiguration().getLong(
+ GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT);
+ if (this.numNodes < this.wrap) {
+ this.wrap = this.numNodes;
+ }
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ try {
+ session.close();
+ client.shutdown();
+ } catch (Exception ex) {
+ // ugh.
+ throw new IOException(ex);
+ }
+ }
+
+ @Override
+ protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
+ current[i] = new byte[key.getLength()];
+ System.arraycopy(key.getBytes(), 0, current[i], 0, key.getLength());
+ if (++i == current.length) {
+ persist(output, current, false);
+ i = 0;
+
+ // Keep track of the first row so that we can point to it at the end.
+ if (first == null) {
+ first = current;
+ }
+ prev = current;
+ current = new byte[this.width][];
+
+ rowId += current.length;
+ output.setStatus("Count " + rowId);
+
+ // Check if it's time to wrap up this batch.
+ if (rowId % wrap == 0) {
+ // this block of code turns the 1 million linked list of length 25 into one giant
+ // circular linked list of 25 million.
+ circularLeftShift(first);
+
+ persist(output, first, true);
+
+ Operation insert = headsTable.newInsert();
+ PartialRow row = insert.getRow();
+ row.addLong(COLUMN_KEY_ONE, Bytes.getLong(first[0]));
+ row.addLong(COLUMN_KEY_TWO, Bytes.getLong(first[0], 8));
+ try {
+ session.apply(insert);
+ session.flush();
+ } catch (Exception e) {
+ throw new IOException("Couldn't flush the head row, " + insert, e);
+ }
+
+ first = null;
+ prev = null;
+ }
+ }
+ }
+
+ private static <T> void circularLeftShift(T[] first) {
+ T ez = first[0];
+ for (int i = 0; i < first.length - 1; i++)
+ first[i] = first[i + 1];
+ first[first.length - 1] = ez;
+ }
+
+ private void persist(Context output, byte[][] data, boolean update)
+ throws IOException {
+ try {
+ for (int i = 0; i < data.length; i++) {
+ Operation put = update ? table.newUpdate() : table.newInsert();
+ PartialRow row = put.getRow();
+
+ long keyOne = Bytes.getLong(data[i]);
+ long keyTwo = Bytes.getLong(data[i], 8);
+
+ row.addLong(COLUMN_KEY_ONE, keyOne);
+ row.addLong(COLUMN_KEY_TWO, keyTwo);
+
+ // prev is null for the first line, we'll update it at the end.
+ if (prev == null) {
+ row.setNull(COLUMN_PREV_ONE);
+ row.setNull(COLUMN_PREV_TWO);
+ } else {
+ row.addLong(COLUMN_PREV_ONE, Bytes.getLong(prev[i]));
+ row.addLong(COLUMN_PREV_TWO, Bytes.getLong(prev[i], 8));
+ }
+
+ if (!update) {
+ // We only add those for new inserts, we don't update the heads with a new row, etc.
+ row.addLong(COLUMN_ROW_ID, rowId + i);
+ row.addString(COLUMN_CLIENT, id);
+ row.addInt(COLUMN_UPDATE_COUNT, 0);
+ }
+ session.apply(put);
+
+ if (i % 1000 == 0) {
+ // Tickle progress every so often else maprunner will think us hung
+ output.progress();
+ }
+ }
+
+ session.flush();
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 4) {
+ System.out.println("Usage : " + Generator.class.getSimpleName() +
+ " <num mappers> <num nodes per map> <num_tablets> <tmp output dir> [<width> <wrap " +
+ "multiplier>]");
+ System.out.println(" where <num nodes per map> should be a multiple of " +
+ " width*wrap multiplier, 25M by default");
+ return 0;
+ }
+
+ int numMappers = Integer.parseInt(args[0]);
+ long numNodes = Long.parseLong(args[1]);
+ int numTablets = Integer.parseInt(args[2]);
+ Path tmpOutput = new Path(args[3]);
+ Integer width = (args.length < 5) ? null : Integer.parseInt(args[4]);
+ Integer wrapMultiplier = (args.length < 6) ? null : Integer.parseInt(args[5]);
+ return run(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier);
+ }
+
+ protected void createTables(int numTablets) throws Exception {
+
+ createSchema(getTableName(getConf()), getTableSchema(), numTablets);
+ createSchema(getHeadsTable(getConf()), getHeadsTableSchema(), numTablets);
+ }
+
+ protected void createSchema(String tableName, Schema schema, int numTablets) throws Exception {
+ CommandLineParser parser = new CommandLineParser(getConf());
+ KuduClient client = parser.getClient();
+ try {
+ if (numTablets < 1) {
+ numTablets = 1;
+ }
+
+ if (client.tableExists(tableName)) {
+ return;
+ }
+
+ CreateTableOptions builder =
+ new CreateTableOptions().setNumReplicas(parser.getNumReplicas())
+ .setRangePartitionColumns(ImmutableList.of("key1", "key2"));
+ if (numTablets > 1) {
+ BigInteger min = BigInteger.valueOf(Long.MIN_VALUE);
+ BigInteger max = BigInteger.valueOf(Long.MAX_VALUE);
+ BigInteger step = max.multiply(BigInteger.valueOf(2)).divide(BigInteger.valueOf
+ (numTablets));
+ LOG.info(min.longValue());
+ LOG.info(max.longValue());
+ LOG.info(step.longValue());
+ PartialRow splitRow = schema.newPartialRow();
+ splitRow.addLong("key2", Long.MIN_VALUE);
+ for (int i = 1; i < numTablets; i++) {
+ long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue();
+ LOG.info("key " + key);
+ splitRow.addLong("key1", key);
+ builder.addSplitRow(splitRow);
+ }
+ }
+
+ client.createTable(tableName, schema, builder);
+ } finally {
+ // Done with this client.
+ client.shutdown();
+ }
+ }
+
+ public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,
+ Integer width, Integer wrapMultiplier) throws Exception {
+ LOG.info("Running RandomInputGenerator with numMappers=" + numMappers
+ + ", numNodes=" + numNodes);
+ Job job = new Job(getConf());
+
+ job.setJobName("Random Input Generator");
+ job.setNumReduceTasks(0);
+ job.setJarByClass(getClass());
+
+ job.setInputFormatClass(GeneratorInputFormat.class);
+ job.setOutputKeyClass(BytesWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ setJobConf(job, numMappers, numNodes, width, wrapMultiplier);
+
+ job.setMapperClass(Mapper.class); //identity mapper
+
+ FileOutputFormat.setOutputPath(job, tmpOutput);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+
+ boolean success = job.waitForCompletion(true);
+
+ return success ? 0 : 1;
+ }
+
+ public int runGenerator(int numMappers, long numNodes, int numTablets, Path tmpOutput,
+ Integer width, Integer wrapMultiplier) throws Exception {
+ LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes);
+ createTables(numTablets);
+
+ Job job = new Job(getConf());
+
+ job.setJobName("Link Generator");
+ job.setNumReduceTasks(0);
+ job.setJarByClass(getClass());
+
+ FileInputFormat.setInputPaths(job, tmpOutput);
+ job.setInputFormatClass(OneFilePerMapperSFIF.class);
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ setJobConf(job, numMappers, numNodes, width, wrapMultiplier);
+
+ job.setMapperClass(GeneratorMapper.class);
+
+ job.setOutputFormatClass(NullOutputFormat.class);
+
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+ // If we fail, retrying will fail again in case we were able to flush at least once since
+ // we'll be creating duplicate rows. Better to just have one try.
+ job.getConfiguration().setInt("mapreduce.map.maxattempts", 1);
+ // Lack of YARN-445 means we can't auto-jstack on timeout, so disabling the timeout gives
+ // us a chance to do it manually.
+ job.getConfiguration().setInt("mapreduce.task.timeout", 0);
+ KuduTableMapReduceUtil.addDependencyJars(job);
+
+ boolean success = job.waitForCompletion(true);
+
+ return success ? 0 : 1;
+ }
+
+ public int run(int numMappers, long numNodes, int numTablets, Path tmpOutput,
+ Integer width, Integer wrapMultiplier) throws Exception {
+ int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier);
+ if (ret > 0) {
+ return ret;
+ }
+ return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier);
+ }
+ }
+
+ /**
+ * A Map Reduce job that verifies that the linked lists generated by
+ * {@link Generator} do not have any holes.
+ */
+ static class Verify extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(Verify.class);
+ private static final BytesWritable DEF = new BytesWritable(NO_KEY);
+ private static final Joiner COMMA_JOINER = Joiner.on(",");
+ private static final byte[] rowKey = new byte[ROWKEY_LENGTH];
+ private static final byte[] prev = new byte[ROWKEY_LENGTH];
+
+ private Job job;
+
+ public static class VerifyMapper extends Mapper<NullWritable, RowResult,
+ BytesWritable, BytesWritable> {
+ private BytesWritable row = new BytesWritable();
+ private BytesWritable ref = new BytesWritable();
+
+ @Override
+ protected void map(NullWritable key, RowResult value, Mapper.Context context)
+ throws IOException ,InterruptedException {
+ Bytes.setLong(rowKey, value.getLong(0));
+ Bytes.setLong(rowKey, value.getLong(1), 8);
+
+ row.set(rowKey, 0, rowKey.length);
+ // Emit that the row is defined
+ context.write(row, DEF);
+ if (value.isNull(2)) {
+ LOG.warn(String.format("Prev is not set for: %s", Bytes.pretty(rowKey)));
+ } else {
+ Bytes.setLong(prev, value.getLong(2));
+ Bytes.setLong(prev, value.getLong(3), 8);
+ ref.set(prev, 0, prev.length);
+ // Emit which row is referenced by this row.
+ context.write(ref, row);
+ }
+ }
+ }
+
+ public enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, EXTRAREFERENCES
+ }
+
+ public static class VerifyReducer extends Reducer<BytesWritable,BytesWritable,Text,Text> {
+ private ArrayList<byte[]> refs = new ArrayList<byte[]>();
+
+ private AtomicInteger rows = new AtomicInteger(0);
+
+ @Override
+ public void reduce(BytesWritable key, Iterable<BytesWritable> values, Context context)
+ throws IOException, InterruptedException {
+
+ int defCount = 0;
+
+ refs.clear();
+ // We only expect two values, a DEF and a reference, but there might be more.
+ for (BytesWritable type : values) {
+ if (type.getLength() == DEF.getLength()) {
+ defCount++;
+ } else {
+ byte[] bytes = new byte[type.getLength()];
+ System.arraycopy(type.getBytes(), 0, bytes, 0, type.getLength());
+ refs.add(bytes);
+ }
+ }
+
+ // TODO check for more than one def, should not happen
+
+ List<String> refsList = new ArrayList<>(refs.size());
+ String keyString = null;
+ if (defCount == 0 || refs.size() != 1) {
+ for (byte[] ref : refs) {
+ refsList.add(COMMA_JOINER.join(Bytes.getLong(ref), Bytes.getLong(ref, 8)));
+ }
+ keyString = COMMA_JOINER.join(Bytes.getLong(key.getBytes()),
+ Bytes.getLong(key.getBytes(), 8));
+
+ LOG.error("Linked List error: Key = " + keyString + " References = " + refsList);
+ }
+
+ if (defCount == 0 && refs.size() > 0) {
+ // this is bad, found a node that is referenced but not defined. It must have been
+ // lost, emit some info about this node for debugging purposes.
+ context.write(new Text(keyString), new Text(refsList.toString()));
+ context.getCounter(Counts.UNDEFINED).increment(1);
+ } else if (defCount > 0 && refs.size() == 0) {
+ // node is defined but not referenced
+ context.write(new Text(keyString), new Text("none"));
+ context.getCounter(Counts.UNREFERENCED).increment(1);
+ } else {
+ if (refs.size() > 1) {
+ if (refsList != null) {
+ context.write(new Text(keyString), new Text(refsList.toString()));
+ }
+ context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1);
+ }
+ // node is defined and referenced
+ context.getCounter(Counts.REFERENCED).increment(1);
+ }
+
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ if (args.length != 2) {
+ System.out.println("Usage : " + Verify.class.getSimpleName() + " <output dir> <num reducers>");
+ return 0;
+ }
+
+ String outputDir = args[0];
+ int numReducers = Integer.parseInt(args[1]);
+
+ return run(outputDir, numReducers);
+ }
+
+ public int run(String outputDir, int numReducers) throws Exception {
+ return run(new Path(outputDir), numReducers);
+ }
+
+ public int run(Path outputDir, int numReducers) throws Exception {
+ LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);
+
+ job = new Job(getConf());
+
+ job.setJobName("Link Verifier");
+ job.setNumReduceTasks(numReducers);
+ job.setJarByClass(getClass());
+
+ Joiner columnsToQuery = Joiner.on(",");
+
+ new KuduTableMapReduceUtil.TableInputFormatConfiguratorWithCommandLineParser(
+ job, getTableName(getConf()),
+ columnsToQuery.join(COLUMN_KEY_ONE, COLUMN_KEY_TWO, COLUMN_PREV_ONE, COLUMN_PREV_TWO))
+ .configure();
+ job.setMapperClass(VerifyMapper.class);
+ job.setMapOutputKeyClass(BytesWritable.class);
+ job.setMapOutputValueClass(BytesWritable.class);
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+
+ job.setReducerClass(VerifyReducer.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ TextOutputFormat.setOutputPath(job, outputDir);
+
+ boolean success = job.waitForCompletion(true);
+
+ return success ? 0 : 1;
+ }
+
+ @SuppressWarnings("deprecation")
+ public boolean verify(long expectedReferenced) throws Exception {
+ if (job == null) {
+ throw new IllegalStateException("You should call run() first");
+ }
+
+ Counters counters = job.getCounters();
+
+ Counter referenced = counters.findCounter(Counts.REFERENCED);
+ Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
+ Counter undefined = counters.findCounter(Counts.UNDEFINED);
+ Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);
+
+ boolean success = true;
+ //assert
+ if (expectedReferenced != referenced.getValue()) {
+ LOG.error("Expected referenced count does not match with actual referenced count. " +
+ "Expected referenced=" + expectedReferenced + ", actual=" + referenced.getValue());
+ success = false;
+ }
+
+ if (unreferenced.getValue() > 0) {
+ boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
+ LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
+ + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
+ success = false;
+ }
+
+ if (undefined.getValue() > 0) {
+ LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
+ success = false;
+ }
+
+ // TODO Add the rows' location on failure.
+ if (!success) {
+ //Configuration conf = job.getConfiguration();
+ //HConnection conn = HConnectionManager.getConnection(conf);
+ //TableName tableName = getTableName(conf);
+ CounterGroup g = counters.getGroup("undef");
+ Iterator<Counter> it = g.iterator();
+ while (it.hasNext()) {
+ String keyString = it.next().getName();
+ //byte[] key = Bytes.toBytes(keyString);
+ //HRegionLocation loc = conn.relocateRegion(tableName, key);
+ LOG.error("undefined row " + keyString /*+ ", " + loc*/);
+ }
+ g = counters.getGroup("unref");
+ it = g.iterator();
+ while (it.hasNext()) {
+ String keyString = it.next().getName();
+ //byte[] key = Bytes.toBytes(keyString);
+ //HRegionLocation loc = conn.relocateRegion(tableName, key);
+ LOG.error("unreferred row " + keyString /*+ ", " + loc*/);
+ }
+ }
+ return success;
+ }
+ }
+
+ /**
+ * Executes Generate and Verify in a loop. Data is not cleaned between runs, so each iteration
+ * adds more data.
+ */
+ static class Loop extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(Loop.class);
+
+ IntegrationTestBigLinkedList it;
+
+ FileSystem fs;
+
+ protected void runGenerator(int numMappers, long numNodes, int numTablets,
+ String outputDir, Integer width, Integer wrapMultiplier) throws Exception {
+ Path outputPath = new Path(outputDir);
+ UUID uuid = UUID.randomUUID(); //create a random UUID.
+ Path generatorOutput = new Path(outputPath, uuid.toString());
+
+ Generator generator = new Generator();
+ generator.setConf(getConf());
+ int retCode = generator.run(numMappers, numNodes, numTablets, generatorOutput, width,
+ wrapMultiplier);
+ if (retCode > 0) {
+ throw new RuntimeException("Generator failed with return code: " + retCode);
+ }
+ fs.delete(generatorOutput, true);
+ }
+
+ protected void runVerify(String outputDir,
+ int numReducers,
+ long expectedNumNodes,
+ int retries) throws Exception {
+ // Kudu doesn't fully support snapshot consistency so we might start reading from a node that
+ // doesn't have all the data. This happens often with under "chaos monkey"-type of setups.
+ for (int i = 0; i < retries; i++) {
+ if (i > 0) {
+ long sleep = 60 * 1000;
+ LOG.info("Retrying in " + sleep + "ms");
+ Thread.sleep(sleep);
+ }
+
+ Path outputPath = new Path(outputDir);
+ UUID uuid = UUID.randomUUID(); //create a random UUID.
+ Path iterationOutput = new Path(outputPath, uuid.toString());
+
+ Verify verify = new Verify();
+ verify.setConf(getConf());
+ int retCode = verify.run(iterationOutput, numReducers);
+ if (retCode > 0) {
+ LOG.warn("Verify.run failed with return code: " + retCode);
+ } else if (!verify.verify(expectedNumNodes)) {
+ LOG.warn("Verify.verify failed");
+ } else {
+ fs.delete(iterationOutput, true);
+ LOG.info("Verify finished with success. Total nodes=" + expectedNumNodes);
+ return;
+ }
+ }
+ throw new RuntimeException("Ran out of retries to verify");
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 6) {
+ System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> " +
+ "<num_tablets> <output dir> <num reducers> [<width> <wrap multiplier>" +
+ "<start expected nodes> <num_verify_retries>]");
+ return 1;
+ }
+ LOG.info("Running Loop with args:" + Arrays.deepToString(args));
+
+ int numIterations = Integer.parseInt(args[0]);
+ int numMappers = Integer.parseInt(args[1]);
+ long numNodes = Long.parseLong(args[2]);
+ int numTablets = Integer.parseInt(args[3]);
+ String outputDir = args[4];
+ int numReducers = Integer.parseInt(args[5]);
+ Integer width = (args.length < 7) ? null : Integer.parseInt(args[6]);
+ Integer wrapMultiplier = (args.length < 8) ? null : Integer.parseInt(args[7]);
+ long expectedNumNodes = (args.length < 9) ? 0 : Long.parseLong(args[8]);
+ int numVerifyRetries = (args.length < 10) ? 3 : Integer.parseInt(args[9]);
+
+ if (numIterations < 0) {
+ numIterations = Integer.MAX_VALUE; // run indefinitely (kind of)
+ }
+
+ fs = FileSystem.get(getConf());
+
+ for (int i = 0; i < numIterations; i++) {
+ LOG.info("Starting iteration = " + i);
+ runGenerator(numMappers, numNodes, numTablets, outputDir, width, wrapMultiplier);
+ expectedNumNodes += numMappers * numNodes;
+
+ runVerify(outputDir, numReducers, expectedNumNodes, numVerifyRetries);
+ }
+
+ return 0;
+ }
+ }
+
+ /**
+ * A stand alone program that prints out portions of a list created by {@link Generator}
+ */
+ private static class Print extends Configured implements Tool {
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+ options.addOption("s", "start", true, "start key, only the first component");
+ options.addOption("e", "end", true, "end key (exclusive), only the first component");
+ options.addOption("l", "limit", true, "number to print");
+
+ GnuParser parser = new GnuParser();
+ CommandLine cmd = null;
+ try {
+ cmd = parser.parse(options, args);
+ if (cmd.getArgs().length != 0) {
+ throw new ParseException("Command takes no arguments");
+ }
+ } catch (ParseException e) {
+ System.err.println("Failed to parse command line " + e.getMessage());
+ System.err.println();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(getClass().getSimpleName(), options);
+ System.exit(-1);
+ }
+
+ CommandLineParser cmdLineParser = new CommandLineParser(getConf());
+ long timeout = cmdLineParser.getOperationTimeoutMs();
+ KuduClient client = cmdLineParser.getClient();
+
+ KuduTable table = client.openTable(getTableName(getConf()));
+ KuduScanner.KuduScannerBuilder builder =
+ client.newScannerBuilder(table)
+ .scanRequestTimeout(timeout);
+
+
+ if (cmd.hasOption("s")) {
+ PartialRow row = table.getSchema().newPartialRow();
+ row.addLong(0, Long.parseLong(cmd.getOptionValue("s")));
+ builder.lowerBound(row);
+ }
+ if (cmd.hasOption("e")) {
+ PartialRow row = table.getSchema().newPartialRow();
+ row.addLong(0, Long.parseLong(cmd.getOptionValue("e")));
+ builder.exclusiveUpperBound(row);
+ }
+
+ int limit = cmd.hasOption("l") ? Integer.parseInt(cmd.getOptionValue("l")) : 100;
+
+ int count = 0;
+
+ KuduScanner scanner = builder.build();
+ while (scanner.hasMoreRows() && count < limit) {
+ RowResultIterator rowResults = scanner.nextRows();
+ count = printNodesAndGetNewCount(count, limit, rowResults);
+ }
+ RowResultIterator rowResults = scanner.close();
+ printNodesAndGetNewCount(count, limit, rowResults);
+
+ client.shutdown();
+
+ return 0;
+ }
+
+ private static int printNodesAndGetNewCount(int oldCount, int limit,
+ RowResultIterator rowResults) {
+ int newCount = oldCount;
+ if (rowResults == null) {
+ return newCount;
+ }
+
+ CINode node = new CINode();
+ for (RowResult result : rowResults) {
+ newCount++;
+ node = getCINode(result, node);
+ printCINodeString(node);
+ if (newCount == limit) {
+ break;
+ }
+ }
+ return newCount;
+ }
+ }
+
+ /**
+ * This tool needs to be run separately from the Generator-Verify loop. It can run while the
+ * other two are running or in between loops.
+ *
+ * Each mapper scans a "heads" table and, for each row, follows the circular linked list and
+ * updates their counter until it reaches the head of the list again.
+ */
+ private static class Updater extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(Updater.class);
+
+ private static final String MAX_LINK_UPDATES_PER_MAPPER = "kudu.updates.per.mapper";
+
+ public enum Counts {
+ // Stats on what we're updating.
+ UPDATED_LINKS,
+ UPDATED_NODES,
+ FIRST_UPDATE,
+ SECOND_UPDATE,
+ THIRD_UPDATE,
+ FOURTH_UPDATE,
+ MORE_THAN_FOUR_UPDATES,
+ // Stats on what's broken.
+ BROKEN_LINKS,
+ BAD_UPDATE_COUNTS
+ }
+
+ public static class UpdaterMapper extends Mapper<NullWritable, RowResult,
+ NullWritable, NullWritable> {
+ private KuduClient client;
+ private KuduTable table;
+ private KuduSession session;
+
+ /**
+ * Schema we use when getting rows from the linked list, we only need the reference and
+ * its update count.
+ */
+ private final List<String> SCAN_COLUMN_NAMES = ImmutableList.of(
+ COLUMN_PREV_ONE, COLUMN_PREV_TWO, COLUMN_UPDATE_COUNT, COLUMN_CLIENT);
+
+ private long numUpdatesPerMapper;
+
+ /**
+ * Processing each linked list takes minutes, meaning that it's easily possible for our
+ * scanner to timeout. Instead, we gather all the linked list heads that we need and
+ * process them all at once in the first map invocation.
+ */
+ private List<Pair<Long, Long>> headsCache;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ CommandLineParser parser = new CommandLineParser(conf);
+ client = parser.getClient();
+ try {
+ table = client.openTable(getTableName(conf));
+ } catch (Exception e) {
+ throw new IOException("Couldn't open the linked list table", e);
+ }
+ session = client.newSession();
+
+ Schema tableSchema = table.getSchema();
+
+
+ numUpdatesPerMapper = conf.getLong(MAX_LINK_UPDATES_PER_MAPPER, 1);
+ headsCache = new ArrayList<Pair<Long, Long>>((int)numUpdatesPerMapper);
+ }
+
+ @Override
+ protected void map(NullWritable key, RowResult value, Mapper.Context context)
+ throws IOException, InterruptedException {
+ // Add as many heads as we need, then we skip the rest.
+ do {
+ if (headsCache.size() < numUpdatesPerMapper) {
+ value = (RowResult)context.getCurrentValue();
+ headsCache.add(new Pair<Long, Long>(value.getLong(0), value.getLong(1)));
+ }
+ } while (context.nextKeyValue());
+
+ // At this point we've exhausted the scanner and hopefully gathered all the linked list
+ // heads we needed.
+ LOG.info("Processing " + headsCache.size() +
+ " linked lists, out of " + numUpdatesPerMapper);
+ processAllHeads(context);
+ }
+
+ private void processAllHeads(Mapper.Context context) throws IOException {
+ for (Pair<Long, Long> value : headsCache) {
+ processHead(value, context);
+ }
+ }
+
+ private void processHead(Pair<Long, Long> head, Mapper.Context context) throws IOException {
+ long headKeyOne = head.getFirst();
+ long headKeyTwo = head.getSecond();
+ long prevKeyOne = headKeyOne;
+ long prevKeyTwo = headKeyTwo;
+ int currentCount = -1;
+ int newCount = -1;
+ String client = null;
+
+ // Always printing this out, really useful when debugging.
+ LOG.info("Head: " + getStringFromKeys(headKeyOne, headKeyTwo));
+
+ do {
+ RowResult prev = nextNode(prevKeyOne, prevKeyTwo);
+ if (prev == null) {
+ context.getCounter(Counts.BROKEN_LINKS).increment(1);
+ LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " doesn't exist");
+ break;
+ }
+
+ // It's possible those columns are null, let's not break trying to read them.
+ if (prev.isNull(0) || prev.isNull(1)) {
+ context.getCounter(Counts.BROKEN_LINKS).increment(1);
+ LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " isn't referencing anywhere");
+ break;
+ }
+
+ int prevCount = prev.getInt(2);
+ String prevClient = prev.getString(3);
+ if (currentCount == -1) {
+ // First time we loop we discover what the count was and set the new one.
+ currentCount = prevCount;
+ newCount = currentCount + 1;
+ client = prevClient;
+ }
+
+ if (prevCount != currentCount) {
+ context.getCounter(Counts.BAD_UPDATE_COUNTS).increment(1);
+ LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " has a wrong updateCount, " +
+ prevCount + " instead of " + currentCount);
+ // Game over, there's corruption.
+ break;
+ }
+
+ if (!prevClient.equals(client)) {
+ context.getCounter(Counts.BROKEN_LINKS).increment(1);
+ LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " has the wrong client, " +
+ "bad reference? Bad client= " + prevClient);
+ break;
+ }
+
+ updateRow(prevKeyOne, prevKeyTwo, newCount);
+ context.getCounter(Counts.UPDATED_NODES).increment(1);
+ if (prevKeyOne % 10 == 0) {
+ context.progress();
+ }
+ prevKeyOne = prev.getLong(0);
+ prevKeyTwo = prev.getLong(1);
+ } while (headKeyOne != prevKeyOne && headKeyTwo != prevKeyTwo);
+
+ updateStatCounters(context, newCount);
+ context.getCounter(Counts.UPDATED_LINKS).increment(1);
+ }
+
+ /**
+ * Finds the next node in the linked list.
+ */
+ private RowResult nextNode(long prevKeyOne, long prevKeyTwo) throws IOException {
+ KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table)
+ .setProjectedColumnNames(SCAN_COLUMN_NAMES);
+
+ configureScannerForRandomRead(builder, table, prevKeyOne, prevKeyTwo);
+
+ try {
+ return getOneRowResult(builder.build());
+ } catch (Exception e) {
+ // Goes right out and fails the job.
+ throw new IOException("Couldn't read the following row: " +
+ getStringFromKeys(prevKeyOne, prevKeyTwo), e);
+ }
+ }
+
+ private void updateRow(long keyOne, long keyTwo, int newCount) throws IOException {
+ Update update = table.newUpdate();
+ PartialRow row = update.getRow();
+ row.addLong(COLUMN_KEY_ONE, keyOne);
+ row.addLong(COLUMN_KEY_TWO, keyTwo);
+ row.addInt(COLUMN_UPDATE_COUNT, newCount);
+ try {
+ session.apply(update);
+ } catch (Exception e) {
+ // Goes right out and fails the job.
+ throw new IOException("Couldn't update the following row: " +
+ getStringFromKeys(keyOne, keyTwo), e);
+ }
+ }
+
+ /**
+ * We keep some statistics about the linked list we update so that we can get a feel of
+ * what's being updated.
+ */
+ private void updateStatCounters(Mapper.Context context, int newCount) {
+ switch (newCount) {
+ case -1:
+ case 0:
+ // TODO We didn't event get the first node?
+ break;
+ case 1:
+ context.getCounter(Counts.FIRST_UPDATE).increment(1);
+ break;
+ case 2:
+ context.getCounter(Counts.SECOND_UPDATE).increment(1);
+ break;
+ case 3:
+ context.getCounter(Counts.THIRD_UPDATE).increment(1);
+ break;
+ case 4:
+ context.getCounter(Counts.FOURTH_UPDATE).increment(1);
+ break;
+ default:
+ context.getCounter(Counts.MORE_THAN_FOUR_UPDATES).increment(1);
+ break;
+ }
+ }
+
+ @Override
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ try {
+ session.close();
+ client.shutdown();
+ } catch (Exception ex) {
+ // Goes right out and fails the job.
+ throw new IOException("Coulnd't close the scanner after the task completed", ex);
+ }
+ }
+ }
+
+ public int run(long maxLinkUpdatesPerMapper) throws Exception {
+ LOG.info("Running Updater with maxLinkUpdatesPerMapper=" + maxLinkUpdatesPerMapper);
+
+ Job job = new Job(getConf());
+
+ job.setJobName("Link Updater");
+ job.setNumReduceTasks(0);
+ job.setJarByClass(getClass());
+
+ Joiner columnsToQuery = Joiner.on(",");
+
+ new KuduTableMapReduceUtil.TableInputFormatConfiguratorWithCommandLineParser(
+ job, getHeadsTable(getConf()),
+ columnsToQuery.join(COLUMN_KEY_ONE, COLUMN_KEY_TWO))
+ .configure();
+
+ job.setMapperClass(UpdaterMapper.class);
+ job.setMapOutputKeyClass(BytesWritable.class);
+ job.setMapOutputValueClass(BytesWritable.class);
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
+ // If something fails we want to exit ASAP.
+ job.getConfiguration().setInt("mapreduce.map.maxattempts", 1);
+ // Lack of YARN-445 means we can't auto-jstack on timeout, so disabling the timeout gives
+ // us a chance to do it manually.
+ job.getConfiguration().setInt("mapreduce.task.timeout", 0);
+ job.getConfiguration().setLong(MAX_LINK_UPDATES_PER_MAPPER, maxLinkUpdatesPerMapper);
+
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+
+ KuduTableMapReduceUtil.addDependencyJars(job);
+
+ boolean success = job.waitForCompletion(true);
+
+ Counters counters = job.getCounters();
+
+ if (success) {
+ // Let's not continue looping if we have broken linked lists.
+ Counter brokenLinks = counters.findCounter(Counts.BROKEN_LINKS);
+ Counter badUpdates = counters.findCounter(Counts.BAD_UPDATE_COUNTS);
+ if (brokenLinks.getValue() > 0 || badUpdates.getValue() > 0) {
+ LOG.error("Corruption was detected, see the job's counters. Ending the update loop.");
+ success = false;
+ }
+ }
+ return success ? 0 : 1;
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 2) {
+ System.err.println("Usage: Update <num iterations> <max link updates per mapper>");
+ System.err.println(" where <num iterations> will be 'infinite' if passed a negative value" +
+ " or zero");
+ return 1;
+ }
+ LOG.info("Running Loop with args:" + Arrays.deepToString(args));
+
+ int numIterations = Integer.parseInt(args[0]);
+ long maxUpdates = Long.parseLong(args[1]);
+
+ if (numIterations <= 0) {
+ numIterations = Integer.MAX_VALUE;
+ }
+
+ if (maxUpdates < 1) {
+ maxUpdates = 1;
+ }
+
+ for (int i = 0; i < numIterations; i++) {
+ LOG.info("Starting iteration = " + i);
+ int ret = run(maxUpdates);
+ if (ret != 0) {
+ LOG.error("Can't continue updating, last run failed.");
+ return ret;
+ }
+ }
+
+ return 0;
+ }
+ }
+
+ /**
+ * A stand alone program that deletes a single node.
+ * TODO
+ */
+ /*private static class Delete extends Configured implements Tool {
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length != 1) {
+ System.out.println("Usage : " + Delete.class.getSimpleName() + " <node to delete>");
+ return 0;
+ }
+ byte[] val = Bytes.toBytesBinary(args[0]);
+
+ org.apache.hadoop.hbase.client.Delete delete
+ = new org.apache.hadoop.hbase.client.Delete(val);
+
+ HTable table = new HTable(getConf(), getTableName(getConf()));
+
+ table.delete(delete);
+ table.flushCommits();
+ table.close();
+
+ System.out.println("Delete successful");
+ return 0;
+ }
+ }*/
+
+ /**
+ * A stand alone program that follows a linked list created by {@link Generator}
+ * and prints timing info.
+ *
+ */
+ private static class Walker extends Configured implements Tool {
+
+ private KuduClient client;
+ private KuduTable table;
+
+ @Override
+ public int run(String[] args) throws IOException {
+ if (args.length < 1) {
+ System.err.println("Usage: Walker <start key> [<num nodes>]");
+ System.err.println(" where <num nodes> defaults to 100 nodes that will be printed out");
+ return 1;
+ }
+ int maxNumNodes = 100;
+ if (args.length == 2) {
+ maxNumNodes = Integer.parseInt(args[1]);
+ }
+ System.out.println("Running Walker with args:" + Arrays.deepToString(args));
+
+ String[] keys = args[0].split(",");
+ if (keys.length != 2) {
+ System.err.println("The row key must be formatted like key1,key2");
+ return 1;
+ }
+
+ long keyOne = Long.parseLong(keys[0]);
+ long keyTwo = Long.parseLong(keys[1]);
+
+ System.out.println("Walking with " + getStringFromKeys(keyOne, keyTwo));
+
+ try {
+ walk(keyOne, keyTwo, maxNumNodes);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return 0;
+ }
+
+ private void walk(long headKeyOne, long headKeyTwo, int maxNumNodes) throws Exception {
+ CommandLineParser parser = new CommandLineParser(getConf());
+ client = parser.getClient();
+ table = client.openTable(getTableName(getConf()));
+
+ long prevKeyOne = headKeyOne;
+ long prevKeyTwo = headKeyTwo;
+ CINode node = new CINode();
+ int nodesCount = 0;
+
+ do {
+ RowResult rr = nextNode(prevKeyOne, prevKeyTwo);
+ if (rr == null) {
+ System.err.println(getStringFromKeys(prevKeyOne, prevKeyTwo) + " doesn't exist!");
+ break;
+ }
+ getCINode(rr, node);
+ printCINodeString(node);
+ if (rr.isNull(2) || rr.isNull(3)) {
+ System.err.println("Last node didn't have a reference, breaking");
+ break;
+ }
+ prevKeyOne = rr.getLong(2);
+ prevKeyTwo = rr.getLong(3);
+ nodesCount++;
+ } while ((headKeyOne != prevKeyOne && headKeyTwo != prevKeyTwo) && (nodesCount <
+ maxNumNodes));
+ }
+
+ private RowResult nextNode(long keyOne, long keyTwo) throws Exception {
+ KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table);
+ configureScannerForRandomRead(builder, table, keyOne, keyTwo);
+
+ return getOneRowResult(builder.build());
+ }
+ }
+
+ private static void configureScannerForRandomRead(AbstractKuduScannerBuilder builder,
+ KuduTable table,
+ long keyOne,
+ long keyTwo) {
+ PartialRow lowerBound = table.getSchema().newPartialRow();
+ lowerBound.addLong(0, keyOne);
+ lowerBound.addLong(1, keyTwo);
+ builder.lowerBound(lowerBound);
+
+ PartialRow upperBound = table.getSchema().newPartialRow();
+ // Adding 1 since we want a single row, and the upper bound is exclusive.
+ upperBound.addLong(0, keyOne + 1);
+ upperBound.addLong(1, keyTwo + 1);
+ builder.exclusiveUpperBound(upperBound);
+ }
+
+ private static String getTableName(Configuration conf) {
+ return conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME);
+ }
+
+ private static String getHeadsTable(Configuration conf) {
+ return conf.get(HEADS_TABLE_NAME_KEY, DEFAULT_HEADS_TABLE_NAME);
+ }
+
+ private static CINode getCINode(RowResult result, CINode node) {
+
+ node.key = getStringFromKeys(result.getLong(0), result.getLong(1));
+ if (result.isNull(2) || result.isNull(3)) {
+ node.prev = "NO_REFERENCE";
+ } else {
+ node.prev = getStringFromKeys(result.getLong(2), result.getLong(3));
+ }
+ node.rowId = result.getInt(4);
+ node.client = result.getString(5);
+ node.updateCount = result.getInt(6);
+ return node;
+ }
+
+ private static void printCINodeString(CINode node) {
+ System.out.printf("%s:%s:%012d:%s:%s\n", node.key, node.prev, node.rowId, node.client,
+ node.updateCount);
+ }
+
+ private static String getStringFromKeys(long key1, long key2) {
+ return new StringBuilder().append(key1).append(",").append(key2).toString();
+ }
+
+ private static RowResult getOneRowResult(KuduScanner scanner) throws Exception {
+ RowResultIterator rowResults;
+ rowResults = scanner.nextRows();
+ if (rowResults.getNumRows() == 0) {
+ return null;
+ }
+ if (rowResults.getNumRows() > 1) {
+ throw new Exception("Received too many rows from scanner " + scanner);
+ }
+ return rowResults.next();
+ }
+
+ private void usage() {
+ System.err.println("Usage: " + this.getClass().getSimpleName() + " COMMAND [COMMAND options]");
+ System.err.println(" where COMMAND is one of:");
+ System.err.println("");
+ System.err.println(" Generator A map only job that generates data.");
+ System.err.println(" Verify A map reduce job that looks for holes");
+ System.err.println(" Look at the counts after running");
+ System.err.println(" REFERENCED and UNREFERENCED are ok");
+ System.err.println(" any UNDEFINED counts are bad. Do not");
+ System.err.println(" run at the same time as the Generator.");
+ System.err.println(" Print A standalone program that prints nodes");
+ System.err.println(" in the linked list.");
+ System.err.println(" Loop A program to Loop through Generator and");
+ System.err.println(" Verify steps");
+ System.err.println(" Update A program to updade the nodes");
+ /* System.err.println(" Delete A standalone program that deletes a");
+ System.err.println(" single node.");*/
+ System.err.println(" Walker A standalong program that starts ");
+ System.err.println(" following a linked list");
+ System.err.println("\t ");
+ System.err.flush();
+ }
+
+ protected void processOptions(String[] args) {
+ //get the class, run with the conf
+ if (args.length < 1) {
+ usage();
+ throw new RuntimeException("Incorrect Number of args.");
+ }
+ toRun = args[0];
+ otherArgs = Arrays.copyOfRange(args, 1, args.length);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Tool tool = null;
+ processOptions(args);
+ if (toRun.equals("Generator")) {
+ tool = new Generator();
+ } else if (toRun.equals("Verify")) {
+ tool = new Verify();
+ } else if (toRun.equals("Loop")) {
+ Loop loop = new Loop();
+ loop.it = this;
+ tool = loop;
+
+ } else if (toRun.equals("Print")) {
+ tool = new Print();
+ } else if (toRun.equals("Update")) {
+ tool = new Updater();
+ } else if (toRun.equals("Walker")) {
+ tool = new Walker();
+ } /*else if (toRun.equals("Delete")) {
+ tool = new Delete();
+ }*/ else {
+ usage();
+ throw new RuntimeException("Unknown arg");
+ }
+
+ return ToolRunner.run(getConf(), tool, otherArgs);
+ }
+
+ private static void setJobConf(Job job, int numMappers, long numNodes,
+ Integer width, Integer wrapMultiplier) {
+ job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
+ job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
+ if (width != null) {
+ job.getConfiguration().setInt(GENERATOR_WIDTH_KEY, width);
+ }
+ if (wrapMultiplier != null) {
+ job.getConfiguration().setInt(GENERATOR_WRAP_KEY, wrapMultiplier);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ int ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
+ System.exit(ret);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java
new file mode 100644
index 0000000..45e7837
--- /dev/null
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce.tools;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.mapreduce.CommandLineParser;
+import org.kududb.mapreduce.KuduTableMapReduceUtil;
+import org.kududb.client.RowResult;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import java.io.IOException;
+
+/**
+ * Map-only job that counts all the rows in the provided table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class RowCounter extends Configured implements Tool {
+
+ static final String NAME = "rowcounter";
+ static final String COLUMN_PROJECTION_KEY = "rowcounter.column.projection";
+
+ /** Counter enumeration to count the actual rows. */
+ public static enum Counters { ROWS }
+
+ /**
+ * Simple row counter
+ */
+ static class RowCounterMapper extends
+ Mapper<NullWritable, RowResult, NullWritable, NullWritable> {
+
+ @Override
+ protected void map(NullWritable key, RowResult value, Context context) throws IOException,
+ InterruptedException {
+ context.getCounter(Counters.ROWS).increment(1);
+ }
+ }
+
+ /**
+ * Sets up the actual job.
+ *
+ * @param conf The current configuration.
+ * @param args The command line parameters.
+ * @return The newly created job.
+ * @throws java.io.IOException When setting up the job fails.
+ */
+ @SuppressWarnings("deprecation")
+ public static Job createSubmittableJob(Configuration conf, String[] args)
+ throws IOException, ClassNotFoundException {
+
+ String columnProjection = conf.get(COLUMN_PROJECTION_KEY);
+
+ Class<RowCounterMapper> mapperClass = RowCounterMapper.class;
+ String tableName = args[0];
+
+ String jobName = NAME + "_" + tableName;
+ Job job = new Job(conf, jobName);
+ job.setJarByClass(mapperClass);
+ job.setMapperClass(mapperClass);
+ job.setNumReduceTasks(0);
+ job.setOutputFormatClass(NullOutputFormat.class);
+ new KuduTableMapReduceUtil.TableInputFormatConfiguratorWithCommandLineParser(
+ job,
+ tableName,
+ columnProjection)
+ .configure();
+ return job;
+ }
+
+ /*
+ * @param errorMsg Error message. Can be null.
+ */
+ private static void usage(final String errorMsg) {
+ if (errorMsg != null && errorMsg.length() > 0) {
+ System.err.println("ERROR: " + errorMsg);
+ }
+ String usage =
+ "Usage: " + NAME + " <table.name>\n\n" +
+ "Counts all the rows in the given table.\n" +
+ "\n" +
+ "Other options that may be specified with -D include:\n" +
+ " -D" + COLUMN_PROJECTION_KEY + "=a,b,c - comma-separated list of columns to read " +
+ "as part of the row count. By default, none are read so that the count is as fast " +
+ "as possible. When specifying columns that are keys, they must be at the beginning" +
+ ".\n" +
+ CommandLineParser.getHelpSnippet();
+
+ System.err.println(usage);
+ }
+
+ @Override
+ public int run(String[] otherArgs) throws Exception {
+ if (otherArgs.length != 1) {
+ usage("Wrong number of arguments: " + otherArgs.length);
+ return -1;
+ }
+ Job job = createSubmittableJob(getConf(), otherArgs);
+ return job.waitForCompletion(true) ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int status = ToolRunner.run(new RowCounter(), args);
+ System.exit(status);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/CsvParser.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/CsvParser.java b/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/CsvParser.java
deleted file mode 100644
index 945b82c..0000000
--- a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/CsvParser.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.mapreduce.tools;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Bytes;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Column-separated values parser that gives access to the different columns inside each line of
- * data.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class CsvParser {
-
- private final byte separatorByte;
-
- private final int maxColumnCount;
-
- private final List<String> columnNames;
-
- /**
- * @param columnsSpecification the list of columns to parse out, comma separated.
- * @param separatorStr The 1 byte separator.
- */
- public CsvParser(String columnsSpecification, String separatorStr) {
- // Configure separator
- byte[] separator = Bytes.fromString(separatorStr);
- Preconditions.checkArgument(separator.length == 1, "CsvParser only supports single-byte " +
- "separators");
- separatorByte = separator[0];
-
- // Configure columns
- columnNames = Lists.newArrayList(Splitter.on(',').trimResults().split(columnsSpecification));
-
- maxColumnCount = columnNames.size();
- }
-
- /**
- * Creates a ParsedLine of a line of data.
- * @param lineBytes Whole line as a byte array.
- * @param length How long the line really is in the byte array
- * @return A parsed line of CSV.
- * @throws BadCsvLineException
- */
- public ParsedLine parse(byte[] lineBytes, int length) throws BadCsvLineException {
- // Enumerate separator offsets
- List<Integer> tabOffsets = new ArrayList<Integer>(maxColumnCount);
- for (int i = 0; i < length; i++) {
- if (lineBytes[i] == separatorByte) {
- tabOffsets.add(i);
- }
- }
- if (tabOffsets.isEmpty()) {
- throw new BadCsvLineException("No delimiter");
- }
-
- // trailing separator shouldn't count as a column
- if (lineBytes[length - 1] != separatorByte) {
- tabOffsets.add(length);
- }
-
- if (tabOffsets.size() > maxColumnCount) {
- throw new BadCsvLineException("Excessive columns");
- }
-
- if (tabOffsets.size() < maxColumnCount) {
- throw new BadCsvLineException("Not enough columns");
- }
-
- return new ParsedLine(tabOffsets, lineBytes);
- }
-
- /**
- * Helper class that knows where the columns are situated in the line.
- */
- class ParsedLine {
- private final List<Integer> tabOffsets;
- private final byte[] lineBytes;
-
- ParsedLine(List<Integer> tabOffsets, byte[] lineBytes) {
- this.tabOffsets = tabOffsets;
- this.lineBytes = lineBytes;
- }
-
- /**
- * Get the position for the given column.
- * @param idx Column to lookup.
- * @return Offset in the line.
- */
- public int getColumnOffset(int idx) {
- if (idx > 0) {
- return tabOffsets.get(idx - 1) + 1;
- } else {
- return 0;
- }
- }
-
- /**
- * Get how many bytes the given column occupies.
- * @param idx Column to lookup.
- * @return Column's length.
- */
- public int getColumnLength(int idx) {
- return tabOffsets.get(idx) - getColumnOffset(idx);
- }
-
- /**
- * Get the number of columns in this file.
- * @return Number of columns.
- */
- public int getColumnCount() {
- return tabOffsets.size();
- }
-
- /**
- * Get the bytes originally given for this line.
- * @return Original byte array.
- */
- public byte[] getLineBytes() {
- return lineBytes;
- }
-
- /**
- * Get the given column's name.
- * @param idx Column to lookup.
- * @return Column's name.
- */
- public String getColumnName(int idx) {
- return columnNames.get(idx);
- }
- }
-
- /**
- * Exception used when the CsvParser is unable to parse a line.
- */
- @SuppressWarnings("serial")
- public static class BadCsvLineException extends Exception {
- public BadCsvLineException(String err) {
- super(err);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsv.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsv.java b/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsv.java
deleted file mode 100644
index f407c5c..0000000
--- a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsv.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.mapreduce.tools;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-
-/**
- * Map-only job that reads CSV files and inserts them into a single Kudu table.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class ImportCsv extends Configured implements Tool {
-
- public static enum Counters { BAD_LINES };
-
- static final String NAME = "importcsv";
- static final String DEFAULT_SEPARATOR = "\t";
- static final String SEPARATOR_CONF_KEY = "importcsv.separator";
- static final String JOB_NAME_CONF_KEY = "importcsv.job.name";
- static final String SKIP_LINES_CONF_KEY = "importcsv.skip.bad.lines";
- static final String COLUMNS_NAMES_KEY = "importcsv.column.names";
-
- /**
- * Sets up the actual job.
- *
- * @param conf The current configuration.
- * @param args The command line parameters.
- * @return The newly created job.
- * @throws java.io.IOException When setting up the job fails.
- */
- @SuppressWarnings("deprecation")
- public static Job createSubmittableJob(Configuration conf, String[] args)
- throws IOException, ClassNotFoundException {
-
- Class<ImportCsvMapper> mapperClass = ImportCsvMapper.class;
- conf.set(COLUMNS_NAMES_KEY, args[0]);
- String tableName = args[1];
- Path inputDir = new Path(args[2]);
-
- String jobName = conf.get(JOB_NAME_CONF_KEY, NAME + "_" + tableName);
- Job job = new Job(conf, jobName);
- job.setJarByClass(mapperClass);
- FileInputFormat.setInputPaths(job, inputDir);
- job.setInputFormatClass(TextInputFormat.class);
- job.setMapperClass(mapperClass);
- job.setNumReduceTasks(0);
- new KuduTableMapReduceUtil.TableOutputFormatConfiguratorWithCommandLineParser(
- job,
- tableName)
- .configure();
- return job;
- }
-
- /*
- * @param errorMsg Error message. Can be null.
- */
- private static void usage(final String errorMsg) {
- if (errorMsg != null && errorMsg.length() > 0) {
- System.err.println("ERROR: " + errorMsg);
- }
- String usage =
- "Usage: " + NAME + " <colAa,colB,colC> <table.name> <input.dir>\n\n" +
- "Imports the given input directory of CSV data into the specified table.\n" +
- "\n" +
- "The column names of the CSV data must be specified in the form of " +
- "comma-separated column names.\n" +
- "Other options that may be specified with -D include:\n" +
- " -D" + SKIP_LINES_CONF_KEY + "=false - fail if encountering an invalid line\n" +
- " '-D" + SEPARATOR_CONF_KEY + "=|' - eg separate on pipes instead of tabs\n" +
- " -D" + JOB_NAME_CONF_KEY + "=jobName - use the specified mapreduce job name for the" +
- " import.\n" +
- CommandLineParser.getHelpSnippet();
-
- System.err.println(usage);
- }
-
- @Override
- public int run(String[] otherArgs) throws Exception {
- if (otherArgs.length < 3) {
- usage("Wrong number of arguments: " + otherArgs.length);
- return -1;
- }
- Job job = createSubmittableJob(getConf(), otherArgs);
- return job.waitForCompletion(true) ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- int status = ToolRunner.run(new ImportCsv(), args);
- System.exit(status);
- }
-}
[13/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java b/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
deleted file mode 100644
index dafb6fc..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
+++ /dev/null
@@ -1,879 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-package org.kududb.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.stumbleupon.async.Deferred;
-
-import org.jboss.netty.handler.timeout.ReadTimeoutException;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.rpc.RpcHeader;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.Pair;
-
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelEvent;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelFutureListener;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.Channels;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.handler.codec.replay.ReplayingDecoder;
-import org.jboss.netty.handler.codec.replay.VoidEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.security.sasl.SaslException;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Stateful handler that manages a connection to a specific TabletServer.
- * <p>
- * This handler manages the RPC IDs, the serialization and de-serialization of
- * RPC requests and responses, and keeps track of the RPC in flights for which
- * a response is currently awaited, as well as temporarily buffered RPCs that
- * are awaiting to be sent to the network.
- * <p>
- * This class needs careful synchronization. It's a non-sharable handler,
- * meaning there is one instance of it per Netty {@link Channel} and each
- * instance is only used by one Netty IO thread at a time. At the same time,
- * {@link AsyncKuduClient} calls methods of this class from random threads at
- * random times. The bottom line is that any data only used in the Netty IO
- * threads doesn't require synchronization, everything else does.
- * <p>
- * Acquiring the monitor on an object of this class will prevent it from
- * accepting write requests as well as buffering requests if the underlying
- * channel isn't connected.
- */
-@InterfaceAudience.Private
-public class TabletClient extends ReplayingDecoder<VoidEnum> {
-
- public static final Logger LOG = LoggerFactory.getLogger(TabletClient.class);
-
- private ArrayList<KuduRpc<?>> pending_rpcs;
-
- public static final byte RPC_CURRENT_VERSION = 9;
- /** Initial part of the header for 0.95 and up. */
- private static final byte[] RPC_HEADER = new byte[] { 'h', 'r', 'p', 'c',
- RPC_CURRENT_VERSION, // RPC version.
- 0,
- 0
- };
- public static final int CONNECTION_CTX_CALL_ID = -3;
-
- /**
- * A monotonically increasing counter for RPC IDs.
- * RPCs can be sent out from any thread, so we need an atomic integer.
- * RPC IDs can be arbitrary. So it's fine if this integer wraps around and
- * becomes negative. They don't even have to start at 0, but we do it for
- * simplicity and ease of debugging.
- */
- private final AtomicInteger rpcid = new AtomicInteger(-1);
-
- /**
- * The channel we're connected to.
- * This will be {@code null} while we're not connected to the TabletServer.
- * This attribute is volatile because {@link #shutdown} may access it from a
- * different thread, and because while we connect various user threads will
- * test whether it's {@code null}. Once we're connected and we know what
- * protocol version the server speaks, we'll set this reference.
- */
- private volatile Channel chan;
-
- /**
- * Set to {@code true} once we've disconnected from the server.
- * This way, if any thread is still trying to use this client after it's
- * been removed from the caches in the {@link AsyncKuduClient}, we will
- * immediately fail / reschedule its requests.
- * <p>
- * Manipulating this value requires synchronizing on `this'.
- */
- private boolean dead = false;
-
- /**
- * Maps an RPC ID to the in-flight RPC that was given this ID.
- * RPCs can be sent out from any thread, so we need a concurrent map.
- */
- private final ConcurrentHashMap<Integer, KuduRpc<?>> rpcs_inflight = new ConcurrentHashMap<>();
-
- private final AsyncKuduClient kuduClient;
-
- private final String uuid;
-
- private final String host;
-
- private final int port;
-
- private final long socketReadTimeoutMs;
-
- private SecureRpcHelper secureRpcHelper;
-
- private final RequestTracker requestTracker;
-
- public TabletClient(AsyncKuduClient client, String uuid, String host, int port) {
- this.kuduClient = client;
- this.uuid = uuid;
- this.socketReadTimeoutMs = client.getDefaultSocketReadTimeoutMs();
- this.host = host;
- this.port = port;
- this.requestTracker = client.getRequestTracker();
- }
-
- <R> void sendRpc(KuduRpc<R> rpc) {
- if (!rpc.deadlineTracker.hasDeadline()) {
- LOG.warn(getPeerUuidLoggingString() + " sending an rpc without a timeout " + rpc);
- }
- Pair<ChannelBuffer, Integer> encodedRpcAndId = null;
- if (chan != null) {
- if (!rpc.getRequiredFeatures().isEmpty() &&
- !secureRpcHelper.getServerFeatures().contains(
- RpcHeader.RpcFeatureFlag.APPLICATION_FEATURE_FLAGS)) {
- Status statusNotSupported = Status.NotSupported("the server does not support the" +
- "APPLICATION_FEATURE_FLAGS RPC feature");
- rpc.errback(new NonRecoverableException(statusNotSupported));
- }
-
- encodedRpcAndId = encode(rpc);
- if (encodedRpcAndId == null) { // Error during encoding.
- return; // Stop here. RPC has been failed already.
- }
-
- final Channel chan = this.chan; // Volatile read.
- if (chan != null) { // Double check if we disconnected during encode().
- Channels.write(chan, encodedRpcAndId.getFirst());
- return;
- }
- }
- boolean tryAgain = false; // True when we notice we are about to get connected to the TS.
- boolean failRpc = false; // True when the connection was closed while encoding.
- synchronized (this) {
- // Check if we got connected while entering this synchronized block.
- if (chan != null) {
- tryAgain = true;
- // Check if we got disconnected.
- } else if (dead) {
- // We got disconnected during the process of encoding this rpc, but we need to check if
- // cleanup() already took care of calling failOrRetryRpc() for us. If it did, the entry we
- // added in rpcs_inflight will be missing. If not, we have to call failOrRetryRpc()
- // ourselves after this synchronized block.
- // `encodedRpcAndId` is null iff `chan` is null.
- if (encodedRpcAndId == null || rpcs_inflight.containsKey(encodedRpcAndId.getSecond())) {
- failRpc = true;
- }
- } else {
- if (pending_rpcs == null) {
- pending_rpcs = new ArrayList<>();
- }
- pending_rpcs.add(rpc);
- }
- }
-
- if (failRpc) {
- Status statusNetworkError =
- Status.NetworkError(getPeerUuidLoggingString() + "Connection reset on " + chan);
- failOrRetryRpc(rpc, new RecoverableException(statusNetworkError));
- } else if (tryAgain) {
- // This recursion will not lead to a loop because we only get here if we
- // connected while entering the synchronized block above. So when trying
- // a second time, we will either succeed to send the RPC if we're still
- // connected, or fail through to the code below if we got disconnected
- // in the mean time.
- sendRpc(rpc);
- }
- }
-
- private <R> Pair<ChannelBuffer, Integer> encode(final KuduRpc<R> rpc) {
- final int rpcid = this.rpcid.incrementAndGet();
- ChannelBuffer payload;
- final String service = rpc.serviceName();
- final String method = rpc.method();
- try {
- final RpcHeader.RequestHeader.Builder headerBuilder = RpcHeader.RequestHeader.newBuilder()
- .setCallId(rpcid)
- .addAllRequiredFeatureFlags(rpc.getRequiredFeatures())
- .setRemoteMethod(
- RpcHeader.RemoteMethodPB.newBuilder().setServiceName(service).setMethodName(method));
-
- // If any timeout is set, find the lowest non-zero one, since this will be the deadline that
- // the server must respect.
- if (rpc.deadlineTracker.hasDeadline() || socketReadTimeoutMs > 0) {
- long millisBeforeDeadline = Long.MAX_VALUE;
- if (rpc.deadlineTracker.hasDeadline()) {
- millisBeforeDeadline = rpc.deadlineTracker.getMillisBeforeDeadline();
- }
-
- long localRpcTimeoutMs = Long.MAX_VALUE;
- if (socketReadTimeoutMs > 0) {
- localRpcTimeoutMs = socketReadTimeoutMs;
- }
-
- headerBuilder.setTimeoutMillis((int) Math.min(millisBeforeDeadline, localRpcTimeoutMs));
- }
-
- if (rpc.isRequestTracked()) {
- RpcHeader.RequestIdPB.Builder requestIdBuilder = RpcHeader.RequestIdPB.newBuilder();
- if (rpc.getSequenceId() == RequestTracker.NO_SEQ_NO) {
- rpc.setSequenceId(requestTracker.newSeqNo());
- }
- requestIdBuilder.setClientId(requestTracker.getClientId());
- requestIdBuilder.setSeqNo(rpc.getSequenceId());
- requestIdBuilder.setAttemptNo(rpc.attempt);
- requestIdBuilder.setFirstIncompleteSeqNo(requestTracker.firstIncomplete());
- headerBuilder.setRequestId(requestIdBuilder);
- }
-
- payload = rpc.serialize(headerBuilder.build());
- } catch (Exception e) {
- LOG.error("Uncaught exception while serializing RPC: " + rpc, e);
- rpc.errback(e); // Make the RPC fail with the exception.
- return null;
- }
- final KuduRpc<?> oldrpc = rpcs_inflight.put(rpcid, rpc);
- if (oldrpc != null) {
- final String wtf = getPeerUuidLoggingString() +
- "WTF? There was already an RPC in flight with"
- + " rpcid=" + rpcid + ": " + oldrpc
- + ". This happened when sending out: " + rpc;
- LOG.error(wtf);
- Status statusIllegalState = Status.IllegalState(wtf);
- // Make it fail. This isn't an expected failure mode.
- oldrpc.errback(new NonRecoverableException(statusIllegalState));
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(getPeerUuidLoggingString() + chan + " Sending RPC #" + rpcid
- + ", payload=" + payload + ' ' + Bytes.pretty(payload));
- }
-
- payload = secureRpcHelper.wrap(payload);
-
- return new Pair<>(payload, rpcid);
- }
-
- /**
- * Quick and dirty way to close a connection to a tablet server, if it wasn't already closed.
- */
- @VisibleForTesting
- void disconnect() {
- Channel chancopy = chan;
- if (chancopy != null && chancopy.isConnected()) {
- Channels.disconnect(chancopy);
- }
- }
-
- /**
- * Forcefully shuts down the connection to this tablet server and fails all the outstanding RPCs.
- * Only use when shutting down a client.
- * @return deferred object to use to track the shutting down of this connection
- */
- public Deferred<Void> shutdown() {
- Status statusNetworkError =
- Status.NetworkError(getPeerUuidLoggingString() + "Client is shutting down");
- NonRecoverableException exception = new NonRecoverableException(statusNetworkError);
- // First, check whether we have RPCs in flight and cancel them.
- for (Iterator<KuduRpc<?>> ite = rpcs_inflight.values().iterator(); ite
- .hasNext();) {
- ite.next().errback(exception);
- ite.remove();
- }
-
- // Same for the pending RPCs.
- synchronized (this) {
- if (pending_rpcs != null) {
- for (Iterator<KuduRpc<?>> ite = pending_rpcs.iterator(); ite.hasNext();) {
- ite.next().errback(exception);
- ite.remove();
- }
- }
- }
-
- final Channel chancopy = chan;
- if (chancopy == null) {
- return Deferred.fromResult(null);
- }
- if (chancopy.isConnected()) {
- Channels.disconnect(chancopy); // ... this is going to set it to null.
- // At this point, all in-flight RPCs are going to be failed.
- }
- if (chancopy.isBound()) {
- Channels.unbind(chancopy);
- }
- // It's OK to call close() on a Channel if it's already closed.
- final ChannelFuture future = Channels.close(chancopy);
- // Now wrap the ChannelFuture in a Deferred.
- final Deferred<Void> d = new Deferred<Void>();
- // Opportunistically check if it's already completed successfully.
- if (future.isSuccess()) {
- d.callback(null);
- } else {
- // If we get here, either the future failed (yeah, that sounds weird)
- // or the future hasn't completed yet (heh).
- future.addListener(new ChannelFutureListener() {
- public void operationComplete(final ChannelFuture future) {
- if (future.isSuccess()) {
- d.callback(null);
- return;
- }
- final Throwable t = future.getCause();
- if (t instanceof Exception) {
- d.callback(t);
- } else {
- // Wrap the Throwable because Deferred doesn't handle Throwables,
- // it only uses Exception.
- Status statusIllegalState = Status.IllegalState("Failed to shutdown: " +
- TabletClient.this);
- d.callback(new NonRecoverableException(statusIllegalState, t));
- }
- }
- });
- }
- return d;
- }
-
- /**
- * The reason we are suppressing the unchecked conversions is because the KuduRpc is coming
- * from a collection that has RPCs with different generics, and there's no way to get "decoded"
- * casted correctly. The best we can do is to rely on the RPC to decode correctly,
- * and to not pass an Exception in the callback.
- */
- @Override
- @SuppressWarnings("unchecked")
- protected Object decode(ChannelHandlerContext ctx, Channel chan, ChannelBuffer buf,
- VoidEnum voidEnum) throws NonRecoverableException {
- final long start = System.nanoTime();
- final int rdx = buf.readerIndex();
- LOG.debug("------------------>> ENTERING DECODE >>------------------");
-
- try {
- buf = secureRpcHelper.handleResponse(buf, chan);
- } catch (SaslException e) {
- String message = getPeerUuidLoggingString() + "Couldn't complete the SASL handshake";
- LOG.error(message);
- Status statusIOE = Status.IOError(message);
- throw new NonRecoverableException(statusIOE, e);
- }
- if (buf == null) {
- return null;
- }
-
- CallResponse response = new CallResponse(buf);
-
- RpcHeader.ResponseHeader header = response.getHeader();
- if (!header.hasCallId()) {
- final int size = response.getTotalResponseSize();
- final String msg = getPeerUuidLoggingString() + "RPC response (size: " + size + ") doesn't"
- + " have a call ID: " + header + ", buf=" + Bytes.pretty(buf);
- LOG.error(msg);
- Status statusIncomplete = Status.Incomplete(msg);
- throw new NonRecoverableException(statusIncomplete);
- }
- final int rpcid = header.getCallId();
-
- @SuppressWarnings("rawtypes")
- final KuduRpc rpc = rpcs_inflight.get(rpcid);
-
- if (rpc == null) {
- final String msg = getPeerUuidLoggingString() + "Invalid rpcid: " + rpcid + " found in "
- + buf + '=' + Bytes.pretty(buf);
- LOG.error(msg);
- Status statusIllegalState = Status.IllegalState(msg);
- // The problem here is that we don't know which Deferred corresponds to
- // this RPC, since we don't have a valid ID. So we're hopeless, we'll
- // never be able to recover because responses are not framed, we don't
- // know where the next response will start... We have to give up here
- // and throw this outside of our Netty handler, so Netty will call our
- // exception handler where we'll close this channel, which will cause
- // all RPCs in flight to be failed.
- throw new NonRecoverableException(statusIllegalState);
- }
-
- Pair<Object, Object> decoded = null;
- Exception exception = null;
- Status retryableHeaderError = Status.OK();
- if (header.hasIsError() && header.getIsError()) {
- RpcHeader.ErrorStatusPB.Builder errorBuilder = RpcHeader.ErrorStatusPB.newBuilder();
- KuduRpc.readProtobuf(response.getPBMessage(), errorBuilder);
- RpcHeader.ErrorStatusPB error = errorBuilder.build();
- if (error.getCode().equals(RpcHeader.ErrorStatusPB.RpcErrorCodePB.ERROR_SERVER_TOO_BUSY)) {
- // We can't return right away, we still need to remove ourselves from 'rpcs_inflight', so we
- // populate 'retryableHeaderError'.
- retryableHeaderError = Status.ServiceUnavailable(error.getMessage());
- } else {
- String message = getPeerUuidLoggingString() +
- "Tablet server sent error " + error.getMessage();
- Status status = Status.RemoteError(message);
- exception = new NonRecoverableException(status);
- LOG.error(message); // can be useful
- }
- } else {
- try {
- decoded = rpc.deserialize(response, this.uuid);
- } catch (Exception ex) {
- exception = ex;
- }
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(getPeerUuidLoggingString() + "rpcid=" + rpcid
- + ", response size=" + (buf.readerIndex() - rdx) + " bytes"
- + ", " + actualReadableBytes() + " readable bytes left"
- + ", rpc=" + rpc);
- }
-
- {
- final KuduRpc<?> removed = rpcs_inflight.remove(rpcid);
- if (removed == null) {
- // The RPC we were decoding was cleaned up already, give up.
- Status statusIllegalState = Status.IllegalState("RPC not found");
- throw new NonRecoverableException(statusIllegalState);
- }
- }
-
- // This check is specifically for the ERROR_SERVER_TOO_BUSY case above.
- if (!retryableHeaderError.ok()) {
- kuduClient.handleRetryableError(rpc, new RecoverableException(retryableHeaderError));
- return null;
- }
-
- // We can get this Message from within the RPC's expected type,
- // so convert it into an exception and nullify decoded so that we use the errback route.
- // Have to do it for both TS and Master errors.
- if (decoded != null) {
- if (decoded.getSecond() instanceof Tserver.TabletServerErrorPB) {
- Tserver.TabletServerErrorPB error = (Tserver.TabletServerErrorPB) decoded.getSecond();
- exception = dispatchTSErrorOrReturnException(rpc, error);
- if (exception == null) {
- // It was taken care of.
- return null;
- } else {
- // We're going to errback.
- decoded = null;
- }
-
- } else if (decoded.getSecond() instanceof Master.MasterErrorPB) {
- Master.MasterErrorPB error = (Master.MasterErrorPB) decoded.getSecond();
- exception = dispatchMasterErrorOrReturnException(rpc, error);
- if (exception == null) {
- // Exception was taken care of.
- return null;
- } else {
- decoded = null;
- }
- }
- }
-
- try {
- if (decoded != null) {
- assert !(decoded.getFirst() instanceof Exception);
- if (kuduClient.isStatisticsEnabled()) {
- rpc.updateStatistics(kuduClient.getStatistics(), decoded.getFirst());
- }
- rpc.callback(decoded.getFirst());
- } else {
- if (kuduClient.isStatisticsEnabled()) {
- rpc.updateStatistics(kuduClient.getStatistics(), null);
- }
- rpc.errback(exception);
- }
- } catch (Exception e) {
- LOG.debug(getPeerUuidLoggingString() + "Unexpected exception while handling RPC #" + rpcid
- + ", rpc=" + rpc + ", buf=" + Bytes.pretty(buf), e);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("------------------<< LEAVING DECODE <<------------------"
- + " time elapsed: " + ((System.nanoTime() - start) / 1000) + "us");
- }
- return null; // Stop processing here. The Deferred does everything else.
- }
-
- /**
- * Takes care of a few kinds of TS errors that we handle differently, like tablets or leaders
- * moving. Builds and returns an exception if we don't know what to do with it.
- * @param rpc The original RPC call that triggered the error.
- * @param error The error the TS sent.
- * @return An exception if we couldn't dispatch the error, or null.
- */
- private Exception dispatchTSErrorOrReturnException(KuduRpc rpc,
- Tserver.TabletServerErrorPB error) {
- WireProtocol.AppStatusPB.ErrorCode code = error.getStatus().getCode();
- Status status = Status.fromTabletServerErrorPB(error);
- if (error.getCode() == Tserver.TabletServerErrorPB.Code.TABLET_NOT_FOUND) {
- kuduClient.handleTabletNotFound(rpc, new RecoverableException(status), this);
- // we're not calling rpc.callback() so we rely on the client to retry that RPC
- } else if (code == WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE) {
- kuduClient.handleRetryableError(rpc, new RecoverableException(status));
- // The following two error codes are an indication that the tablet isn't a leader.
- } else if (code == WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE ||
- code == WireProtocol.AppStatusPB.ErrorCode.ABORTED) {
- kuduClient.handleNotLeader(rpc, new RecoverableException(status), this);
- } else {
- return new NonRecoverableException(status);
- }
- return null;
- }
-
- /**
- * Provides different handling for various kinds of master errors: re-uses the
- * mechanisms already in place for handling tablet server errors as much as possible.
- * @param rpc The original RPC call that triggered the error.
- * @param error The error the master sent.
- * @return An exception if we couldn't dispatch the error, or null.
- */
- private Exception dispatchMasterErrorOrReturnException(KuduRpc rpc,
- Master.MasterErrorPB error) {
- WireProtocol.AppStatusPB.ErrorCode code = error.getStatus().getCode();
- Status status = Status.fromMasterErrorPB(error);
- if (error.getCode() == Master.MasterErrorPB.Code.NOT_THE_LEADER) {
- kuduClient.handleNotLeader(rpc, new RecoverableException(status), this);
- } else if (code == WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE) {
- if (rpc instanceof GetMasterRegistrationRequest) {
- // Special case:
- // We never want to retry this RPC, we only use it to poke masters to learn where the leader
- // is. If the error is truly non recoverable, it'll be handled later.
- return new RecoverableException(status);
- } else {
- // TODO: This is a crutch until we either don't have to retry RPCs going to the
- // same server or use retry policies.
- kuduClient.handleRetryableError(rpc, new RecoverableException(status));
- }
- } else {
- return new NonRecoverableException(status);
- }
- return null;
- }
-
- /**
- * Decodes the response of an RPC and triggers its {@link Deferred}.
- * <p>
- * This method is used by FrameDecoder when the channel gets
- * disconnected. The buffer for that channel is passed to this method in
- * case there's anything left in it.
- * @param ctx Unused.
- * @param chan The channel on which the response came.
- * @param buf The buffer containing the raw RPC response.
- * @return {@code null}, always.
- */
- @Override
- protected Object decodeLast(final ChannelHandlerContext ctx,
- final Channel chan,
- final ChannelBuffer buf,
- final VoidEnum unused) throws NonRecoverableException {
- // When we disconnect, decodeLast is called instead of decode.
- // We simply check whether there's any data left in the buffer, in which
- // case we attempt to process it. But if there's no data left, then we
- // don't even bother calling decode() as it'll complain that the buffer
- // doesn't contain enough data, which unnecessarily pollutes the logs.
- if (buf.readable()) {
- try {
- return decode(ctx, chan, buf, unused);
- } finally {
- if (buf.readable()) {
- LOG.error(getPeerUuidLoggingString() + "After decoding the last message on " + chan
- + ", there was still some undecoded bytes in the channel's"
- + " buffer (which are going to be lost): "
- + buf + '=' + Bytes.pretty(buf));
- }
- }
- } else {
- return null;
- }
- }
-
- /**
- * Tells whether or not this handler should be used.
- * <p>
- * This method is not synchronized. You need to synchronize on this
- * instance if you need a memory visibility guarantee. You may not need
- * this guarantee if you're OK with the RPC finding out that the connection
- * has been reset "the hard way" and you can retry the RPC. In this case,
- * you can call this method as a hint. After getting the initial exception
- * back, this thread is guaranteed to see this method return {@code false}
- * without synchronization needed.
- * @return {@code false} if this handler is known to have been disconnected
- * from the server and sending an RPC (via {@link #sendRpc} or any other
- * indirect means such as {@code GetTableLocations()}) will fail immediately
- * by having the RPC's {@link Deferred} called back immediately with a
- * {@link RecoverableException}. This typically means that you got a
- * stale reference (or that the reference to this instance is just about to
- * be invalidated) and that you shouldn't use this instance.
- */
- public boolean isAlive() {
- return !dead;
- }
-
- /**
- * Ensures that at least a {@code nbytes} are readable from the given buffer.
- * If there aren't enough bytes in the buffer this will raise an exception
- * and cause the {@link ReplayingDecoder} to undo whatever we did thus far
- * so we can wait until we read more from the socket.
- * @param buf Buffer to check.
- * @param nbytes Number of bytes desired.
- */
- static void ensureReadable(final ChannelBuffer buf, final int nbytes) {
- buf.markReaderIndex();
- buf.skipBytes(nbytes); // can puke with Throwable
- buf.resetReaderIndex();
- }
-
- @Override
- public void channelConnected(final ChannelHandlerContext ctx,
- final ChannelStateEvent e) {
- final Channel chan = e.getChannel();
- ChannelBuffer header = connectionHeaderPreamble();
- header.writerIndex(RPC_HEADER.length);
- Channels.write(chan, header);
-
- secureRpcHelper = new SecureRpcHelper(this);
- secureRpcHelper.sendHello(chan);
- }
-
- @Override
- public void handleUpstream(final ChannelHandlerContext ctx,
- final ChannelEvent e) throws Exception {
- if (LOG.isDebugEnabled()) {
- LOG.debug(getPeerUuidLoggingString() + e.toString());
- }
- super.handleUpstream(ctx, e);
- }
-
- @Override
- public void channelDisconnected(final ChannelHandlerContext ctx,
- final ChannelStateEvent e) throws Exception {
- chan = null;
- super.channelDisconnected(ctx, e); // Let the ReplayingDecoder cleanup.
- cleanup(e.getChannel());
- }
-
- @Override
- public void channelClosed(final ChannelHandlerContext ctx,
- final ChannelStateEvent e) {
- chan = null;
- // No need to call super.channelClosed() because we already called
- // super.channelDisconnected(). If we get here without getting a
- // DISCONNECTED event, then we were never connected in the first place so
- // the ReplayingDecoder has nothing to cleanup.
- cleanup(e.getChannel());
- }
-
- /**
- * Cleans up any outstanding or lingering RPC (used when shutting down).
- * <p>
- * All RPCs in flight will fail with a {@link RecoverableException} and
- * all edits buffered will be re-scheduled.
- */
- private void cleanup(final Channel chan) {
- final ArrayList<KuduRpc<?>> rpcs;
-
- // The timing of this block is critical. If this TabletClient is 'dead' then it means that
- // rpcs_inflight was emptied and that anything added to it after won't be handled and needs
- // to be sent to failOrRetryRpc.
- synchronized (this) {
- // Cleanup can be called multiple times, but we only want to run it once so that we don't
- // clear up rpcs_inflight multiple times.
- if (dead) {
- return;
- }
- dead = true;
- rpcs = pending_rpcs == null ? new ArrayList<KuduRpc<?>>(rpcs_inflight.size()) : pending_rpcs;
-
- for (Iterator<KuduRpc<?>> iterator = rpcs_inflight.values().iterator(); iterator.hasNext();) {
- KuduRpc<?> rpc = iterator.next();
- rpcs.add(rpc);
- iterator.remove();
- }
- // After this, rpcs_inflight might still have entries since they could have been added
- // concurrently, and those RPCs will be handled by their caller in sendRpc.
-
- pending_rpcs = null;
- }
- Status statusNetworkError =
- Status.NetworkError(getPeerUuidLoggingString() + "Connection reset on " + chan);
- RecoverableException exception = new RecoverableException(statusNetworkError);
-
- failOrRetryRpcs(rpcs, exception);
- }
-
- /**
- * Retry all the given RPCs.
- * @param rpcs a possibly empty but non-{@code null} collection of RPCs to retry or fail
- * @param exception an exception to propagate with the RPCs
- */
- private void failOrRetryRpcs(final Collection<KuduRpc<?>> rpcs,
- final RecoverableException exception) {
- for (final KuduRpc<?> rpc : rpcs) {
- failOrRetryRpc(rpc, exception);
- }
- }
-
- /**
- * Retry the given RPC.
- * @param rpc an RPC to retry or fail
- * @param exception an exception to propagate with the RPC
- */
- private void failOrRetryRpc(final KuduRpc<?> rpc,
- final RecoverableException exception) {
- AsyncKuduClient.RemoteTablet tablet = rpc.getTablet();
- // Note As of the time of writing (03/11/16), a null tablet doesn't make sense, if we see a null
- // tablet it's because we didn't set it properly before calling sendRpc().
- if (tablet == null) { // Can't retry, dunno where this RPC should go.
- rpc.errback(exception);
- } else {
- kuduClient.handleRetryableError(rpc, exception);
- }
- }
-
-
- @Override
- public void exceptionCaught(final ChannelHandlerContext ctx,
- final ExceptionEvent event) {
- final Throwable e = event.getCause();
- final Channel c = event.getChannel();
-
- if (e instanceof RejectedExecutionException) {
- LOG.warn(getPeerUuidLoggingString() + "RPC rejected by the executor,"
- + " ignore this if we're shutting down", e);
- } else if (e instanceof ReadTimeoutException) {
- LOG.debug(getPeerUuidLoggingString() + "Encountered a read timeout, will close the channel");
- } else {
- LOG.error(getPeerUuidLoggingString() + "Unexpected exception from downstream on " + c, e);
- // For any other exception, likely a connection error, we clear the leader state
- // for those tablets that this TS is the cached leader of.
- kuduClient.demoteAsLeaderForAllTablets(this);
- }
- if (c.isOpen()) {
- Channels.close(c); // Will trigger channelClosed(), which will cleanup()
- } else { // else: presumably a connection timeout.
- cleanup(c); // => need to cleanup() from here directly.
- }
- }
-
-
- private ChannelBuffer connectionHeaderPreamble() {
- return ChannelBuffers.wrappedBuffer(RPC_HEADER);
- }
-
- public void becomeReady(Channel chan) {
- this.chan = chan;
- sendQueuedRpcs();
- }
-
- /**
- * Sends the queued RPCs to the server, once we're connected to it.
- * This gets called after {@link #channelConnected}, once we were able to
- * handshake with the server
- */
- private void sendQueuedRpcs() {
- ArrayList<KuduRpc<?>> rpcs;
- synchronized (this) {
- rpcs = pending_rpcs;
- pending_rpcs = null;
- }
- if (rpcs != null) {
- for (final KuduRpc<?> rpc : rpcs) {
- LOG.debug(getPeerUuidLoggingString() + "Executing RPC queued: " + rpc);
- sendRpc(rpc);
- }
- }
- }
-
- void sendContext(Channel channel) {
- Channels.write(channel, header());
- becomeReady(channel);
- }
-
- private ChannelBuffer header() {
- RpcHeader.ConnectionContextPB.Builder builder = RpcHeader.ConnectionContextPB.newBuilder();
- RpcHeader.UserInformationPB.Builder userBuilder = RpcHeader.UserInformationPB.newBuilder();
- userBuilder.setEffectiveUser(SecureRpcHelper.USER_AND_PASSWORD); // TODO set real user
- userBuilder.setRealUser(SecureRpcHelper.USER_AND_PASSWORD);
- builder.setUserInfo(userBuilder.build());
- RpcHeader.ConnectionContextPB pb = builder.build();
- RpcHeader.RequestHeader header = RpcHeader.RequestHeader.newBuilder().setCallId
- (CONNECTION_CTX_CALL_ID).build();
- return KuduRpc.toChannelBuffer(header, pb);
- }
-
- private String getPeerUuidLoggingString() {
- return "[Peer " + uuid + "] ";
- }
-
- /**
- * Returns this tablet server's uuid.
- * @return a string that contains this tablet server's uuid
- */
- String getUuid() {
- return uuid;
- }
-
- /**
- * Returns this tablet server's port.
- * @return a port number that this tablet server is bound to
- */
- int getPort() {
- return port;
- }
-
- /**
- * Returns this tablet server's hostname. We might get many hostnames from the master for a single
- * TS, and this is the one we picked to connect to originally.
- * @returna string that contains this tablet server's hostname
- */
- String getHost() {
- return host;
- }
-
- public String toString() {
- final StringBuilder buf = new StringBuilder(13 + 10 + 6 + 64 + 7 + 32 + 16 + 1 + 17 + 2 + 1);
- buf.append("TabletClient@") // =13
- .append(hashCode()) // ~10
- .append("(chan=") // = 6
- .append(chan) // ~64 (up to 66 when using IPv4)
- .append(", uuid=") // = 7
- .append(uuid) // = 32
- .append(", #pending_rpcs="); // =16
- int npending_rpcs;
- synchronized (this) {
- npending_rpcs = pending_rpcs == null ? 0 : pending_rpcs.size();
- }
- buf.append(npending_rpcs); // = 1
- buf.append(", #rpcs_inflight=") // =17
- .append(rpcs_inflight.size()) // ~ 2
- .append(')'); // = 1
- return buf.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Update.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Update.java b/java/kudu-client/src/main/java/org/kududb/client/Update.java
deleted file mode 100644
index 3db2026..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Update.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Operation to update columns on an existing row. Instances of this class should not be reused.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Update extends Operation {
-
- Update(KuduTable table) {
- super(table);
- }
-
- @Override
- ChangeType getChangeType() {
- return ChangeType.UPDATE;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Upsert.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Upsert.java b/java/kudu-client/src/main/java/org/kududb/client/Upsert.java
deleted file mode 100644
index 4ba2635..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Upsert.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Represents a single row upsert. Instances of this class should not be reused.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Upsert extends Operation {
-
- Upsert(KuduTable table) {
- super(table);
- }
-
- @Override
- ChangeType getChangeType() {
- return ChangeType.UPSERT;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/util/AsyncUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/util/AsyncUtil.java b/java/kudu-client/src/main/java/org/kududb/util/AsyncUtil.java
deleted file mode 100644
index a93d1b9..0000000
--- a/java/kudu-client/src/main/java/org/kududb/util/AsyncUtil.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.util;
-
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-
-import org.kududb.annotations.InterfaceAudience;
-
-/**
- * Utility methods for various parts of async, such as Deferred.
- * TODO (KUDU-602): Some of these methods could eventually be contributed back to async or to a
- * custom fork/derivative of async.
- */
-@InterfaceAudience.Private
-public class AsyncUtil {
-
- /**
- * Register a callback and an "errback".
- * <p>
- * This has the exact same effect as {@link Deferred#addCallbacks(Callback, Callback)}
- * keeps the type information "correct" when the callback and errback return a
- * {@code Deferred}.
- * @param d The {@code Deferred} we want to add the callback and errback to.
- * @param cb The callback to register.
- * @param eb The errback to register.
- * @return {@code d} with an "updated" type.
- */
- @SuppressWarnings("unchecked")
- public static <T, R, D extends Deferred<R>, E>
- Deferred<R> addCallbacksDeferring(final Deferred<T> d,
- final Callback<D, T> cb,
- final Callback<D, E> eb) {
- return d.addCallbacks((Callback<R, T>) cb, eb);
- }
-
- /**
- * Workaround for {@link Deferred#addBoth}'s failure to use generics correctly. Allows callers
- * to provide a {@link Callback} which takes an {@link Object} instead of the type of the deferred
- * it is applied to, which avoids a runtime {@link ClassCastException} when the deferred fails.
- */
- @SuppressWarnings("unchecked")
- public static <T, U> Deferred<U> addBoth(final Deferred<T> deferred,
- final Callback<? extends U, Object> callback) {
- return ((Deferred) deferred).addBoth(callback);
- }
-
- /**
- * Workaround for {@link Deferred#addBothDeferring}'s failure to use generics correctly. Allows
- * callers to provide a {@link Callback} which takes an {@link Object} instead of the type of the
- * deferred it is applied to, which avoids a runtime {@link ClassCastException} when the deferred
- * fails.
- */
- @SuppressWarnings("unchecked")
- public static <T, U> Deferred<U> addBothDeferring(final Deferred<T> deferred,
- final Callback<Deferred<U>, Object> callback) {
- return ((Deferred) deferred).addBothDeferring(callback);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/util/HybridTimeUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/util/HybridTimeUtil.java b/java/kudu-client/src/main/java/org/kududb/util/HybridTimeUtil.java
deleted file mode 100644
index 31436e7..0000000
--- a/java/kudu-client/src/main/java/org/kududb/util/HybridTimeUtil.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.util;
-
-import org.kududb.annotations.InterfaceAudience;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Set of common utility methods to handle HybridTime and related timestamps.
- */
-@InterfaceAudience.Private
-public class HybridTimeUtil {
-
- public static final int hybridTimeNumBitsToShift = 12;
- public static final int hybridTimeLogicalBitsMask = (1 << hybridTimeNumBitsToShift) - 1;
-
- /**
- * Converts the provided timestamp, in the provided unit, to the HybridTime timestamp
- * format. Logical bits are set to 0.
- *
- * @param timestamp the value of the timestamp, must be greater than 0
- * @param timeUnit the time unit of the timestamp
- * @throws IllegalArgumentException if the timestamp is less than 0
- */
- public static long clockTimestampToHTTimestamp(long timestamp, TimeUnit timeUnit) {
- if (timestamp < 0) {
- throw new IllegalArgumentException("Timestamp cannot be less than 0");
- }
- long timestampInMicros = TimeUnit.MICROSECONDS.convert(timestamp, timeUnit);
- return timestampInMicros << hybridTimeNumBitsToShift;
- }
-
- /**
- * Extracts the physical and logical values from an HT timestamp.
- *
- * @param htTimestamp the encoded HT timestamp
- * @return a pair of {physical, logical} long values in an array
- */
- public static long[] HTTimestampToPhysicalAndLogical(long htTimestamp) {
- long timestampInMicros = htTimestamp >> hybridTimeNumBitsToShift;
- long logicalValues = htTimestamp & hybridTimeLogicalBitsMask;
- return new long[] {timestampInMicros, logicalValues};
- }
-
- /**
- * Encodes separate physical and logical components into a single HT timestamp
- *
- * @param physical the physical component, in microseconds
- * @param logical the logical component
- * @return an encoded HT timestamp
- */
- public static long physicalAndLogicalToHTTimestamp(long physical, long logical) {
- return (physical << hybridTimeNumBitsToShift) + logical;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/util/NetUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/util/NetUtil.java b/java/kudu-client/src/main/java/org/kududb/util/NetUtil.java
deleted file mode 100644
index 1ff77a2..0000000
--- a/java/kudu-client/src/main/java/org/kududb/util/NetUtil.java
+++ /dev/null
@@ -1,78 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.util;
-
-import com.google.common.base.Functions;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.net.HostAndPort;
-import org.kududb.annotations.InterfaceAudience;
-
-import java.util.List;
-
-/**
- * Networking related methods.
- */
-@InterfaceAudience.Private
-public class NetUtil {
-
- /**
- * Convert a list of {@link HostAndPort} objects to a comma separate string.
- * The inverse of {@link #parseStrings(String, int)}.
- *
- * @param hostsAndPorts A list of {@link HostAndPort} objects.
- * @return Comma separate list of "host:port" pairs.
- */
- public static String hostsAndPortsToString(List<HostAndPort> hostsAndPorts) {
- return Joiner.on(",").join(Lists.transform(hostsAndPorts, Functions.toStringFunction()));
- }
-
- /**
- * Parse a "host:port" pair into a {@link HostAndPort} object. If there is no
- * port specified in the string, then 'defaultPort' is used.
- *
- * @param addrString A host or a "host:port" pair.
- * @param defaultPort Default port to use if no port is specified in addrString.
- * @return The HostAndPort object constructed from addrString.
- */
- public static HostAndPort parseString(String addrString, int defaultPort) {
- return addrString.indexOf(':') == -1 ? HostAndPort.fromParts(addrString, defaultPort) :
- HostAndPort.fromString(addrString);
- }
-
- /**
- * Parse a comma separated list of "host:port" pairs into a list of
- * {@link HostAndPort} objects. If no port is specified for an entry in
- * the comma separated list, then a default port is used.
- * The inverse of {@link #hostsAndPortsToString(List)}.
- *
- * @param commaSepAddrs The comma separated list of "host:port" pairs.
- * @param defaultPort The default port to use if no port is specified.
- * @return A list of HostAndPort objects constructed from commaSepAddrs.
- */
- public static List<HostAndPort> parseStrings(final String commaSepAddrs, int defaultPort) {
- Iterable<String> addrStrings = Splitter.on(',').trimResults().split(commaSepAddrs);
- List<HostAndPort> hostsAndPorts = Lists.newArrayListWithCapacity(Iterables.size(addrStrings));
- for (String addrString : addrStrings) {
- HostAndPort hostAndPort = parseString(addrString, defaultPort);
- hostsAndPorts.add(hostAndPort);
- }
- return hostsAndPorts;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/util/Pair.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/util/Pair.java b/java/kudu-client/src/main/java/org/kududb/util/Pair.java
deleted file mode 100644
index 341ec10..0000000
--- a/java/kudu-client/src/main/java/org/kududb/util/Pair.java
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.util;
-
-import com.google.common.base.Objects;
-import org.kududb.annotations.InterfaceAudience;
-
-@InterfaceAudience.Private
-public class Pair<A, B> {
- private final A first;
- private final B second;
-
- public Pair(A first, B second) {
- this.first = first;
- this.second = second;
- }
-
- public A getFirst() {
- return first;
- }
-
- public B getSecond() {
- return second;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- Pair<?, ?> pair = (Pair<?, ?>) o;
-
- if (first != null ? !first.equals(pair.first) : pair.first != null) return false;
- if (second != null ? !second.equals(pair.second) : pair.second != null) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(first, second);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/util/Slice.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/util/Slice.java b/java/kudu-client/src/main/java/org/kududb/util/Slice.java
deleted file mode 100644
index c9d2719..0000000
--- a/java/kudu-client/src/main/java/org/kududb/util/Slice.java
+++ /dev/null
@@ -1,702 +0,0 @@
-/*
- * Copyright 2009 Red Hat, Inc.
- *
- * Red Hat licenses this file to you under the Apache License, version 2.0
- * (the "License"); you may not use this file except in compliance with the
- * License. You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- *
- * Copyright 2011 Dain Sundstrom <da...@iq80.com>
- * Copyright 2011 FuseSource Corp. http://fusesource.com
- */
-package org.kududb.util;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import com.google.common.primitives.Shorts;
-import org.kududb.annotations.InterfaceAudience;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedChannelException;
-import java.nio.channels.FileChannel;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.ScatteringByteChannel;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-
-import static java.nio.ByteOrder.LITTLE_ENDIAN;
-
-/**
- * Little Endian slice of a byte array.
- */
-@InterfaceAudience.Private
-public final class Slice implements Comparable<Slice>
-{
- private final byte[] data;
- private final int offset;
- private final int length;
-
- private int hash;
-
- public Slice(int length)
- {
- data = new byte[length];
- this.offset = 0;
- this.length = length;
- }
-
- public Slice(byte[] data)
- {
- Preconditions.checkNotNull(data, "array is null");
- this.data = data;
- this.offset = 0;
- this.length = data.length;
- }
-
- public Slice(byte[] data, int offset, int length)
- {
- Preconditions.checkNotNull(data, "array is null");
- this.data = data;
- this.offset = offset;
- this.length = length;
- }
-
- /**
- * Length of this slice.
- */
- public int length()
- {
- return length;
- }
-
- /**
- * Gets the array underlying this slice.
- */
- public byte[] getRawArray()
- {
- return data;
- }
-
- /**
- * Gets the offset of this slice in the underlying array.
- */
- public int getRawOffset()
- {
- return offset;
- }
-
- /**
- * Gets a byte at the specified absolute {@code index} in this buffer.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 1} is greater than {@code this.capacity}
- */
- public byte getByte(int index)
- {
- Preconditions.checkPositionIndexes(index, index + 1, this.length);
- index += offset;
- return data[index];
- }
-
- /**
- * Gets an unsigned byte at the specified absolute {@code index} in this
- * buffer.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 1} is greater than {@code this.capacity}
- */
- public short getUnsignedByte(int index)
- {
- return (short) (getByte(index) & 0xFF);
- }
-
- /**
- * Gets a 16-bit short integer at the specified absolute {@code index} in
- * this slice.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 2} is greater than {@code this.capacity}
- */
- public short getShort(int index)
- {
- Preconditions.checkPositionIndexes(index, index + Shorts.BYTES, this.length);
- index += offset;
- return (short) (data[index] & 0xFF | data[index + 1] << 8);
- }
-
- /**
- * Gets a 32-bit integer at the specified absolute {@code index} in
- * this buffer.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 4} is greater than {@code this.capacity}
- */
- public int getInt(int index)
- {
- Preconditions.checkPositionIndexes(index, index + Ints.BYTES, this.length);
- index += offset;
- return (data[index] & 0xff) |
- (data[index + 1] & 0xff) << 8 |
- (data[index + 2] & 0xff) << 16 |
- (data[index + 3] & 0xff) << 24;
- }
-
- /**
- * Gets a 64-bit long integer at the specified absolute {@code index} in
- * this buffer.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 8} is greater than {@code this.capacity}
- */
- public long getLong(int index)
- {
- Preconditions.checkPositionIndexes(index, index + Longs.BYTES, this.length);
- index += offset;
- return ((long) data[index] & 0xff) |
- ((long) data[index + 1] & 0xff) << 8 |
- ((long) data[index + 2] & 0xff) << 16 |
- ((long) data[index + 3] & 0xff) << 24 |
- ((long) data[index + 4] & 0xff) << 32 |
- ((long) data[index + 5] & 0xff) << 40 |
- ((long) data[index + 6] & 0xff) << 48 |
- ((long) data[index + 7] & 0xff) << 56;
- }
-
- /**
- * Transfers this buffer's data to the specified destination starting at
- * the specified absolute {@code index}.
- *
- * @param dstIndex the first index of the destination
- * @param length the number of bytes to transfer
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
- * if the specified {@code dstIndex} is less than {@code 0},
- * if {@code index + length} is greater than
- * {@code this.capacity}, or
- * if {@code dstIndex + length} is greater than
- * {@code dst.capacity}
- */
- public void getBytes(int index, Slice dst, int dstIndex, int length)
- {
- getBytes(index, dst.data, dstIndex, length);
- }
-
- /**
- * Transfers this buffer's data to the specified destination starting at
- * the specified absolute {@code index}.
- *
- * @param destinationIndex the first index of the destination
- * @param length the number of bytes to transfer
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
- * if the specified {@code dstIndex} is less than {@code 0},
- * if {@code index + length} is greater than
- * {@code this.capacity}, or
- * if {@code dstIndex + length} is greater than
- * {@code dst.length}
- */
- public void getBytes(int index, byte[] destination, int destinationIndex, int length)
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- Preconditions.checkPositionIndexes(destinationIndex, destinationIndex + length, destination.length);
- index += offset;
- System.arraycopy(data, index, destination, destinationIndex, length);
- }
-
- public byte[] getBytes()
- {
- return getBytes(0, length);
- }
-
- public byte[] getBytes(int index, int length)
- {
- index += offset;
- if (index == 0) {
- return Arrays.copyOf(data, length);
- } else {
- byte[] value = new byte[length];
- System.arraycopy(data, index, value, 0, length);
- return value;
- }
- }
-
- /**
- * Transfers this buffer's data to the specified destination starting at
- * the specified absolute {@code index} until the destination's position
- * reaches its limit.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * if {@code index + dst.remaining()} is greater than
- * {@code this.capacity}
- */
- public void getBytes(int index, ByteBuffer destination)
- {
- Preconditions.checkPositionIndex(index, this.length);
- index += offset;
- destination.put(data, index, Math.min(length, destination.remaining()));
- }
-
- /**
- * Transfers this buffer's data to the specified stream starting at the
- * specified absolute {@code index}.
- *
- * @param length the number of bytes to transfer
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * if {@code index + length} is greater than
- * {@code this.capacity}
- * @throws java.io.IOException if the specified stream threw an exception during I/O
- */
- public void getBytes(int index, OutputStream out, int length)
- throws IOException
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- index += offset;
- out.write(data, index, length);
- }
-
- /**
- * Transfers this buffer's data to the specified channel starting at the
- * specified absolute {@code index}.
- *
- * @param length the maximum number of bytes to transfer
- * @return the actual number of bytes written out to the specified channel
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * if {@code index + length} is greater than
- * {@code this.capacity}
- * @throws java.io.IOException if the specified channel threw an exception during I/O
- */
- public int getBytes(int index, GatheringByteChannel out, int length)
- throws IOException
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- index += offset;
- return out.write(ByteBuffer.wrap(data, index, length));
- }
-
- /**
- * Sets the specified 16-bit short integer at the specified absolute
- * {@code index} in this buffer. The 16 high-order bits of the specified
- * value are ignored.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 2} is greater than {@code this.capacity}
- */
- public void setShort(int index, int value)
- {
- Preconditions.checkPositionIndexes(index, index + Shorts.BYTES, this.length);
- index += offset;
- data[index] = (byte) (value);
- data[index + 1] = (byte) (value >>> 8);
- }
-
- /**
- * Sets the specified 32-bit integer at the specified absolute
- * {@code index} in this buffer.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 4} is greater than {@code this.capacity}
- */
- public void setInt(int index, int value)
- {
- Preconditions.checkPositionIndexes(index, index + Ints.BYTES, this.length);
- index += offset;
- data[index] = (byte) (value);
- data[index + 1] = (byte) (value >>> 8);
- data[index + 2] = (byte) (value >>> 16);
- data[index + 3] = (byte) (value >>> 24);
- }
-
- /**
- * Sets the specified 64-bit long integer at the specified absolute
- * {@code index} in this buffer.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 8} is greater than {@code this.capacity}
- */
- public void setLong(int index, long value)
- {
- Preconditions.checkPositionIndexes(index, index + Longs.BYTES, this.length);
- index += offset;
- data[index] = (byte) (value);
- data[index + 1] = (byte) (value >>> 8);
- data[index + 2] = (byte) (value >>> 16);
- data[index + 3] = (byte) (value >>> 24);
- data[index + 4] = (byte) (value >>> 32);
- data[index + 5] = (byte) (value >>> 40);
- data[index + 6] = (byte) (value >>> 48);
- data[index + 7] = (byte) (value >>> 56);
- }
-
- /**
- * Sets the specified byte at the specified absolute {@code index} in this
- * buffer. The 24 high-order bits of the specified value are ignored.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * {@code index + 1} is greater than {@code this.capacity}
- */
- public void setByte(int index, int value)
- {
- Preconditions.checkPositionIndexes(index, index + 1, this.length);
- index += offset;
- data[index] = (byte) value;
- }
-
- /**
- * Transfers the specified source buffer's data to this buffer starting at
- * the specified absolute {@code index}.
- *
- * @param srcIndex the first index of the source
- * @param length the number of bytes to transfer
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
- * if the specified {@code srcIndex} is less than {@code 0},
- * if {@code index + length} is greater than
- * {@code this.capacity}, or
- * if {@code srcIndex + length} is greater than
- * {@code src.capacity}
- */
- public void setBytes(int index, Slice src, int srcIndex, int length)
- {
- setBytes(index, src.data, src.offset + srcIndex, length);
- }
-
- /**
- * Transfers the specified source array's data to this buffer starting at
- * the specified absolute {@code index}.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
- * if the specified {@code srcIndex} is less than {@code 0},
- * if {@code index + length} is greater than
- * {@code this.capacity}, or
- * if {@code srcIndex + length} is greater than {@code src.length}
- */
- public void setBytes(int index, byte[] source, int sourceIndex, int length)
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- Preconditions.checkPositionIndexes(sourceIndex, sourceIndex + length, source.length);
- index += offset;
- System.arraycopy(source, sourceIndex, data, index, length);
- }
-
- /**
- * Transfers the specified source buffer's data to this buffer starting at
- * the specified absolute {@code index} until the source buffer's position
- * reaches its limit.
- *
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * if {@code index + src.remaining()} is greater than
- * {@code this.capacity}
- */
- public void setBytes(int index, ByteBuffer source)
- {
- Preconditions.checkPositionIndexes(index, index + source.remaining(), this.length);
- index += offset;
- source.get(data, index, source.remaining());
- }
-
- /**
- * Transfers the content of the specified source stream to this buffer
- * starting at the specified absolute {@code index}.
- *
- * @param length the number of bytes to transfer
- * @return the actual number of bytes read in from the specified channel.
- * {@code -1} if the specified channel is closed.
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * if {@code index + length} is greater than {@code this.capacity}
- * @throws java.io.IOException if the specified stream threw an exception during I/O
- */
- public int setBytes(int index, InputStream in, int length)
- throws IOException
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- index += offset;
- int readBytes = 0;
- do {
- int localReadBytes = in.read(data, index, length);
- if (localReadBytes < 0) {
- if (readBytes == 0) {
- return -1;
- }
- else {
- break;
- }
- }
- readBytes += localReadBytes;
- index += localReadBytes;
- length -= localReadBytes;
- } while (length > 0);
-
- return readBytes;
- }
-
- /**
- * Transfers the content of the specified source channel to this buffer
- * starting at the specified absolute {@code index}.
- *
- * @param length the maximum number of bytes to transfer
- * @return the actual number of bytes read in from the specified channel.
- * {@code -1} if the specified channel is closed.
- * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
- * if {@code index + length} is greater than {@code this.capacity}
- * @throws java.io.IOException if the specified channel threw an exception during I/O
- */
- public int setBytes(int index, ScatteringByteChannel in, int length)
- throws IOException
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- index += offset;
- ByteBuffer buf = ByteBuffer.wrap(data, index, length);
- int readBytes = 0;
-
- do {
- int localReadBytes;
- try {
- localReadBytes = in.read(buf);
- }
- catch (ClosedChannelException e) {
- localReadBytes = -1;
- }
- if (localReadBytes < 0) {
- if (readBytes == 0) {
- return -1;
- }
- else {
- break;
- }
- }
- else if (localReadBytes == 0) {
- break;
- }
- readBytes += localReadBytes;
- } while (readBytes < length);
-
- return readBytes;
- }
-
- public int setBytes(int index, FileChannel in, int position, int length)
- throws IOException
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- index += offset;
- ByteBuffer buf = ByteBuffer.wrap(data, index, length);
- int readBytes = 0;
-
- do {
- int localReadBytes;
- try {
- localReadBytes = in.read(buf, position + readBytes);
- }
- catch (ClosedChannelException e) {
- localReadBytes = -1;
- }
- if (localReadBytes < 0) {
- if (readBytes == 0) {
- return -1;
- }
- else {
- break;
- }
- }
- else if (localReadBytes == 0) {
- break;
- }
- readBytes += localReadBytes;
- } while (readBytes < length);
-
- return readBytes;
- }
-
- public Slice copySlice()
- {
- return copySlice(0, length);
- }
-
- /**
- * Returns a copy of this buffer's sub-region. Modifying the content of
- * the returned buffer or this buffer does not affect each other at all.
- */
- public Slice copySlice(int index, int length)
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
-
- index += offset;
- byte[] copiedArray = new byte[length];
- System.arraycopy(data, index, copiedArray, 0, length);
- return new Slice(copiedArray);
- }
-
- public byte[] copyBytes()
- {
- return copyBytes(0, length);
- }
-
- public byte[] copyBytes(int index, int length)
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- index += offset;
- if (index == 0) {
- return Arrays.copyOf(data, length);
- } else {
- byte[] value = new byte[length];
- System.arraycopy(data, index, value, 0, length);
- return value;
- }
- }
-
- /**
- * Returns a slice of this buffer's readable bytes. Modifying the content
- * of the returned buffer or this buffer affects each other's content
- * while they maintain separate indexes and marks.
- */
- public Slice slice()
- {
- return slice(0, length);
- }
-
- /**
- * Returns a slice of this buffer's sub-region. Modifying the content of
- * the returned buffer or this buffer affects each other's content while
- * they maintain separate indexes and marks.
- */
- public Slice slice(int index, int length)
- {
- if (index == 0 && length == this.length) {
- return this;
- }
-
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- if (index >= 0 && length == 0) {
- return Slices.EMPTY_SLICE;
- }
- return new Slice(data, offset + index, length);
- }
-
- /**
- * Converts this buffer's readable bytes into a NIO buffer. The returned
- * buffer shares the content with this buffer.
- */
- public ByteBuffer toByteBuffer()
- {
- return toByteBuffer(0, length);
- }
-
- /**
- * Converts this buffer's sub-region into a NIO buffer. The returned
- * buffer shares the content with this buffer.
- */
- public ByteBuffer toByteBuffer(int index, int length)
- {
- Preconditions.checkPositionIndexes(index, index + length, this.length);
- index += offset;
- return ByteBuffer.wrap(data, index, length).order(LITTLE_ENDIAN);
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- Slice slice = (Slice) o;
-
- // do lengths match
- if (length != slice.length) {
- return false;
- }
-
- // if arrays have same base offset, some optimizations can be taken...
- if (offset == slice.offset && data == slice.data) {
- return true;
- }
- for (int i = 0; i < length; i++) {
- if (data[offset + i] != slice.data[slice.offset + i]) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public int hashCode()
- {
- if (hash != 0) {
- return hash;
- }
-
- int result = length;
- for (int i = offset; i < offset + length; i++) {
- result = 31 * result + data[i];
- }
- if (result == 0) {
- result = 1;
- }
- hash = result;
- return hash;
- }
-
- /**
- * Compares the content of the specified buffer to the content of this
- * buffer. This comparison is performed byte by byte using an unsigned
- * comparison.
- */
- public int compareTo(Slice that)
- {
- if (this == that) {
- return 0;
- }
- if (this.data == that.data && length == that.length && offset == that.offset) {
- return 0;
- }
-
- int minLength = Math.min(this.length, that.length);
- for (int i = 0; i < minLength; i++) {
- int thisByte = 0xFF & this.data[this.offset + i];
- int thatByte = 0xFF & that.data[that.offset + i];
- if (thisByte != thatByte) {
- return (thisByte) - (thatByte);
- }
- }
- return this.length - that.length;
- }
-
- /**
- * Decodes this buffer's readable bytes into a string with the specified
- * character set name.
- */
- public String toString(Charset charset)
- {
- return toString(0, length, charset);
- }
-
- /**
- * Decodes this buffer's sub-region into a string with the specified
- * character set.
- */
- public String toString(int index, int length, Charset charset)
- {
- if (length == 0) {
- return "";
- }
-
- return Slices.decodeString(toByteBuffer(index, length), charset);
- }
-
- public String toString()
- {
- return getClass().getSimpleName() + '(' +
- "length=" + length() +
- ')';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/util/Slices.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/util/Slices.java b/java/kudu-client/src/main/java/org/kududb/util/Slices.java
deleted file mode 100644
index c2cdbde..0000000
--- a/java/kudu-client/src/main/java/org/kududb/util/Slices.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * Copyright 2011 Dain Sundstrom <da...@iq80.com>
- * Copyright 2011 FuseSource Corp. http://fusesource.com
- */
-package org.kududb.util;
-
-import com.google.common.base.Preconditions;
-import org.kududb.annotations.InterfaceAudience;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.CharBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CoderResult;
-import java.nio.charset.CodingErrorAction;
-import java.util.IdentityHashMap;
-import java.util.Map;
-
-@InterfaceAudience.Private
-public final class Slices
-{
- /**
- * A buffer whose capacity is {@code 0}.
- */
- public static final Slice EMPTY_SLICE = new Slice(0);
-
- private Slices()
- {
- }
-
- public static Slice ensureSize(Slice existingSlice, int minWritableBytes)
- {
- if (existingSlice == null) {
- existingSlice = EMPTY_SLICE;
- }
-
- if (minWritableBytes <= existingSlice.length()) {
- return existingSlice;
- }
-
- int newCapacity;
- if (existingSlice.length() == 0) {
- newCapacity = 1;
- }
- else {
- newCapacity = existingSlice.length();
- }
- int minNewCapacity = existingSlice.length() + minWritableBytes;
- while (newCapacity < minNewCapacity) {
- newCapacity <<= 1;
- }
-
- Slice newSlice = Slices.allocate(newCapacity);
- newSlice.setBytes(0, existingSlice, 0, existingSlice.length());
- return newSlice;
- }
-
- public static Slice allocate(int capacity)
- {
- if (capacity == 0) {
- return EMPTY_SLICE;
- }
- return new Slice(capacity);
- }
-
- public static Slice wrappedBuffer(byte[] array)
- {
- if (array.length == 0) {
- return EMPTY_SLICE;
- }
- return new Slice(array);
- }
-
- public static Slice copiedBuffer(ByteBuffer source, int sourceOffset, int length)
- {
- Preconditions.checkNotNull(source, "source is null");
- int newPosition = source.position() + sourceOffset;
- return copiedBuffer((ByteBuffer) source.duplicate().order(ByteOrder.LITTLE_ENDIAN).clear().limit(newPosition + length).position(newPosition));
- }
-
- public static Slice copiedBuffer(ByteBuffer source)
- {
- Preconditions.checkNotNull(source, "source is null");
- Slice copy = allocate(source.limit() - source.position());
- copy.setBytes(0, source.duplicate().order(ByteOrder.LITTLE_ENDIAN));
- return copy;
- }
-
- public static Slice copiedBuffer(String string, Charset charset)
- {
- Preconditions.checkNotNull(string, "string is null");
- Preconditions.checkNotNull(charset, "charset is null");
-
- return wrappedBuffer(string.getBytes(charset));
- }
-
- public static ByteBuffer encodeString(CharBuffer src, Charset charset)
- {
- final CharsetEncoder encoder = getEncoder(charset);
- final ByteBuffer dst = ByteBuffer.allocate(
- (int) ((double) src.remaining() * encoder.maxBytesPerChar()));
- try {
- CoderResult cr = encoder.encode(src, dst, true);
- if (!cr.isUnderflow()) {
- cr.throwException();
- }
- cr = encoder.flush(dst);
- if (!cr.isUnderflow()) {
- cr.throwException();
- }
- }
- catch (CharacterCodingException x) {
- throw new IllegalStateException(x);
- }
- dst.flip();
- return dst;
- }
-
- public static String decodeString(ByteBuffer src, Charset charset)
- {
- final CharsetDecoder decoder = getDecoder(charset);
- final CharBuffer dst = CharBuffer.allocate(
- (int) ((double) src.remaining() * decoder.maxCharsPerByte()));
- try {
- CoderResult cr = decoder.decode(src, dst, true);
- if (!cr.isUnderflow()) {
- cr.throwException();
- }
- cr = decoder.flush(dst);
- if (!cr.isUnderflow()) {
- cr.throwException();
- }
- }
- catch (CharacterCodingException x) {
- throw new IllegalStateException(x);
- }
- return dst.flip().toString();
- }
-
- /**
- * Toggles the endianness of the specified 16-bit short integer.
- */
- public static short swapShort(short value)
- {
- return (short) (value << 8 | value >>> 8 & 0xff);
- }
-
- /**
- * Toggles the endianness of the specified 32-bit integer.
- */
- public static int swapInt(int value)
- {
- return swapShort((short) value) << 16 |
- swapShort((short) (value >>> 16)) & 0xffff;
- }
-
- /**
- * Toggles the endianness of the specified 64-bit long integer.
- */
- public static long swapLong(long value)
- {
- return (long) swapInt((int) value) << 32 |
- swapInt((int) (value >>> 32)) & 0xffffffffL;
- }
-
- private static final ThreadLocal<Map<Charset, CharsetEncoder>> encoders =
- new ThreadLocal<Map<Charset, CharsetEncoder>>()
- {
- @Override
- protected Map<Charset, CharsetEncoder> initialValue()
- {
- return new IdentityHashMap<Charset, CharsetEncoder>();
- }
- };
-
- private static final ThreadLocal<Map<Charset, CharsetDecoder>> decoders =
- new ThreadLocal<Map<Charset, CharsetDecoder>>()
- {
- @Override
- protected Map<Charset, CharsetDecoder> initialValue()
- {
- return new IdentityHashMap<Charset, CharsetDecoder>();
- }
- };
-
- /**
- * Returns a cached thread-local {@link CharsetEncoder} for the specified
- * <tt>charset</tt>.
- */
- private static CharsetEncoder getEncoder(Charset charset)
- {
- if (charset == null) {
- throw new NullPointerException("charset");
- }
-
- Map<Charset, CharsetEncoder> map = encoders.get();
- CharsetEncoder e = map.get(charset);
- if (e != null) {
- e.reset();
- e.onMalformedInput(CodingErrorAction.REPLACE);
- e.onUnmappableCharacter(CodingErrorAction.REPLACE);
- return e;
- }
-
- e = charset.newEncoder();
- e.onMalformedInput(CodingErrorAction.REPLACE);
- e.onUnmappableCharacter(CodingErrorAction.REPLACE);
- map.put(charset, e);
- return e;
- }
-
-
- /**
- * Returns a cached thread-local {@link CharsetDecoder} for the specified
- * <tt>charset</tt>.
- */
- private static CharsetDecoder getDecoder(Charset charset)
- {
- if (charset == null) {
- throw new NullPointerException("charset");
- }
-
- Map<Charset, CharsetDecoder> map = decoders.get();
- CharsetDecoder d = map.get(charset);
- if (d != null) {
- d.reset();
- d.onMalformedInput(CodingErrorAction.REPLACE);
- d.onUnmappableCharacter(CodingErrorAction.REPLACE);
- return d;
- }
-
- d = charset.newDecoder();
- d.onMalformedInput(CodingErrorAction.REPLACE);
- d.onUnmappableCharacter(CodingErrorAction.REPLACE);
- map.put(charset, d);
- return d;
- }
-
-}
[10/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java
new file mode 100644
index 0000000..4915a18
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduPredicate.java
@@ -0,0 +1,628 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.kududb.client;
+
+import com.google.common.base.Preconditions;
+import org.junit.Assert;
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.Type;
+
+import static org.kududb.client.KuduPredicate.ComparisonOp.EQUAL;
+import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER;
+import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
+import static org.kududb.client.KuduPredicate.ComparisonOp.LESS;
+import static org.kududb.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
+import static org.kududb.client.KuduPredicate.PredicateType.RANGE;
+
+public class TestKuduPredicate {
+
+ private static final ColumnSchema boolCol =
+ new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build();
+
+ private static final ColumnSchema byteCol =
+ new ColumnSchema.ColumnSchemaBuilder("byte", Type.INT8).build();
+
+ private static final ColumnSchema shortCol =
+ new ColumnSchema.ColumnSchemaBuilder("short", Type.INT16).build();
+
+ private static final ColumnSchema intCol =
+ new ColumnSchema.ColumnSchemaBuilder("int", Type.INT32).build();
+
+ private static final ColumnSchema longCol =
+ new ColumnSchema.ColumnSchemaBuilder("long", Type.INT64).build();
+
+ private static final ColumnSchema floatCol =
+ new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build();
+
+ private static final ColumnSchema doubleCol =
+ new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build();
+
+ private static final ColumnSchema stringCol =
+ new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).build();
+
+ private static final ColumnSchema binaryCol =
+ new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).build();
+
+ private static KuduPredicate intRange(int lower, int upper) {
+ Preconditions.checkArgument(lower < upper);
+ return new KuduPredicate(RANGE, intCol, Bytes.fromInt(lower), Bytes.fromInt(upper));
+ }
+
+ private void testMerge(KuduPredicate a,
+ KuduPredicate b,
+ KuduPredicate expected) {
+
+ Assert.assertEquals(expected, a.merge(b));
+ Assert.assertEquals(expected, b.merge(a));
+ }
+
+ /**
+ * Tests merges on all types of integer predicates.
+ */
+ @Test
+ public void testMergeInt() {
+
+ // Equality + Equality
+
+ // |
+ // |
+ // =
+ // |
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0));
+ // |
+ // |
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 1),
+ KuduPredicate.none(intCol));
+
+ // Range + Equality
+
+ // [-------->
+ // |
+ // =
+ // |
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 10),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 10));
+
+ // [-------->
+ // |
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 10),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
+ KuduPredicate.none(intCol));
+
+ // <--------)
+ // |
+ // =
+ // |
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5));
+
+ // <--------)
+ // |
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 10),
+ KuduPredicate.none(intCol));
+
+ // Unbounded Range + Unbounded Range
+
+ // [--------> AND
+ // [-------->
+ // =
+ // [-------->
+
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0));
+
+ // [--------> AND
+ // [----->
+ // =
+ // [----->
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
+ KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5));
+
+ // <--------) AND
+ // <--------)
+ // =
+ // <--------)
+
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 0));
+
+ // <--------) AND
+ // <----)
+ // =
+ // <----)
+
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, -10),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, -10));
+
+ // [--------> AND
+ // <-------)
+ // =
+ // [----)
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
+ intRange(0, 10));
+
+ // [-----> AND
+ // <----)
+ // =
+ // |
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 6),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5));
+
+ // [-----> AND
+ // <---)
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 5),
+ KuduPredicate.none(intCol));
+
+ // [-----> AND
+ // <---)
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 3),
+ KuduPredicate.none(intCol));
+
+ // Range + Range
+
+ // [--------) AND
+ // [--------)
+ // =
+ // [--------)
+
+ testMerge(intRange(0, 10),
+ intRange(0, 10),
+ intRange(0, 10));
+
+ // [--------) AND
+ // [----)
+ // =
+ // [----)
+ testMerge(intRange(0, 10),
+ intRange(0, 5),
+ intRange(0, 5));
+
+ // [--------) AND
+ // [----)
+ // =
+ // [----)
+ testMerge(intRange(0, 10),
+ intRange(3, 8),
+ intRange(3, 8));
+
+ // [-----) AND
+ // [------)
+ // =
+ // [---)
+ testMerge(intRange(0, 8),
+ intRange(3, 10),
+ intRange(3, 8));
+ // [--) AND
+ // [---)
+ // =
+ // None
+ testMerge(intRange(0, 5),
+ intRange(5, 10),
+ KuduPredicate.none(intCol));
+
+ // [--) AND
+ // [---)
+ // =
+ // None
+ testMerge(intRange(0, 3),
+ intRange(5, 10),
+ KuduPredicate.none(intCol));
+
+ // Lower Bound + Range
+
+ // [------------>
+ // [---)
+ // =
+ // [---)
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
+ intRange(5, 10),
+ intRange(5, 10));
+
+ // [------------>
+ // [--------)
+ // =
+ // [--------)
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
+ intRange(5, 10),
+ intRange(5, 10));
+
+ // [------------>
+ // [--------)
+ // =
+ // [---)
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
+ intRange(0, 10),
+ intRange(5, 10));
+
+ // [------->
+ // [-----)
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 10),
+ intRange(0, 5),
+ KuduPredicate.none(intCol));
+
+ // Upper Bound + Range
+
+ // <------------)
+ // [---)
+ // =
+ // [---)
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
+ intRange(3, 8),
+ intRange(3, 8));
+
+ // <------------)
+ // [--------)
+ // =
+ // [--------)
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
+ intRange(5, 10),
+ intRange(5, 10));
+
+
+ // <------------)
+ // [--------)
+ // =
+ // [----)
+ testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 5),
+ intRange(0, 10),
+ intRange(0, 5));
+
+ // Range + Equality
+
+ // [---) AND
+ // |
+ // =
+ // None
+ testMerge(intRange(3, 5),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 1),
+ KuduPredicate.none(intCol));
+
+ // [---) AND
+ // |
+ // =
+ // |
+ testMerge(intRange(0, 5),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0));
+
+ // [---) AND
+ // |
+ // =
+ // |
+ testMerge(intRange(0, 5),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 3),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 3));
+
+ // [---) AND
+ // |
+ // =
+ // None
+ testMerge(intRange(0, 5),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5),
+ KuduPredicate.none(intCol));
+
+ // [---) AND
+ // |
+ // =
+ // None
+ testMerge(intRange(0, 5),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 7),
+ KuduPredicate.none(intCol));
+
+ // None
+
+ // None AND
+ // [---->
+ // =
+ // None
+ testMerge(KuduPredicate.none(intCol),
+ KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
+ KuduPredicate.none(intCol));
+ // None AND
+ // <----)
+ // =
+ // None
+ testMerge(KuduPredicate.none(intCol),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
+ KuduPredicate.none(intCol));
+
+ // None AND
+ // [----)
+ // =
+ // None
+ testMerge(KuduPredicate.none(intCol),
+ intRange(3, 7),
+ KuduPredicate.none(intCol));
+
+ // None AND
+ // |
+ // =
+ // None
+ testMerge(KuduPredicate.none(intCol),
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5),
+ KuduPredicate.none(intCol));
+
+ // None AND
+ // None
+ // =
+ // None
+ testMerge(KuduPredicate.none(intCol),
+ KuduPredicate.none(intCol),
+ KuduPredicate.none(intCol));
+ }
+
+ /**
+ * Tests tricky merges on a var length type.
+ */
+ @Test
+ public void testMergeString() {
+
+ // [----->
+ // <-----)
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "b\0"),
+ KuduPredicate.newComparisonPredicate(stringCol, LESS, "b"),
+ KuduPredicate.none(stringCol));
+
+ // [----->
+ // <-----)
+ // =
+ // None
+ testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "b"),
+ KuduPredicate.newComparisonPredicate(stringCol, LESS, "b"),
+ KuduPredicate.none(stringCol));
+
+ // [----->
+ // <----)
+ // =
+ // |
+ testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "b"),
+ KuduPredicate.newComparisonPredicate(stringCol, LESS, "b\0"),
+ KuduPredicate.newComparisonPredicate(stringCol, EQUAL, "b"));
+
+ // [----->
+ // <-----)
+ // =
+ // [--)
+ testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "a"),
+ KuduPredicate.newComparisonPredicate(stringCol, LESS, "a\0\0"),
+ new KuduPredicate(RANGE, stringCol,
+ Bytes.fromString("a"), Bytes.fromString("a\0\0")));
+ }
+
+ @Test
+ public void testBoolean() {
+
+ // b >= false
+ Assert.assertEquals(KuduPredicate.newIsNotNullPredicate(boolCol),
+ KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, false));
+ // b > false
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true),
+ KuduPredicate.newComparisonPredicate(boolCol, GREATER, false));
+ // b = false
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false),
+ KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false));
+ // b < false
+ Assert.assertEquals(KuduPredicate.none(boolCol),
+ KuduPredicate.newComparisonPredicate(boolCol, LESS, false));
+ // b <= false
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false),
+ KuduPredicate.newComparisonPredicate(boolCol, LESS_EQUAL, false));
+
+ // b >= true
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true),
+ KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, true));
+ // b > true
+ Assert.assertEquals(KuduPredicate.none(boolCol),
+ KuduPredicate.newComparisonPredicate(boolCol, GREATER, true));
+ // b = true
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true),
+ KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true));
+ // b < true
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false),
+ KuduPredicate.newComparisonPredicate(boolCol, LESS, true));
+ // b <= true
+ Assert.assertEquals(KuduPredicate.newIsNotNullPredicate(boolCol),
+ KuduPredicate.newComparisonPredicate(boolCol, LESS_EQUAL, true));
+ }
+
+ /**
+ * Tests basic predicate merges across all types.
+ */
+ @Test
+ public void testAllTypesMerge() {
+
+ testMerge(KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, false),
+ KuduPredicate.newComparisonPredicate(boolCol, LESS, true),
+ new KuduPredicate(KuduPredicate.PredicateType.EQUALITY,
+ boolCol,
+ Bytes.fromBoolean(false),
+ null));
+
+ testMerge(KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, false),
+ KuduPredicate.newComparisonPredicate(boolCol, LESS_EQUAL, true),
+ KuduPredicate.newIsNotNullPredicate(boolCol));
+
+ testMerge(KuduPredicate.newComparisonPredicate(byteCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(byteCol, LESS, 10),
+ new KuduPredicate(RANGE,
+ byteCol,
+ new byte[] { (byte) 0 },
+ new byte[] { (byte) 10 }));
+
+ testMerge(KuduPredicate.newComparisonPredicate(shortCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(shortCol, LESS, 10),
+ new KuduPredicate(RANGE,
+ shortCol,
+ Bytes.fromShort((short) 0),
+ Bytes.fromShort((short) 10)));
+
+ testMerge(KuduPredicate.newComparisonPredicate(longCol, GREATER_EQUAL, 0),
+ KuduPredicate.newComparisonPredicate(longCol, LESS, 10),
+ new KuduPredicate(RANGE,
+ longCol,
+ Bytes.fromLong(0),
+ Bytes.fromLong(10)));
+
+ testMerge(KuduPredicate.newComparisonPredicate(floatCol, GREATER_EQUAL, 123.45f),
+ KuduPredicate.newComparisonPredicate(floatCol, LESS, 678.90f),
+ new KuduPredicate(RANGE,
+ floatCol,
+ Bytes.fromFloat(123.45f),
+ Bytes.fromFloat(678.90f)));
+
+ testMerge(KuduPredicate.newComparisonPredicate(doubleCol, GREATER_EQUAL, 123.45),
+ KuduPredicate.newComparisonPredicate(doubleCol, LESS, 678.90),
+ new KuduPredicate(RANGE,
+ doubleCol,
+ Bytes.fromDouble(123.45),
+ Bytes.fromDouble(678.90)));
+
+ testMerge(KuduPredicate.newComparisonPredicate(binaryCol, GREATER_EQUAL,
+ new byte[] { 0, 1, 2, 3, 4, 5, 6 }),
+ KuduPredicate.newComparisonPredicate(binaryCol, LESS, new byte[] { 10 }),
+ new KuduPredicate(RANGE,
+ binaryCol,
+ new byte[] { 0, 1, 2, 3, 4, 5, 6 },
+ new byte[] { 10 }));
+ }
+
+ @Test
+ public void testLessEqual() {
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(byteCol, LESS_EQUAL, 10),
+ KuduPredicate.newComparisonPredicate(byteCol, LESS, 11));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(shortCol, LESS_EQUAL, 10),
+ KuduPredicate.newComparisonPredicate(shortCol, LESS, 11));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(intCol, LESS_EQUAL, 10),
+ KuduPredicate.newComparisonPredicate(intCol, LESS, 11));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(longCol, LESS_EQUAL, 10),
+ KuduPredicate.newComparisonPredicate(longCol, LESS, 11));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, LESS_EQUAL, 12.345f),
+ KuduPredicate.newComparisonPredicate(floatCol, LESS, Math.nextAfter(12.345f, Float.POSITIVE_INFINITY)));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, LESS_EQUAL, 12.345),
+ KuduPredicate.newComparisonPredicate(doubleCol, LESS, Math.nextAfter(12.345, Float.POSITIVE_INFINITY)));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(stringCol, LESS_EQUAL, "a"),
+ KuduPredicate.newComparisonPredicate(stringCol, LESS, "a\0"));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(binaryCol, LESS_EQUAL, new byte[] { (byte) 10 }),
+ KuduPredicate.newComparisonPredicate(binaryCol, LESS, new byte[] { (byte) 10, (byte) 0 }));
+
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(byteCol, LESS_EQUAL, Byte.MAX_VALUE),
+ KuduPredicate.newIsNotNullPredicate(byteCol));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(shortCol, LESS_EQUAL, Short.MAX_VALUE),
+ KuduPredicate.newIsNotNullPredicate(shortCol));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(intCol, LESS_EQUAL, Integer.MAX_VALUE),
+ KuduPredicate.newIsNotNullPredicate(intCol));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(longCol, LESS_EQUAL, Long.MAX_VALUE),
+ KuduPredicate.newIsNotNullPredicate(longCol));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, LESS_EQUAL, Float.MAX_VALUE),
+ KuduPredicate.newComparisonPredicate(floatCol, LESS, Float.POSITIVE_INFINITY));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, LESS_EQUAL, Float.POSITIVE_INFINITY),
+ KuduPredicate.newIsNotNullPredicate(floatCol));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, LESS_EQUAL, Double.MAX_VALUE),
+ KuduPredicate.newComparisonPredicate(doubleCol, LESS, Double.POSITIVE_INFINITY));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, LESS_EQUAL, Double.POSITIVE_INFINITY),
+ KuduPredicate.newIsNotNullPredicate(doubleCol));
+ }
+
+ @Test
+ public void testGreater() {
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(byteCol, GREATER_EQUAL, 11),
+ KuduPredicate.newComparisonPredicate(byteCol, GREATER, 10));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(shortCol, GREATER_EQUAL, 11),
+ KuduPredicate.newComparisonPredicate(shortCol, GREATER, 10));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 11),
+ KuduPredicate.newComparisonPredicate(intCol, GREATER, 10));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(longCol, GREATER_EQUAL, 11),
+ KuduPredicate.newComparisonPredicate(longCol, GREATER, 10));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, GREATER_EQUAL, Math.nextAfter(12.345f, Float.MAX_VALUE)),
+ KuduPredicate.newComparisonPredicate(floatCol, GREATER, 12.345f));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, GREATER_EQUAL, Math.nextAfter(12.345, Float.MAX_VALUE)),
+ KuduPredicate.newComparisonPredicate(doubleCol, GREATER, 12.345));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "a\0"),
+ KuduPredicate.newComparisonPredicate(stringCol, GREATER, "a"));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(binaryCol, GREATER_EQUAL, new byte[] { (byte) 10, (byte) 0 }),
+ KuduPredicate.newComparisonPredicate(binaryCol, GREATER, new byte[] { (byte) 10 }));
+
+ Assert.assertEquals(KuduPredicate.none(byteCol),
+ KuduPredicate.newComparisonPredicate(byteCol, GREATER, Byte.MAX_VALUE));
+ Assert.assertEquals(KuduPredicate.none(shortCol),
+ KuduPredicate.newComparisonPredicate(shortCol, GREATER, Short.MAX_VALUE));
+ Assert.assertEquals(KuduPredicate.none(intCol),
+ KuduPredicate.newComparisonPredicate(intCol, GREATER, Integer.MAX_VALUE));
+ Assert.assertEquals(KuduPredicate.none(longCol),
+ KuduPredicate.newComparisonPredicate(longCol, GREATER, Long.MAX_VALUE));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, GREATER_EQUAL, Float.POSITIVE_INFINITY),
+ KuduPredicate.newComparisonPredicate(floatCol, GREATER, Float.MAX_VALUE));
+ Assert.assertEquals(KuduPredicate.none(floatCol),
+ KuduPredicate.newComparisonPredicate(floatCol, GREATER, Float.POSITIVE_INFINITY));
+ Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, GREATER_EQUAL, Double.POSITIVE_INFINITY),
+ KuduPredicate.newComparisonPredicate(doubleCol, GREATER, Double.MAX_VALUE));
+ Assert.assertEquals(KuduPredicate.none(doubleCol),
+ KuduPredicate.newComparisonPredicate(doubleCol, GREATER, Double.POSITIVE_INFINITY));
+ }
+
+ @Test
+ public void testToString() {
+ Assert.assertEquals("`bool` = true",
+ KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true).toString());
+ Assert.assertEquals("`byte` = 11",
+ KuduPredicate.newComparisonPredicate(byteCol, EQUAL, 11).toString());
+ Assert.assertEquals("`short` = 11",
+ KuduPredicate.newComparisonPredicate(shortCol, EQUAL, 11).toString());
+ Assert.assertEquals("`int` = -123",
+ KuduPredicate.newComparisonPredicate(intCol, EQUAL, -123).toString());
+ Assert.assertEquals("`long` = 5454",
+ KuduPredicate.newComparisonPredicate(longCol, EQUAL, 5454).toString());
+ Assert.assertEquals("`float` = 123.456",
+ KuduPredicate.newComparisonPredicate(floatCol, EQUAL, 123.456f).toString());
+ Assert.assertEquals("`double` = 123.456",
+ KuduPredicate.newComparisonPredicate(doubleCol, EQUAL, 123.456).toString());
+ Assert.assertEquals("`string` = \"my string\"",
+ KuduPredicate.newComparisonPredicate(stringCol, EQUAL, "my string").toString());
+ Assert.assertEquals("`binary` = 0xAB01CD", KuduPredicate.newComparisonPredicate(
+ binaryCol, EQUAL, new byte[] { (byte) 0xAB, (byte) 0x01, (byte) 0xCD }).toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
new file mode 100644
index 0000000..df2367f
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduSession.java
@@ -0,0 +1,337 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import static org.junit.Assert.*;
+
+import com.google.common.collect.ImmutableList;
+
+public class TestKuduSession extends BaseKuduTest {
+ @Rule
+ public TestName name = new TestName();
+
+ private KuduTable table;
+
+ @Test(timeout = 100000)
+ public void testBasicOps() throws Exception {
+ String tableName = name.getMethodName();
+ table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
+
+ KuduSession session = syncClient.newSession();
+ for (int i = 0; i < 10; i++) {
+ session.apply(createInsert(i));
+ }
+ assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build()));
+
+ OperationResponse resp = session.apply(createInsert(0));
+ assertTrue(resp.hasRowError());
+
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+
+ for (int i = 10; i < 20; i++) {
+ session.apply(createInsert(i));
+ }
+ session.flush();
+ assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build()));
+ }
+
+ @Test(timeout = 100000)
+ public void testIgnoreAllDuplicateRows() throws Exception {
+ String tableName = name.getMethodName();
+ table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
+
+ KuduSession session = syncClient.newSession();
+ session.setIgnoreAllDuplicateRows(true);
+ for (int i = 0; i < 10; i++) {
+ session.apply(createInsert(i));
+ }
+ for (SessionConfiguration.FlushMode mode : SessionConfiguration.FlushMode.values()) {
+ session.setFlushMode(mode);
+ for (int i = 0; i < 10; i++) {
+ OperationResponse resp = session.apply(createInsert(i));
+ if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC) {
+ assertFalse(resp.hasRowError());
+ }
+ }
+ if (mode == SessionConfiguration.FlushMode.MANUAL_FLUSH) {
+ List<OperationResponse> responses = session.flush();
+ for (OperationResponse resp : responses) {
+ assertFalse(resp.hasRowError());
+ }
+ } else if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) {
+ while (session.hasPendingOperations()) {
+ Thread.sleep(100);
+ }
+ assertEquals(0, session.countPendingErrors());
+ }
+ }
+ }
+
+ @Test(timeout = 100000)
+ public void testBatchWithSameRow() throws Exception {
+ String tableName = name.getMethodName();
+ table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
+
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+
+ // Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end,
+ // while also clearing the cache between each batch half the time. The delete is added here
+ // so that a misplaced update would fail if it happens later than its delete.
+ for (int i = 0; i < 25; i++) {
+ session.apply(createInsert(i));
+ for (int j = 0; j < 50; j++) {
+ Update update = table.newUpdate();
+ PartialRow row = update.getRow();
+ row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
+ row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000);
+ session.apply(update);
+ }
+ Delete del = table.newDelete();
+ PartialRow row = del.getRow();
+ row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
+ session.apply(del);
+ session.flush();
+ if (i % 2 == 0) {
+ client.emptyTabletsCacheForTable(table.getTableId());
+ }
+ }
+ assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
+ }
+
+ /**
+ * Regression test for KUDU-1226. Calls to session.flush() concurrent with AUTO_FLUSH_BACKGROUND
+ * can end up giving ConvertBatchToListOfResponsesCB a list with nulls if a tablet was already
+ * flushed. Only happens with multiple tablets.
+ */
+ @Test(timeout = 10000)
+ public void testConcurrentFlushes() throws Exception {
+ String tableName = name.getMethodName();
+ CreateTableOptions builder = getBasicCreateTableOptions();
+ int numTablets = 4;
+ int numRowsPerTablet = 100;
+
+ // Create a 4 tablets table split on 1000, 2000, and 3000.
+ for (int i = 1; i < numTablets; i++) {
+ PartialRow split = basicSchema.newPartialRow();
+ split.addInt(0, i * numRowsPerTablet);
+ builder.addSplitRow(split);
+ }
+ table = createTable(tableName, basicSchema, builder);
+
+ // Configure the session to background flush as often as it can (every 1ms).
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+ session.setFlushInterval(1);
+
+ // Fill each tablet in parallel 1 by 1 then flush. Without the fix this would quickly get an
+ // NPE.
+ for (int i = 0; i < numRowsPerTablet; i++) {
+ for (int j = 0; j < numTablets; j++) {
+ session.apply(createInsert(i + (numRowsPerTablet * j)));
+ }
+ session.flush();
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testOverWritingValues() throws Exception {
+ String tableName = name.getMethodName();
+ table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
+ KuduSession session = syncClient.newSession();
+ Insert insert = createInsert(0);
+ PartialRow row = insert.getRow();
+
+ // Overwrite all the normal columns.
+ int magicNumber = 9999;
+ row.addInt(1, magicNumber);
+ row.addInt(2, magicNumber);
+ row.addBoolean(4, false);
+ // Spam the string column since it's backed by an array.
+ for (int i = 0; i <= magicNumber; i++) {
+ row.addString(3, i + "");
+ }
+ // We're supposed to keep a constant size.
+ assertEquals(5, row.getVarLengthData().size());
+ session.apply(insert);
+
+ KuduScanner scanner = syncClient.newScannerBuilder(table).build();
+ RowResult rr = scanner.nextRows().next();
+ assertEquals(magicNumber, rr.getInt(1));
+ assertEquals(magicNumber, rr.getInt(2));
+ assertEquals(magicNumber + "", rr.getString(3));
+ assertEquals(false, rr.getBoolean(4));
+
+ // Test setting a value post-apply.
+ try {
+ row.addInt(1, 0);
+ fail("Row should be frozen and throw");
+ } catch (IllegalStateException ex) {
+ // Ok.
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testUpsert() throws Exception {
+ String tableName = name.getMethodName();
+ table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
+ KuduSession session = syncClient.newSession();
+
+ // Test an Upsert that acts as an Insert.
+ assertFalse(session.apply(createUpsert(1, 1, false)).hasRowError());
+
+ List<String> rowStrings = scanTableToStrings(table);
+ assertEquals(1, rowStrings.size());
+ assertEquals(
+ "INT32 key=1, INT32 column1_i=1, INT32 column2_i=3, " +
+ "STRING column3_s=a string, BOOL column4_b=true",
+ rowStrings.get(0));
+
+ // Test an Upsert that acts as an Update.
+ assertFalse(session.apply(createUpsert(1, 2, false)).hasRowError());
+ rowStrings = scanTableToStrings(table);
+ assertEquals(
+ "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " +
+ "STRING column3_s=a string, BOOL column4_b=true",
+ rowStrings.get(0));
+ }
+
+ @Test(timeout = 10000)
+ public void testInsertManualFlushNonCoveredRange() throws Exception {
+ String tableName = name.getMethodName();
+ CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange();
+ createOptions.setNumReplicas(1);
+ syncClient.createTable(tableName, basicSchema, createOptions);
+ KuduTable table = syncClient.openTable(tableName);
+
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+
+ // Insert in reverse sorted order so that more table location lookups occur
+ // (the extra results in table location lookups always occur past the inserted key).
+ List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50);
+ for (int key : nonCoveredKeys) {
+ assertNull(session.apply(createBasicSchemaInsert(table, key)));
+ }
+ List<OperationResponse> results = session.flush();
+ assertEquals(nonCoveredKeys.size(), results.size());
+ for (OperationResponse result : results) {
+ assertTrue(result.hasRowError());
+ assertTrue(result.getRowError().getErrorStatus().isNotFound());
+ }
+
+ // Insert a batch of some valid and some invalid.
+ for (int key = 90; key < 110; key++) {
+ session.apply(createBasicSchemaInsert(table, key));
+ }
+ results = session.flush();
+
+ int failures = 0;
+ for (OperationResponse result : results) {
+ if (result.hasRowError()) {
+ failures++;
+ assertTrue(result.getRowError().getErrorStatus().isNotFound());
+ }
+ }
+ assertEquals(10, failures);
+ }
+
+ @Test(timeout = 10000)
+ public void testInsertAutoFlushSyncNonCoveredRange() throws Exception {
+ String tableName = name.getMethodName();
+ CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange();
+ createOptions.setNumReplicas(1);
+ syncClient.createTable(tableName, basicSchema, createOptions);
+ KuduTable table = syncClient.openTable(tableName);
+
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
+
+ List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50);
+ for (int key : nonCoveredKeys) {
+ try {
+ session.apply(createBasicSchemaInsert(table, key));
+ fail("apply should have thrown");
+ } catch (KuduException e) {
+ assertTrue(e.getStatus().isNotFound());
+ }
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testInsertAutoFlushBackgrounNonCoveredRange() throws Exception {
+ String tableName = name.getMethodName();
+ CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange();
+ createOptions.setNumReplicas(1);
+ syncClient.createTable(tableName, basicSchema, createOptions);
+ KuduTable table = syncClient.openTable(tableName);
+
+ AsyncKuduSession session = client.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+
+ List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50);
+ for (int key : nonCoveredKeys) {
+ OperationResponse result = session.apply(createBasicSchemaInsert(table, key)).join(5000);
+ assertTrue(result.hasRowError());
+ assertTrue(result.getRowError().getErrorStatus().isNotFound());
+ }
+
+ RowErrorsAndOverflowStatus errors = session.getPendingErrors();
+ assertEquals(nonCoveredKeys.size(), errors.getRowErrors().length);
+ for (RowError error : errors.getRowErrors()) {
+ assertTrue(error.getErrorStatus().isNotFound());
+ }
+
+ // Insert a batch of some valid and some invalid.
+ for (int key = 90; key < 110; key++) {
+ session.apply(createBasicSchemaInsert(table, key));
+ }
+ session.flush();
+
+ errors = session.getPendingErrors();
+ assertEquals(10, errors.getRowErrors().length);
+ for (RowError error : errors.getRowErrors()) {
+ assertTrue(error.getErrorStatus().isNotFound());
+ }
+ }
+
+ private Insert createInsert(int key) {
+ return createBasicSchemaInsert(table, key);
+ }
+
+ private Upsert createUpsert(int key, int secondVal, boolean hasNull) {
+ Upsert upsert = table.newUpsert();
+ PartialRow row = upsert.getRow();
+ row.addInt(0, key);
+ row.addInt(1, secondVal);
+ row.addInt(2, 3);
+ if (hasNull) {
+ row.setNull(3);
+ } else {
+ row.addString(3, "a string");
+ }
+ row.addBoolean(4, true);
+ return upsert;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
new file mode 100644
index 0000000..4e41a29
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
@@ -0,0 +1,301 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class TestKuduTable extends BaseKuduTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TestKuduTable.class);
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static Schema schema = getBasicSchema();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ }
+
+ @Test(timeout = 100000)
+ public void testAlterTable() throws Exception {
+ String tableName = name.getMethodName();
+ createTable(tableName, basicSchema, getBasicCreateTableOptions());
+ try {
+
+ // Add a col.
+ AlterTableOptions ato = new AlterTableOptions().addColumn("testaddint", Type.INT32, 4);
+ submitAlterAndCheck(ato, tableName);
+
+ // Rename that col.
+ ato = new AlterTableOptions().renameColumn("testaddint", "newtestaddint");
+ submitAlterAndCheck(ato, tableName);
+
+ // Delete it.
+ ato = new AlterTableOptions().dropColumn("newtestaddint");
+ submitAlterAndCheck(ato, tableName);
+
+ String newTableName = tableName +"new";
+
+ // Rename our table.
+ ato = new AlterTableOptions().renameTable(newTableName);
+ submitAlterAndCheck(ato, tableName, newTableName);
+
+ // Rename it back.
+ ato = new AlterTableOptions().renameTable(tableName);
+ submitAlterAndCheck(ato, newTableName, tableName);
+
+ // Try adding two columns, where one is nullable.
+ ato = new AlterTableOptions()
+ .addColumn("testaddmulticolnotnull", Type.INT32, 4)
+ .addNullableColumn("testaddmulticolnull", Type.STRING);
+ submitAlterAndCheck(ato, tableName);
+
+
+ // Try altering a table that doesn't exist.
+ String nonExistingTableName = "table_does_not_exist";
+ try {
+ syncClient.alterTable(nonExistingTableName, ato);
+ fail("Shouldn't be able to alter a table that doesn't exist");
+ } catch (KuduException ex) {
+ assertTrue(ex.getStatus().isNotFound());
+ }
+
+ try {
+ syncClient.isAlterTableDone(nonExistingTableName);
+ fail("Shouldn't be able to query if an alter table is done here");
+ } catch (KuduException ex) {
+ assertTrue(ex.getStatus().isNotFound());
+ }
+ } finally {
+ // Normally Java tests accumulate tables without issue, deleting them all
+ // when shutting down the mini cluster at the end of every test class.
+ // However, testGetLocations below expects a certain table count, so
+ // we'll delete our table to ensure there's no interaction between them.
+ syncClient.deleteTable(tableName);
+ }
+ }
+
+ /**
+ * Helper method to submit an Alter and wait for it to happen, using the default table name to
+ * check.
+ */
+ private void submitAlterAndCheck(AlterTableOptions ato, String tableToAlter)
+ throws Exception {
+ submitAlterAndCheck(ato, tableToAlter, tableToAlter);
+ }
+
+ private void submitAlterAndCheck(AlterTableOptions ato,
+ String tableToAlter, String tableToCheck) throws
+ Exception {
+ if (masterHostPorts.size() > 1) {
+ LOG.info("Alter table is not yet supported with multiple masters. Specify " +
+ "-DnumMasters=1 on the command line to start a single-master cluster to run this test.");
+ return;
+ }
+ AlterTableResponse alterResponse = syncClient.alterTable(tableToAlter, ato);
+ boolean done = syncClient.isAlterTableDone(tableToCheck);
+ assertTrue(done);
+ }
+
+ /**
+ * Test creating tables of different sizes and see that we get the correct number of tablets back
+ * @throws Exception
+ */
+ @Test
+ public void testGetLocations() throws Exception {
+ String table1 = name.getMethodName() + System.currentTimeMillis();
+
+ // Test a non-existing table
+ try {
+ openTable(table1);
+ fail("Should receive an exception since the table doesn't exist");
+ } catch (Exception ex) {
+ // expected
+ }
+ // Test with defaults
+ String tableWithDefault = name.getMethodName() + "WithDefault" + System.currentTimeMillis();
+ CreateTableOptions builder = getBasicCreateTableOptions();
+ List<ColumnSchema> columns = new ArrayList<ColumnSchema>(schema.getColumnCount());
+ int defaultInt = 30;
+ String defaultString = "data";
+ for (ColumnSchema columnSchema : schema.getColumns()) {
+
+ Object defaultValue;
+
+ if (columnSchema.getType() == Type.INT32) {
+ defaultValue = defaultInt;
+ } else if (columnSchema.getType() == Type.BOOL) {
+ defaultValue = true;
+ } else {
+ defaultValue = defaultString;
+ }
+ columns.add(
+ new ColumnSchema.ColumnSchemaBuilder(columnSchema.getName(), columnSchema.getType())
+ .key(columnSchema.isKey())
+ .nullable(columnSchema.isNullable())
+ .defaultValue(defaultValue).build());
+ }
+ Schema schemaWithDefault = new Schema(columns);
+ KuduTable kuduTable = createTable(tableWithDefault, schemaWithDefault, builder);
+ assertEquals(defaultInt, kuduTable.getSchema().getColumnByIndex(0).getDefaultValue());
+ assertEquals(defaultString,
+ kuduTable.getSchema().getColumnByIndex(columns.size() - 2).getDefaultValue());
+ assertEquals(true,
+ kuduTable.getSchema().getColumnByIndex(columns.size() - 1).getDefaultValue());
+
+ // Make sure the table's schema includes column IDs.
+ assertTrue(kuduTable.getSchema().hasColumnIds());
+
+ // Test we can open a table that was already created.
+ openTable(tableWithDefault);
+
+ // Test splitting and reading those splits
+ KuduTable kuduTableWithoutDefaults = createTableWithSplitsAndTest(0);
+ // finish testing read defaults
+ assertNull(kuduTableWithoutDefaults.getSchema().getColumnByIndex(0).getDefaultValue());
+ createTableWithSplitsAndTest(3);
+ createTableWithSplitsAndTest(10);
+
+ KuduTable table = createTableWithSplitsAndTest(30);
+
+ List<LocatedTablet>tablets = table.getTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP);
+ assertEquals(9, tablets.size());
+ assertEquals(9, table.asyncGetTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP).join().size());
+
+ tablets = table.getTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP);
+ assertEquals(9, tablets.size());
+ assertEquals(9, table.asyncGetTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP).join().size());
+
+ tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP);
+ assertEquals(4, tablets.size());
+ assertEquals(4, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP).join().size());
+
+ tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP);
+ assertEquals(9, tablets.size());
+ assertEquals(9, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP).join().size());
+
+ tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(31), DEFAULT_SLEEP);
+ assertEquals(26, tablets.size());
+ assertEquals(26, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(31), DEFAULT_SLEEP).join().size());
+
+ tablets = table.getTabletsLocations(getKeyInBytes(5), null, DEFAULT_SLEEP);
+ assertEquals(26, tablets.size());
+ assertEquals(26, table.asyncGetTabletsLocations(getKeyInBytes(5), null, DEFAULT_SLEEP).join().size());
+
+ tablets = table.getTabletsLocations(null, getKeyInBytes(10000), DEFAULT_SLEEP);
+ assertEquals(31, tablets.size());
+ assertEquals(31, table.asyncGetTabletsLocations(null, getKeyInBytes(10000), DEFAULT_SLEEP).join().size());
+
+ tablets = table.getTabletsLocations(getKeyInBytes(20), getKeyInBytes(10000), DEFAULT_SLEEP);
+ assertEquals(11, tablets.size());
+ assertEquals(11, table.asyncGetTabletsLocations(getKeyInBytes(20), getKeyInBytes(10000), DEFAULT_SLEEP).join().size());
+
+ // Test listing tables.
+ assertEquals(0, client.getTablesList(table1).join(DEFAULT_SLEEP).getTablesList().size());
+ assertEquals(1, client.getTablesList(tableWithDefault)
+ .join(DEFAULT_SLEEP).getTablesList().size());
+ assertEquals(5, client.getTablesList().join(DEFAULT_SLEEP).getTablesList().size());
+ assertFalse(client.getTablesList(tableWithDefault).
+ join(DEFAULT_SLEEP).getTablesList().isEmpty());
+
+ assertFalse(client.tableExists(table1).join(DEFAULT_SLEEP));
+ assertTrue(client.tableExists(tableWithDefault).join(DEFAULT_SLEEP));
+ }
+
+ @Test(timeout = 100000)
+ public void testLocateTableNonCoveringRange() throws Exception {
+ String tableName = name.getMethodName();
+ syncClient.createTable(tableName, basicSchema, getBasicTableOptionsWithNonCoveredRange());
+ KuduTable table = syncClient.openTable(tableName);
+
+ List<LocatedTablet> tablets;
+
+ // all tablets
+ tablets = table.getTabletsLocations(null, null, 100000);
+ assertEquals(3, tablets.size());
+ assertArrayEquals(getKeyInBytes(0), tablets.get(0).getPartition().getPartitionKeyStart());
+ assertArrayEquals(getKeyInBytes(50), tablets.get(0).getPartition().getPartitionKeyEnd());
+ assertArrayEquals(getKeyInBytes(50), tablets.get(1).getPartition().getPartitionKeyStart());
+ assertArrayEquals(getKeyInBytes(100), tablets.get(1).getPartition().getPartitionKeyEnd());
+ assertArrayEquals(getKeyInBytes(200), tablets.get(2).getPartition().getPartitionKeyStart());
+ assertArrayEquals(getKeyInBytes(300), tablets.get(2).getPartition().getPartitionKeyEnd());
+
+ // key < 50
+ tablets = table.getTabletsLocations(null, getKeyInBytes(50), 100000);
+ assertEquals(1, tablets.size());
+ assertArrayEquals(getKeyInBytes(0), tablets.get(0).getPartition().getPartitionKeyStart());
+ assertArrayEquals(getKeyInBytes(50), tablets.get(0).getPartition().getPartitionKeyEnd());
+
+ // key >= 300
+ tablets = table.getTabletsLocations(getKeyInBytes(300), null, 100000);
+ assertEquals(0, tablets.size());
+
+ // key >= 299
+ tablets = table.getTabletsLocations(getKeyInBytes(299), null, 100000);
+ assertEquals(1, tablets.size());
+ assertArrayEquals(getKeyInBytes(200), tablets.get(0).getPartition().getPartitionKeyStart());
+ assertArrayEquals(getKeyInBytes(300), tablets.get(0).getPartition().getPartitionKeyEnd());
+
+ // key >= 150 && key < 250
+ tablets = table.getTabletsLocations(getKeyInBytes(150), getKeyInBytes(250), 100000);
+ assertEquals(1, tablets.size());
+ assertArrayEquals(getKeyInBytes(200), tablets.get(0).getPartition().getPartitionKeyStart());
+ assertArrayEquals(getKeyInBytes(300), tablets.get(0).getPartition().getPartitionKeyEnd());
+ }
+
+ public byte[] getKeyInBytes(int i) {
+ PartialRow row = schema.newPartialRow();
+ row.addInt(0, i);
+ return row.encodePrimaryKey();
+ }
+
+ public KuduTable createTableWithSplitsAndTest(int splitsCount) throws Exception {
+ String tableName = name.getMethodName() + System.currentTimeMillis();
+ CreateTableOptions builder = getBasicCreateTableOptions();
+
+ if (splitsCount != 0) {
+ for (int i = 1; i <= splitsCount; i++) {
+ PartialRow row = schema.newPartialRow();
+ row.addInt(0, i);
+ builder.addSplitRow(row);
+ }
+ }
+ KuduTable table = createTable(tableName, schema, builder);
+
+ List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
+ assertEquals(splitsCount + 1, tablets.size());
+ assertEquals(splitsCount + 1, table.asyncGetTabletsLocations(DEFAULT_SLEEP).join().size());
+ for (LocatedTablet tablet : tablets) {
+ assertEquals(3, tablet.getReplicas().size());
+ }
+ return table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java
new file mode 100644
index 0000000..49ac502
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestLeaderFailover.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestLeaderFailover extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ TestLeaderFailover.class.getName() + "-" + System.currentTimeMillis();
+ private static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+
+ CreateTableOptions builder = getBasicCreateTableOptions();
+ createTable(TABLE_NAME, basicSchema, builder);
+
+ table = openTable(TABLE_NAME);
+ }
+
+ /**
+ * This test writes 3 rows, kills the leader, then tries to write another 3 rows. Finally it
+ * counts to make sure we have 6 of them.
+ *
+ * This test won't run if we didn't start the cluster.
+ */
+ @Test(timeout = 100000)
+ public void testFailover() throws Exception {
+ KuduSession session = syncClient.newSession();
+ for (int i = 0; i < 3; i++) {
+ session.apply(createBasicSchemaInsert(table, i));
+ }
+
+ // Make sure the rows are in there before messing things up.
+ AsyncKuduScanner scanner = client.newScannerBuilder(table).build();
+ assertEquals(3, countRowsInScan(scanner));
+
+ killTabletLeader(table);
+
+ for (int i = 3; i < 6; i++) {
+ OperationResponse resp = session.apply(createBasicSchemaInsert(table, i));
+ if (resp.hasRowError()) {
+ fail("Encountered a row error " + resp.getRowError());
+ }
+ }
+
+ scanner = client.newScannerBuilder(table).build();
+ assertEquals(6, countRowsInScan(scanner));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java
new file mode 100644
index 0000000..2f91a6e
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMasterFailover.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+
+
+/**
+ * Tests {@link AsyncKuduClient} with multiple masters.
+ */
+public class TestMasterFailover extends BaseKuduTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TestMasterFailover.class);
+ private static final String TABLE_NAME =
+ TestMasterFailover.class.getName() + "-" + System.currentTimeMillis();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions());
+ }
+
+ /**
+ * This test is disabled as we're not supporting multi-master just yet.
+ */
+ @Test(timeout = 30000)
+ @Ignore
+ public void testKillLeader() throws Exception {
+ int countMasters = masterHostPorts.size();
+ if (countMasters < 3) {
+ LOG.info("This test requires at least 3 master servers, but only " + countMasters +
+ " are specified.");
+ return;
+ }
+ killMasterLeader();
+
+ // Test that we can open a previously created table after killing the leader master.
+ KuduTable table = openTable(TABLE_NAME);
+ assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
+
+ // Test that we can create a new table when one of the masters is down.
+ String newTableName = TABLE_NAME + "-afterLeaderIsDead";
+ createTable(newTableName, basicSchema, new CreateTableOptions());
+ table = openTable(newTableName);
+ assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
+
+ // Test that we can initialize a client when one of the masters specified in the
+ // connection string is down.
+ AsyncKuduClient newClient = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses).build();
+ table = newClient.openTable(newTableName).join(DEFAULT_SLEEP);
+ assertEquals(0, countRowsInScan(newClient.newScannerBuilder(table).build()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java
new file mode 100644
index 0000000..82ffacb
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMiniKuduCluster.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.client;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.Socket;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestMiniKuduCluster {
+
+ private static final int NUM_TABLET_SERVERS = 3;
+ private static final int DEFAULT_NUM_MASTERS = 1;
+
+ private MiniKuduCluster cluster;
+
+ @Before
+ public void before() throws Exception {
+ cluster = new MiniKuduCluster.MiniKuduClusterBuilder()
+ .numMasters(DEFAULT_NUM_MASTERS)
+ .numTservers(NUM_TABLET_SERVERS)
+ .build();
+ assertTrue(cluster.waitForTabletServers(NUM_TABLET_SERVERS));
+ }
+
+ @After
+ public void after() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 50000)
+ public void test() throws Exception {
+
+ assertEquals(DEFAULT_NUM_MASTERS, cluster.getMasterProcesses().size());
+ assertEquals(NUM_TABLET_SERVERS, cluster.getTabletServerProcesses().size());
+
+ {
+ // Kill the master.
+ int masterPort = cluster.getMasterProcesses().keySet().iterator().next();
+ testPort(masterPort, true, 1000);
+ cluster.killMasterOnPort(masterPort);
+
+ testPort(masterPort, false, 2000);
+
+ // Restart the master.
+ cluster.restartDeadMasterOnPort(masterPort);
+
+ // Test we can reach it.
+ testPort(masterPort, true, 3000);
+ }
+
+
+ {
+ // Kill the first TS.
+ int tsPort = cluster.getTabletServerProcesses().keySet().iterator().next();
+ testPort(tsPort, true, 1000);
+ cluster.killTabletServerOnPort(tsPort);
+
+ testPort(tsPort, false, 2000);
+
+ // Restart it.
+ cluster.restartDeadTabletServerOnPort(tsPort);
+
+ testPort(tsPort, true, 3000);
+ }
+
+ assertEquals(DEFAULT_NUM_MASTERS, cluster.getMasterProcesses().size());
+ assertEquals(NUM_TABLET_SERVERS, cluster.getTabletServerProcesses().size());
+ }
+
+ /**
+ * Test without the specified is open or closed, waiting up to a certain time.
+ * The longer you expect it might for the socket to become open or closed.
+ * @param port the port to test
+ * @param testIsOpen true if we should want it to be open, false if we want it closed
+ * @param timeout how long we're willing to wait before it happens
+ * @throws InterruptedException
+ */
+ private void testPort(int port, boolean testIsOpen, long timeout) throws InterruptedException {
+ DeadlineTracker tracker = new DeadlineTracker();
+ while (tracker.getElapsedMillis() < timeout) {
+ try {
+ Socket socket = new Socket(TestUtils.getUniqueLocalhost(), port);
+ socket.close();
+ if (testIsOpen) {
+ return;
+ }
+ } catch (IOException e) {
+ if (!testIsOpen) {
+ return;
+ }
+ }
+ Thread.sleep(200);
+ }
+ fail("Port " + port + " is still " + (testIsOpen ? "closed " : "open"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
new file mode 100644
index 0000000..f305fbf
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestOperation.java
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.WireProtocol.RowOperationsPB;
+import org.kududb.client.Operation.ChangeType;
+import org.kududb.tserver.Tserver.WriteRequestPBOrBuilder;
+import org.mockito.Mockito;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.primitives.Longs;
+
+/**
+ * Unit tests for Operation
+ */
+public class TestOperation {
+
+ private Schema createManyStringsSchema() {
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.STRING).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.STRING).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.STRING).nullable(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.STRING).nullable(true).build());
+ return new Schema(columns);
+ }
+
+ @Test
+ public void testSetStrings() {
+ KuduTable table = Mockito.mock(KuduTable.class);
+ Mockito.doReturn(createManyStringsSchema()).when(table).getSchema();
+ Insert insert = new Insert(table);
+ PartialRow row = insert.getRow();
+ row.addString("c0", "c0_val");
+ row.addString("c2", "c2_val");
+ row.addString("c1", "c1_val");
+ row.addString("c3", "c3_val");
+ row.addString("c4", "c4_val");
+
+ {
+ WriteRequestPBOrBuilder pb =
+ Operation.createAndFillWriteRequestPB(ImmutableList.<Operation>of(insert));
+ RowOperationsPB rowOps = pb.getRowOperations();
+ assertEquals(6 * 5, rowOps.getIndirectData().size());
+ assertEquals("c0_valc1_valc2_valc3_valc4_val", rowOps.getIndirectData().toStringUtf8());
+ byte[] rows = rowOps.getRows().toByteArray();
+ assertEquals(ChangeType.INSERT.toEncodedByte(), rows[0]);
+ // The "isset" bitset should have 5 bits set
+ assertEquals(0x1f, rows[1]);
+ // The "null" bitset should have no bits set
+ assertEquals(0, rows[2]);
+
+ // Check the strings.
+ int offset = 3;
+ for (int i = 0; i <= 4; i++) {
+ // The offset into the indirect buffer
+ assertEquals(6 * i, Bytes.getLong(rows, offset));
+ offset += Longs.BYTES;
+ // The length of the pointed-to string.
+ assertEquals(6, Bytes.getLong(rows, offset));
+ offset += Longs.BYTES;
+ }
+
+ // Should have used up whole buffer.
+ assertEquals(rows.length, offset);
+ }
+
+ // Setting a field to NULL should add to the null bitmap and remove
+ // the old value from the indirect buffer.
+ row.setNull("c3");
+ {
+ WriteRequestPBOrBuilder pb =
+ Operation.createAndFillWriteRequestPB(ImmutableList.<Operation>of(insert));
+ RowOperationsPB rowOps = pb.getRowOperations();
+ assertEquals(6 * 4, rowOps.getIndirectData().size());
+ assertEquals("c0_valc1_valc2_valc4_val", rowOps.getIndirectData().toStringUtf8());
+ byte[] rows = rowOps.getRows().toByteArray();
+ assertEquals(ChangeType.INSERT.toEncodedByte(), rows[0]);
+ // The "isset" bitset should have 5 bits set
+ assertEquals(0x1f, rows[1]);
+ // The "null" bitset should have 1 bit set for the null column
+ assertEquals(1 << 3, rows[2]);
+
+ // Check the strings.
+ int offset = 3;
+ int indirOffset = 0;
+ for (int i = 0; i <= 4; i++) {
+ if (i == 3) continue;
+ // The offset into the indirect buffer
+ assertEquals(indirOffset, Bytes.getLong(rows, offset));
+ indirOffset += 6;
+ offset += Longs.BYTES;
+ // The length of the pointed-to string.
+ assertEquals(6, Bytes.getLong(rows, offset));
+ offset += Longs.BYTES;
+ }
+ // Should have used up whole buffer.
+ assertEquals(rows.length, offset);
+ }
+ }
+
+ private Schema createAllTypesKeySchema() {
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(7);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.INT8).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.INT16).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.INT32).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.INT64).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.TIMESTAMP).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c5", Type.STRING).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c6", Type.BINARY).key(true).build());
+ return new Schema(columns);
+ }
+
+ @Test
+ public void testRowKeyStringify() {
+ KuduTable table = Mockito.mock(KuduTable.class);
+ Mockito.doReturn(createAllTypesKeySchema()).when(table).getSchema();
+ Insert insert = new Insert(table);
+ PartialRow row = insert.getRow();
+ row.addByte("c0", (byte) 1);
+ row.addShort("c1", (short) 2);
+ row.addInt("c2", 3);
+ row.addLong("c3", 4);
+ row.addLong("c4", 5);
+ row.addString("c5", "c5_val");
+ row.addBinary("c6", Bytes.fromString("c6_val"));
+
+ assertEquals("(int8 c0=1, int16 c1=2, int32 c2=3, int64 c3=4, timestamp c4=5, string" +
+ " c5=c5_val, binary c6=\"c6_val\")",
+ insert.getRow().stringifyRowKey());
+
+ // Test an incomplete row key.
+ insert = new Insert(table);
+ row = insert.getRow();
+ row.addByte("c0", (byte) 1);
+ try {
+ row.stringifyRowKey();
+ fail("Should not be able to stringifyRowKey when not all keys are specified");
+ } catch (IllegalStateException ise) {
+ // Expected.
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java
new file mode 100644
index 0000000..7528de6
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRequestTracker.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class TestRequestTracker {
+
+ @Test(timeout = 10000)
+ public void test() {
+ RequestTracker tracker = new RequestTracker("test");
+
+ // A new tracker should have no incomplete RPCs.
+ assertEquals(RequestTracker.NO_SEQ_NO, tracker.firstIncomplete());
+
+ int max = 10;
+
+ for (int i = 0; i < max; i++) {
+ tracker.newSeqNo();
+ }
+
+ // The first RPC is the incomplete one.
+ assertEquals(1, tracker.firstIncomplete());
+
+ // Mark the first as complete, incomplete should advance by 1.
+ tracker.rpcCompleted(1);
+ assertEquals(2, tracker.firstIncomplete());
+
+ // Mark the RPC in the middle as complete, first incomplete doesn't change.
+ tracker.rpcCompleted(5);
+ assertEquals(2, tracker.firstIncomplete());
+
+ // Mark the first half as complete.
+ // Note that we're also testing that rpcCompleted is idempotent.
+ for (int i = 1; i < max / 2; i++) {
+ tracker.rpcCompleted(i);
+ }
+
+ assertEquals(6, tracker.firstIncomplete());
+
+ // Get a few more sequence numbers.
+ long lastSeqNo = 0;
+ for (int i = max / 2; i <= max; i++) {
+ lastSeqNo = tracker.newSeqNo();
+ }
+
+ // Mark them all as complete except the last one.
+ while (tracker.firstIncomplete() != lastSeqNo) {
+ tracker.rpcCompleted(tracker.firstIncomplete());
+ }
+
+ assertEquals(lastSeqNo, tracker.firstIncomplete());
+ tracker.rpcCompleted(lastSeqNo);
+
+ // Test that we get back to NO_SEQ_NO after marking them all.
+ assertEquals(RequestTracker.NO_SEQ_NO, tracker.firstIncomplete());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java
new file mode 100644
index 0000000..90d11aa
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowErrors.java
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+import com.google.common.collect.ImmutableList;
+
+public class TestRowErrors extends BaseKuduTest {
+
+ private static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+
+ }
+
+ @Test(timeout = 100000)
+ public void singleTabletTest() throws Exception {
+ String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis();
+ createTable(tableName, basicSchema, getBasicCreateTableOptions());
+ table = openTable(tableName);
+ AsyncKuduSession session = client.newSession();
+
+ // Insert 3 rows to play with.
+ for (int i = 0; i < 3; i++) {
+ session.apply(createInsert(i)).join(DEFAULT_SLEEP);
+ }
+
+ // Try a single dupe row insert with AUTO_FLUSH_SYNC.
+ Insert dupeForZero = createInsert(0);
+ OperationResponse resp = session.apply(dupeForZero).join(DEFAULT_SLEEP);
+ assertTrue(resp.hasRowError());
+ assertTrue(resp.getRowError().getOperation() == dupeForZero);
+
+ // Now try inserting two dupes and one good row, make sure we get only two errors back.
+ dupeForZero = createInsert(0);
+ Insert dupeForTwo = createInsert(2);
+ session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
+ session.apply(dupeForZero);
+ session.apply(dupeForTwo);
+ session.apply(createInsert(4));
+
+ List<OperationResponse> responses = session.flush().join(DEFAULT_SLEEP);
+ List<RowError> errors = OperationResponse.collectErrors(responses);
+ assertEquals(2, errors.size());
+ assertTrue(errors.get(0).getOperation() == dupeForZero);
+ assertTrue(errors.get(1).getOperation() == dupeForTwo);
+ }
+
+ /**
+ * Test collecting errors from multiple tablets.
+ * @throws Exception
+ */
+ @Test(timeout = 100000)
+ public void multiTabletTest() throws Exception {
+ String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis();
+ createFourTabletsTableWithNineRows(tableName);
+ table = openTable(tableName);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
+
+ int dupRows = 3;
+ session.apply(createInsert(12));
+ session.apply(createInsert(22));
+ session.apply(createInsert(32));
+
+ session.flush();
+
+ RowErrorsAndOverflowStatus reos = session.getPendingErrors();
+ assertEquals(dupRows, reos.getRowErrors().length);
+ assertEquals(0, session.countPendingErrors());
+ }
+
+ private Insert createInsert(int key) {
+ return createBasicSchemaInsert(table, key);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java
new file mode 100644
index 0000000..1b302c1
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestRowResult.java
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kududb.Type;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestRowResult extends BaseKuduTest {
+
+ // Generate a unique table name
+ private static final String TABLE_NAME =
+ TestRowResult.class.getName() + "-" + System.currentTimeMillis();
+
+ private static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ createTable(TABLE_NAME, allTypesSchema, getAllTypesCreateTableOptions());
+ table = openTable(TABLE_NAME);
+ }
+
+ @Test(timeout = 10000)
+ public void test() throws Exception {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+
+ row.addByte(0, (byte) 1);
+ row.addShort(1, (short) 2);
+ row.addInt(2, 3);
+ row.addLong(3, 4l);
+ row.addBoolean(4, true);
+ row.addFloat(5, 5.6f);
+ row.addDouble(6, 7.8);
+ row.addString(7, "string-value");
+ row.addBinary(8, "binary-array".getBytes());
+ ByteBuffer bb = ByteBuffer.wrap("binary-bytebuffer".getBytes());
+ bb.position(7); // We're only inserting the bytebuffer part of the original array.
+ row.addBinary(9, bb);
+ row.setNull(10);
+ row.addLong(11, 11l);
+
+ KuduSession session = syncClient.newSession();
+ session.apply(insert);
+
+ KuduScanner scanner = syncClient.newScannerBuilder(table).build();
+ while (scanner.hasMoreRows()) {
+ RowResultIterator it = scanner.nextRows();
+ assertTrue(it.hasNext());
+ RowResult rr = it.next();
+
+ assertEquals((byte) 1, rr.getByte(0));
+ assertEquals((byte) 1, rr.getByte(allTypesSchema.getColumnByIndex(0).getName()));
+
+ assertEquals((short) 2, rr.getShort(1));
+ assertEquals((short) 2, rr.getShort(allTypesSchema.getColumnByIndex(1).getName()));
+
+ assertEquals(3, rr.getInt(2));
+ assertEquals(3, rr.getInt(allTypesSchema.getColumnByIndex(2).getName()));
+
+ assertEquals(4, rr.getLong(3));
+ assertEquals(4, rr.getLong(allTypesSchema.getColumnByIndex(3).getName()));
+
+ assertEquals(true, rr.getBoolean(4));
+ assertEquals(true, rr.getBoolean(allTypesSchema.getColumnByIndex(4).getName()));
+
+ assertEquals(5.6f, rr.getFloat(5), .001f);
+ assertEquals(5.6f, rr.getFloat(allTypesSchema.getColumnByIndex(5).getName()), .001f);
+
+ assertEquals(7.8, rr.getDouble(6), .001);
+ assertEquals(7.8, rr.getDouble(allTypesSchema.getColumnByIndex(6).getName()), .001f);
+
+ assertEquals("string-value", rr.getString(7));
+ assertEquals("string-value", rr.getString(allTypesSchema.getColumnByIndex(7).getName()));
+
+ assertArrayEquals("binary-array".getBytes(), rr.getBinaryCopy(8));
+ assertArrayEquals("binary-array".getBytes(),
+ rr.getBinaryCopy(allTypesSchema.getColumnByIndex(8).getName()));
+
+ ByteBuffer buffer = rr.getBinary(8);
+ assertEquals(buffer, rr.getBinary(allTypesSchema.getColumnByIndex(8).getName()));
+ byte[] binaryValue = new byte[buffer.remaining()];
+ buffer.get(binaryValue);
+ assertArrayEquals("binary-array".getBytes(), binaryValue);
+
+ assertArrayEquals("bytebuffer".getBytes(), rr.getBinaryCopy(9));
+
+ assertEquals(true, rr.isNull(10));
+ assertEquals(true, rr.isNull(allTypesSchema.getColumnByIndex(10).getName()));
+
+ assertEquals(11, rr.getLong(11));
+ assertEquals(11, rr.getLong(allTypesSchema.getColumnByIndex(11).getName()));
+
+ // We test with the column name once since it's the same method for all types, unlike above.
+ assertEquals(Type.INT8, rr.getColumnType(allTypesSchema.getColumnByIndex(0).getName()));
+ assertEquals(Type.INT8, rr.getColumnType(0));
+ assertEquals(Type.INT16, rr.getColumnType(1));
+ assertEquals(Type.INT32, rr.getColumnType(2));
+ assertEquals(Type.INT64, rr.getColumnType(3));
+ assertEquals(Type.BOOL, rr.getColumnType(4));
+ assertEquals(Type.FLOAT, rr.getColumnType(5));
+ assertEquals(Type.DOUBLE, rr.getColumnType(6));
+ assertEquals(Type.STRING, rr.getColumnType(7));
+ assertEquals(Type.BINARY, rr.getColumnType(8));
+ assertEquals(Type.TIMESTAMP, rr.getColumnType(11));
+ }
+ }
+}
[17/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationReceived.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationReceived.java b/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationReceived.java
deleted file mode 100644
index f1a9075..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationReceived.java
+++ /dev/null
@@ -1,231 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Functions;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.common.net.HostAndPort;
-import com.google.protobuf.ByteString;
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.Common;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
-import org.kududb.util.NetUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Class grouping the callback and the errback for GetMasterRegistration calls
- * made in getMasterTableLocationsPB.
- */
-@InterfaceAudience.Private
-final class GetMasterRegistrationReceived {
-
- private static final Logger LOG = LoggerFactory.getLogger(GetMasterRegistrationReceived.class);
-
- private final List<HostAndPort> masterAddrs;
- private final Deferred<Master.GetTableLocationsResponsePB> responseD;
- private final int numMasters;
-
- // Used to avoid calling 'responseD' twice.
- private final AtomicBoolean responseDCalled = new AtomicBoolean(false);
-
- // Number of responses we've receives: used to tell whether or not we've received
- // errors/replies from all of the masters, or if there are any
- // GetMasterRegistrationRequests still pending.
- private final AtomicInteger countResponsesReceived = new AtomicInteger(0);
-
- // Exceptions received so far: kept for debugging purposes.
- // (see: NoLeaderMasterFoundException#create() for how this is used).
- private final List<Exception> exceptionsReceived =
- Collections.synchronizedList(new ArrayList<Exception>());
-
- /**
- * Creates an object that holds the state needed to retrieve master table's location.
- * @param masterAddrs Addresses of all master replicas that we want to retrieve the
- * registration from.
- * @param responseD Deferred object that will hold the GetTableLocationsResponsePB object for
- * the master table.
- */
- public GetMasterRegistrationReceived(List<HostAndPort> masterAddrs,
- Deferred<Master.GetTableLocationsResponsePB> responseD) {
- this.masterAddrs = masterAddrs;
- this.responseD = responseD;
- this.numMasters = masterAddrs.size();
- }
-
- /**
- * Creates a callback for a GetMasterRegistrationRequest that was sent to 'hostAndPort'.
- * @see GetMasterRegistrationCB
- * @param hostAndPort Host and part for the RPC we're attaching this to. Host and port must
- * be valid.
- * @return The callback object that can be added to the RPC request.
- */
- public Callback<Void, GetMasterRegistrationResponse> callbackForNode(HostAndPort hostAndPort) {
- return new GetMasterRegistrationCB(hostAndPort);
- }
-
- /**
- * Creates an errback for a GetMasterRegistrationRequest that was sent to 'hostAndPort'.
- * @see GetMasterRegistrationErrCB
- * @param hostAndPort Host and port for the RPC we're attaching this to. Used for debugging
- * purposes.
- * @return The errback object that can be added to the RPC request.
- */
- public Callback<Void, Exception> errbackForNode(HostAndPort hostAndPort) {
- return new GetMasterRegistrationErrCB(hostAndPort);
- }
-
- /**
- * Checks if we've already received a response or an exception from every master that
- * we've sent a GetMasterRegistrationRequest to. If so -- and no leader has been found
- * (that is, 'responseD' was never called) -- pass a {@link NoLeaderMasterFoundException}
- * to responseD.
- */
- private void incrementCountAndCheckExhausted() {
- if (countResponsesReceived.incrementAndGet() == numMasters) {
- if (responseDCalled.compareAndSet(false, true)) {
- boolean allUnrecoverable = true;
- for (Exception ex : exceptionsReceived) {
- if (!(ex instanceof NonRecoverableException)) {
- allUnrecoverable = false;
- break;
- }
- }
- String allHosts = NetUtil.hostsAndPortsToString(masterAddrs);
- // Doing a negative check because allUnrecoverable stays true if there are no exceptions.
- if (!allUnrecoverable) {
- String message = "Master config (" + allHosts + ") has no leader.";
- Exception ex;
- if (exceptionsReceived.isEmpty()) {
- LOG.warn("None of the provided masters (" + allHosts + ") is a leader, will retry.");
- ex = new NoLeaderMasterFoundException(Status.ServiceUnavailable(message));
- } else {
- LOG.warn("Unable to find the leader master (" + allHosts + "), will retry");
- String joinedMsg = message + ". Exceptions received: " +
- Joiner.on(",").join(
- Lists.transform(exceptionsReceived, Functions.toStringFunction()));
- Status statusServiceUnavailable = Status.ServiceUnavailable(joinedMsg);
- ex = new NoLeaderMasterFoundException(
- statusServiceUnavailable,
- exceptionsReceived.get(exceptionsReceived.size() - 1));
- }
- responseD.callback(ex);
- } else {
- Status statusConfigurationError = Status.ConfigurationError(
- "Couldn't find a valid master in (" + allHosts +
- "), exceptions: " + exceptionsReceived);
- // This will stop retries.
- responseD.callback(new NonRecoverableException(statusConfigurationError));
- }
- }
- }
- }
-
- /**
- * Callback for each GetMasterRegistrationRequest sent in getMasterTableLocations() above.
- * If a request (paired to a specific master) returns a reply that indicates it's a leader,
- * the callback in 'responseD' is invoked with an initialized GetTableLocationResponsePB
- * object containing the leader's RPC address.
- * If the master is not a leader, increment 'countResponsesReceived': if the count equals to
- * the number of masters, pass {@link NoLeaderMasterFoundException} into
- * 'responseD' if no one else had called 'responseD' before; otherwise, do nothing.
- */
- final class GetMasterRegistrationCB implements Callback<Void, GetMasterRegistrationResponse> {
- private final HostAndPort hostAndPort;
-
- public GetMasterRegistrationCB(HostAndPort hostAndPort) {
- this.hostAndPort = hostAndPort;
- }
-
- @Override
- public Void call(GetMasterRegistrationResponse r) throws Exception {
- Master.TabletLocationsPB.ReplicaPB.Builder replicaBuilder =
- Master.TabletLocationsPB.ReplicaPB.newBuilder();
-
- Master.TSInfoPB.Builder tsInfoBuilder = Master.TSInfoPB.newBuilder();
- tsInfoBuilder.addRpcAddresses(ProtobufHelper.hostAndPortToPB(hostAndPort));
- tsInfoBuilder.setPermanentUuid(r.getInstanceId().getPermanentUuid());
- replicaBuilder.setTsInfo(tsInfoBuilder);
- if (r.getRole().equals(Metadata.RaftPeerPB.Role.LEADER)) {
- replicaBuilder.setRole(r.getRole());
- Master.TabletLocationsPB.Builder locationBuilder = Master.TabletLocationsPB.newBuilder();
- locationBuilder.setPartition(
- Common.PartitionPB.newBuilder().setPartitionKeyStart(ByteString.EMPTY)
- .setPartitionKeyEnd(ByteString.EMPTY));
- locationBuilder.setTabletId(
- ByteString.copyFromUtf8(AsyncKuduClient.MASTER_TABLE_NAME_PLACEHOLDER));
- locationBuilder.addReplicas(replicaBuilder);
- // No one else has called this before us.
- if (responseDCalled.compareAndSet(false, true)) {
- responseD.callback(
- Master.GetTableLocationsResponsePB.newBuilder().addTabletLocations(
- locationBuilder.build()).build()
- );
- } else {
- LOG.debug("Callback already invoked, discarding response(" + r.toString() + ") from " +
- hostAndPort.toString());
- }
- } else {
- incrementCountAndCheckExhausted();
- }
- return null;
- }
-
- @Override
- public String toString() {
- return "get master registration for " + hostAndPort.toString();
- }
- }
-
- /**
- * Errback for each GetMasterRegistrationRequest sent in getMasterTableLocations() above.
- * Stores each exception in 'exceptionsReceived'. Increments 'countResponseReceived': if
- * the count is equal to the number of masters and no one else had called 'responseD' before,
- * pass a {@link NoLeaderMasterFoundException} into 'responseD'; otherwise, do
- * nothing.
- */
- final class GetMasterRegistrationErrCB implements Callback<Void, Exception> {
- private final HostAndPort hostAndPort;
-
- public GetMasterRegistrationErrCB(HostAndPort hostAndPort) {
- this.hostAndPort = hostAndPort;
- }
-
- @Override
- public Void call(Exception e) throws Exception {
- LOG.warn("Error receiving a response from: " + hostAndPort, e);
- exceptionsReceived.add(e);
- incrementCountAndCheckExhausted();
- return null;
- }
-
- @Override
- public String toString() {
- return "get master registration errback for " + hostAndPort.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationRequest.java b/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationRequest.java
deleted file mode 100644
index bc3d81e..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationRequest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import static org.kududb.consensus.Metadata.*;
-import static org.kududb.master.Master.*;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * Package-private RPC that can only go to master.
- */
-@InterfaceAudience.Private
-public class GetMasterRegistrationRequest extends KuduRpc<GetMasterRegistrationResponse> {
- private static final String GET_MASTER_REGISTRATION = "GetMasterRegistration";
-
- public GetMasterRegistrationRequest(KuduTable masterTable) {
- super(masterTable);
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- final GetMasterRegistrationRequestPB.Builder builder =
- GetMasterRegistrationRequestPB.newBuilder();
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return GET_MASTER_REGISTRATION;
- }
-
- @Override
- Pair<GetMasterRegistrationResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- final GetMasterRegistrationResponsePB.Builder respBuilder =
- GetMasterRegistrationResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), respBuilder);
- RaftPeerPB.Role role = RaftPeerPB.Role.FOLLOWER;
- if (!respBuilder.hasError() || respBuilder.getError().getCode() !=
- MasterErrorPB.Code.CATALOG_MANAGER_NOT_INITIALIZED) {
- role = respBuilder.getRole();
- }
- GetMasterRegistrationResponse response = new GetMasterRegistrationResponse(
- deadlineTracker.getElapsedMillis(),
- tsUUID,
- role,
- respBuilder.getRegistration(),
- respBuilder.getInstanceId());
- return new Pair<GetMasterRegistrationResponse, Object>(
- response, respBuilder.hasError() ? respBuilder.getError() : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationResponse.java b/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationResponse.java
deleted file mode 100644
index 292710c..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/GetMasterRegistrationResponse.java
+++ /dev/null
@@ -1,88 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
-
-/**
- * Response for {@link GetMasterRegistrationRequest}.
- */
-@InterfaceAudience.Private
-public class GetMasterRegistrationResponse extends KuduRpcResponse {
-
- private final Metadata.RaftPeerPB.Role role;
- private final WireProtocol.ServerRegistrationPB serverRegistration;
- private final WireProtocol.NodeInstancePB instanceId;
-
- /**
- * Describes a response to a {@link GetMasterRegistrationRequest}, built from
- * {@link Master.GetMasterRegistrationResponsePB}.
- *
- * @param role Master's role in the config.
- * @param serverRegistration server registration (RPC and HTTP addresses) for this master.
- * @param instanceId Node instance (permanent uuid and
- */
- public GetMasterRegistrationResponse(long elapsedMillis, String tsUUID,
- Metadata.RaftPeerPB.Role role,
- WireProtocol.ServerRegistrationPB serverRegistration,
- WireProtocol.NodeInstancePB instanceId) {
- super(elapsedMillis, tsUUID);
- this.role = role;
- this.serverRegistration = serverRegistration;
- this.instanceId = instanceId;
- }
-
- /**
- * Returns this master's role in the config.
- *
- * @see Metadata.RaftPeerPB.Role
- * @return Node's role in the cluster, or FOLLOWER if the node is not initialized.
- */
- public Metadata.RaftPeerPB.Role getRole() {
- return role;
- }
-
- /**
- * Returns the server registration (list of RPC and HTTP ports) for this master.
- *
- * @return The {@link WireProtocol.ServerRegistrationPB} object for this master.
- */
- public WireProtocol.ServerRegistrationPB getServerRegistration() {
- return serverRegistration;
- }
-
- /**
- * The node instance (initial sequence number and permanent uuid) for this master.
- *
- * @return The {@link WireProtocol.NodeInstancePB} object for this master.
- */
- public WireProtocol.NodeInstancePB getInstanceId() {
- return instanceId;
- }
-
- @Override
- public String toString() {
- return "GetMasterRegistrationResponse{" +
- "role=" + role +
- ", serverRegistration=" + serverRegistration +
- ", instanceId=" + instanceId +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/GetTableLocationsRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/GetTableLocationsRequest.java b/java/kudu-client/src/main/java/org/kududb/client/GetTableLocationsRequest.java
deleted file mode 100644
index 616b523..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/GetTableLocationsRequest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.Message;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * Package-private RPC that can only go to a master.
- */
-@InterfaceAudience.Private
-class GetTableLocationsRequest extends KuduRpc<Master.GetTableLocationsResponsePB> {
-
- private final byte[] startPartitionKey;
- private final byte[] endKey;
- private final String tableId;
-
- GetTableLocationsRequest(KuduTable table, byte[] startPartitionKey,
- byte[] endPartitionKey, String tableId) {
- super(table);
- if (startPartitionKey != null && endPartitionKey != null
- && Bytes.memcmp(startPartitionKey, endPartitionKey) > 0) {
- throw new IllegalArgumentException(
- "The start partition key must be smaller or equal to the end partition key");
- }
- this.startPartitionKey = startPartitionKey;
- this.endKey = endPartitionKey;
- this.tableId = tableId;
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return "GetTableLocations";
- }
-
- @Override
- Pair<Master.GetTableLocationsResponsePB, Object> deserialize(
- final CallResponse callResponse, String tsUUID)
- throws Exception {
- Master.GetTableLocationsResponsePB.Builder builder = Master.GetTableLocationsResponsePB
- .newBuilder();
- readProtobuf(callResponse.getPBMessage(), builder);
- Master.GetTableLocationsResponsePB resp = builder.build();
- return new Pair<Master.GetTableLocationsResponsePB, Object>(
- resp, builder.hasError() ? builder.getError() : null);
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- final Master.GetTableLocationsRequestPB.Builder builder = Master
- .GetTableLocationsRequestPB.newBuilder();
- builder.setTable(Master.TableIdentifierPB.newBuilder().
- setTableId(ByteString.copyFromUtf8(tableId)));
- if (startPartitionKey != null) {
- builder.setPartitionKeyStart(ZeroCopyLiteralByteString.wrap(startPartitionKey));
- }
- if (endKey != null) {
- builder.setPartitionKeyEnd(ZeroCopyLiteralByteString.wrap(endKey));
- }
- return toChannelBuffer(header, builder.build());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaRequest.java b/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaRequest.java
deleted file mode 100644
index bb17816..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaRequest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import static org.kududb.master.Master.*;
-
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * RPC to fetch a table's schema
- */
-@InterfaceAudience.Private
-public class GetTableSchemaRequest extends KuduRpc<GetTableSchemaResponse> {
- static final String GET_TABLE_SCHEMA = "GetTableSchema";
- private final String name;
-
-
- GetTableSchemaRequest(KuduTable masterTable, String name) {
- super(masterTable);
- this.name = name;
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- final GetTableSchemaRequestPB.Builder builder = GetTableSchemaRequestPB.newBuilder();
- TableIdentifierPB tableID =
- TableIdentifierPB.newBuilder().setTableName(name).build();
- builder.setTable(tableID);
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return GET_TABLE_SCHEMA;
- }
-
- @Override
- Pair<GetTableSchemaResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- final GetTableSchemaResponsePB.Builder respBuilder = GetTableSchemaResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), respBuilder);
- Schema schema = ProtobufHelper.pbToSchema(respBuilder.getSchema());
- GetTableSchemaResponse response = new GetTableSchemaResponse(
- deadlineTracker.getElapsedMillis(),
- tsUUID,
- schema,
- respBuilder.getTableId().toStringUtf8(),
- ProtobufHelper.pbToPartitionSchema(respBuilder.getPartitionSchema(), schema),
- respBuilder.getCreateTableDone());
- return new Pair<GetTableSchemaResponse, Object>(
- response, respBuilder.hasError() ? respBuilder.getError() : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaResponse.java b/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaResponse.java
deleted file mode 100644
index 72ac68e..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/GetTableSchemaResponse.java
+++ /dev/null
@@ -1,79 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-
-@InterfaceAudience.Private
-public class GetTableSchemaResponse extends KuduRpcResponse {
-
- private final Schema schema;
- private final PartitionSchema partitionSchema;
- private final boolean createTableDone;
- private final String tableId;
-
- /**
- * @param ellapsedMillis Time in milliseconds since RPC creation to now
- * @param schema the table's schema
- * @param partitionSchema the table's partition schema
- */
- GetTableSchemaResponse(long ellapsedMillis,
- String tsUUID,
- Schema schema,
- String tableId,
- PartitionSchema partitionSchema,
- boolean createTableDone) {
- super(ellapsedMillis, tsUUID);
- this.schema = schema;
- this.partitionSchema = partitionSchema;
- this.createTableDone = createTableDone;
- this.tableId = tableId;
- }
-
- /**
- * Get the table's schema.
- * @return Table's schema
- */
- public Schema getSchema() {
- return schema;
- }
-
- /**
- * Get the table's partition schema.
- * @return the table's partition schema
- */
- public PartitionSchema getPartitionSchema() {
- return partitionSchema;
- }
-
- /**
- * Tells if the original CreateTable call has completed and the tablets are ready.
- * @return true if the table is created, otherwise false
- */
- public boolean isCreateTableDone() {
- return createTableDone;
- }
-
- /**
- * Get the table's unique identifier.
- * @return the table's tableId
- */
- public String getTableId() {
- return tableId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/HasFailedRpcException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/HasFailedRpcException.java b/java/kudu-client/src/main/java/org/kududb/client/HasFailedRpcException.java
deleted file mode 100644
index 08dda52..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/HasFailedRpcException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Interface implemented by {@link KuduException}s that can tell you which
- * RPC failed.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public interface HasFailedRpcException {
-
- /**
- * Returns the RPC that caused this exception.
- */
- KuduRpc<?> getFailedRpc();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/IPCUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/IPCUtil.java b/java/kudu-client/src/main/java/org/kududb/client/IPCUtil.java
deleted file mode 100644
index 45240dd..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/IPCUtil.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.kududb.client;
-
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Helper methods for RPCs.
- */
-@InterfaceAudience.Private
-public class IPCUtil {
- /**
- * Write out header, param, and cell block if there is one.
- * @param dos
- * @param header
- * @param param
- * @return Total number of bytes written.
- * @throws java.io.IOException
- */
- public static int write(final OutputStream dos, final Message header, final Message param)
- throws IOException {
- // Must calculate total size and write that first so other side can read it all in in one
- // swoop. This is dictated by how the server is currently written. Server needs to change
- // if we are to be able to write without the length prefixing.
- int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(header, param);
- return write(dos, header, param, totalSize);
- }
-
- private static int write(final OutputStream dos, final Message header, final Message param,
- final int totalSize)
- throws IOException {
- // I confirmed toBytes does same as say DataOutputStream#writeInt.
- dos.write(toBytes(totalSize));
- header.writeDelimitedTo(dos);
- if (param != null) param.writeDelimitedTo(dos);
- dos.flush();
- return totalSize;
- }
-
- /**
- * @return Size on the wire when the two messages are written with writeDelimitedTo
- */
- public static int getTotalSizeWhenWrittenDelimited(Message ... messages) {
- int totalSize = 0;
- for (Message m: messages) {
- if (m == null) continue;
- totalSize += m.getSerializedSize();
- totalSize += CodedOutputStream.computeRawVarint32Size(m.getSerializedSize());
- }
- return totalSize;
- }
-
- public static byte[] toBytes(int val) {
- byte [] b = new byte[4];
- for(int i = 3; i > 0; i--) {
- b[i] = (byte) val;
- val >>>= 8;
- }
- b[0] = (byte) val;
- return b;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Insert.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Insert.java b/java/kudu-client/src/main/java/org/kududb/client/Insert.java
deleted file mode 100644
index 67b389f..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Insert.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Represents a single row insert. Instances of this class should not be reused.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Insert extends Operation {
-
- Insert(KuduTable table) {
- super(table);
- }
-
- @Override
- ChangeType getChangeType() {
- return ChangeType.INSERT;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneRequest.java b/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneRequest.java
deleted file mode 100644
index ca161f5..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneRequest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import static org.kududb.master.Master.*;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * RPC used to check if an alter is running for the specified table
- */
-@InterfaceAudience.Private
-class IsAlterTableDoneRequest extends KuduRpc<IsAlterTableDoneResponse> {
-
- static final String IS_ALTER_TABLE_DONE = "IsAlterTableDone";
- private final String name;
-
-
- IsAlterTableDoneRequest(KuduTable masterTable, String name) {
- super(masterTable);
- this.name = name;
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- final IsAlterTableDoneRequestPB.Builder builder = IsAlterTableDoneRequestPB.newBuilder();
- TableIdentifierPB tableID =
- TableIdentifierPB.newBuilder().setTableName(name).build();
- builder.setTable(tableID);
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return IS_ALTER_TABLE_DONE;
- }
-
- @Override
- Pair<IsAlterTableDoneResponse, Object> deserialize(final CallResponse callResponse,
- String tsUUID) throws Exception {
- final IsAlterTableDoneResponsePB.Builder respBuilder = IsAlterTableDoneResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), respBuilder);
- IsAlterTableDoneResponse resp = new IsAlterTableDoneResponse(deadlineTracker.getElapsedMillis(),
- tsUUID, respBuilder.getDone());
- return new Pair<IsAlterTableDoneResponse, Object>(
- resp, respBuilder.hasError() ? respBuilder.getError() : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneResponse.java b/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneResponse.java
deleted file mode 100644
index 356c085..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/IsAlterTableDoneResponse.java
+++ /dev/null
@@ -1,44 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Response to a isAlterTableDone command to use to know if an alter table is currently running on
- * the specified table.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class IsAlterTableDoneResponse extends KuduRpcResponse {
-
- private final boolean done;
-
- IsAlterTableDoneResponse(long elapsedMillis, String tsUUID, boolean done) {
- super(elapsedMillis, tsUUID);
- this.done = done;
- }
-
- /**
- * Tells if the table is done being altered or not.
- * @return whether the table alter is done
- */
- public boolean isDone() {
- return done;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/IsCreateTableDoneRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/IsCreateTableDoneRequest.java b/java/kudu-client/src/main/java/org/kududb/client/IsCreateTableDoneRequest.java
deleted file mode 100644
index 8e4679c..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/IsCreateTableDoneRequest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- * Package-private RPC that can only go to a master.
- */
-@InterfaceAudience.Private
-class IsCreateTableDoneRequest extends KuduRpc<Master.IsCreateTableDoneResponsePB> {
-
- private final String tableId;
-
- IsCreateTableDoneRequest(KuduTable table, String tableId) {
- super(table);
- this.tableId = tableId;
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return "IsCreateTableDone";
- }
-
- @Override
- Pair<Master.IsCreateTableDoneResponsePB, Object> deserialize(
- final CallResponse callResponse, String tsUUID) throws Exception {
- Master.IsCreateTableDoneResponsePB.Builder builder = Master.IsCreateTableDoneResponsePB
- .newBuilder();
- readProtobuf(callResponse.getPBMessage(), builder);
- Master.IsCreateTableDoneResponsePB resp = builder.build();
- return new Pair<Master.IsCreateTableDoneResponsePB, Object>(
- resp, builder.hasError() ? builder.getError() : null);
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- final Master.IsCreateTableDoneRequestPB.Builder builder = Master
- .IsCreateTableDoneRequestPB.newBuilder();
- builder.setTable(Master.TableIdentifierPB.newBuilder().setTableId(
- ByteString.copyFromUtf8(tableId)));
- return toChannelBuffer(header, builder.build());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KeyEncoder.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KeyEncoder.java b/java/kudu-client/src/main/java/org/kududb/client/KeyEncoder.java
deleted file mode 100644
index 2fbde58..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KeyEncoder.java
+++ /dev/null
@@ -1,193 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.primitives.UnsignedLongs;
-import com.sangupta.murmur.Murmur2;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.client.PartitionSchema.HashBucketSchema;
-
-import java.io.ByteArrayOutputStream;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.List;
-
-/**
- * Utility class for encoding rows into primary and partition keys.
- */
-@InterfaceAudience.Private
-class KeyEncoder {
-
- private final ByteArrayOutputStream buf = new ByteArrayOutputStream();
-
- /**
- * Encodes the primary key of the row.
- *
- * @param row the row to encode
- * @return the encoded primary key of the row
- */
- public byte[] encodePrimaryKey(final PartialRow row) {
- buf.reset();
-
- final Schema schema = row.getSchema();
- for (int columnIdx = 0; columnIdx < schema.getPrimaryKeyColumnCount(); columnIdx++) {
- final boolean isLast = columnIdx + 1 == schema.getPrimaryKeyColumnCount();
- encodeColumn(row, columnIdx, isLast);
- }
- return extractByteArray();
- }
-
- /**
- * Encodes the provided row into a partition key according to the partition schema.
- *
- * @param row the row to encode
- * @param partitionSchema the partition schema describing the table's partitioning
- * @return an encoded partition key
- */
- public byte[] encodePartitionKey(PartialRow row, PartitionSchema partitionSchema) {
- buf.reset();
- if (!partitionSchema.getHashBucketSchemas().isEmpty()) {
- ByteBuffer bucketBuf = ByteBuffer.allocate(4 * partitionSchema.getHashBucketSchemas().size());
- bucketBuf.order(ByteOrder.BIG_ENDIAN);
-
- for (final HashBucketSchema hashBucketSchema : partitionSchema.getHashBucketSchemas()) {
- encodeColumns(row, hashBucketSchema.getColumnIds());
- byte[] encodedColumns = extractByteArray();
- long hash = Murmur2.hash64(encodedColumns,
- encodedColumns.length,
- hashBucketSchema.getSeed());
- int bucket = (int) UnsignedLongs.remainder(hash, hashBucketSchema.getNumBuckets());
- bucketBuf.putInt(bucket);
- }
-
- assert bucketBuf.arrayOffset() == 0;
- buf.write(bucketBuf.array(), 0, bucketBuf.position());
- }
-
- encodeColumns(row, partitionSchema.getRangeSchema().getColumns());
- return extractByteArray();
- }
-
- /**
- * Encodes a sequence of columns from the row.
- * @param row the row containing the columns to encode
- * @param columnIds the IDs of each column to encode
- */
- private void encodeColumns(PartialRow row, List<Integer> columnIds) {
- for (int i = 0; i < columnIds.size(); i++) {
- boolean isLast = i + 1 == columnIds.size();
- encodeColumn(row, row.getSchema().getColumnIndex(columnIds.get(i)), isLast);
- }
- }
-
- /**
- * Encodes a single column of a row.
- * @param row the row being encoded
- * @param columnIdx the column index of the column to encode
- * @param isLast whether the column is the last component of the key
- */
- private void encodeColumn(PartialRow row, int columnIdx, boolean isLast) {
- final Schema schema = row.getSchema();
- final ColumnSchema column = schema.getColumnByIndex(columnIdx);
- if (!row.isSet(columnIdx)) {
- throw new IllegalStateException(String.format("Primary key column %s is not set",
- column.getName()));
- }
- final Type type = column.getType();
-
- if (type == Type.STRING || type == Type.BINARY) {
- addBinaryComponent(row.getVarLengthData().get(columnIdx), isLast);
- } else {
- addComponent(row.getRowAlloc(),
- schema.getColumnOffset(columnIdx),
- type.getSize(),
- type);
- }
- }
-
- /**
- * Encodes a byte buffer into the key.
- * @param value the value to encode
- * @param isLast whether the value is the final component in the key
- */
- private void addBinaryComponent(ByteBuffer value, boolean isLast) {
- value.reset();
-
- // TODO find a way to not have to read byte-by-byte that doesn't require extra copies. This is
- // especially slow now that users can pass direct byte buffers.
- while (value.hasRemaining()) {
- byte currentByte = value.get();
- buf.write(currentByte);
- if (!isLast && currentByte == 0x00) {
- // If we're a middle component of a composite key, we need to add a \x00
- // at the end in order to separate this component from the next one. However,
- // if we just did that, we'd have issues where a key that actually has
- // \x00 in it would compare wrong, so we have to instead add \x00\x00, and
- // encode \x00 as \x00\x01. -- key_encoder.h
- buf.write(0x01);
- }
- }
-
- if (!isLast) {
- buf.write(0x00);
- buf.write(0x00);
- }
- }
-
- /**
- * Encodes a value of the given type into the key.
- * @param value the value to encode
- * @param offset the offset into the {@code value} buffer that the value begins
- * @param len the length of the value
- * @param type the type of the value to encode
- */
- private void addComponent(byte[] value, int offset, int len, Type type) {
- switch (type) {
- case INT8:
- case INT16:
- case INT32:
- case INT64:
- case TIMESTAMP:
- // Picking the first byte because big endian.
- byte lastByte = value[offset + (len - 1)];
- lastByte = Bytes.xorLeftMostBit(lastByte);
- buf.write(lastByte);
- if (len > 1) {
- for (int i = len - 2; i >= 0; i--) {
- buf.write(value[offset + i]);
- }
- }
- break;
- default:
- throw new IllegalArgumentException(String.format(
- "The column type %s is not a valid key component type", type));
- }
- }
-
- /**
- * Returns the encoded key, and resets the key encoder to be used for another key.
- * @return the encoded key which has been built through calls to {@link #addComponent}
- */
- private byte[] extractByteArray() {
- byte[] bytes = buf.toByteArray();
- buf.reset();
- return bytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
deleted file mode 100644
index cfd4662..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduClient.java
+++ /dev/null
@@ -1,415 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.stumbleupon.async.Deferred;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Executor;
-
-/**
- * A synchronous and thread-safe client for Kudu.
- * <p>
- * This class acts as a wrapper around {@link AsyncKuduClient}. The {@link Deferred} objects are
- * joined against using the default admin operation timeout
- * (see {@link org.kududb.client.KuduClient.KuduClientBuilder#defaultAdminOperationTimeoutMs(long)} (long)}).
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduClient implements AutoCloseable {
-
- public static final Logger LOG = LoggerFactory.getLogger(AsyncKuduClient.class);
-
- private final AsyncKuduClient asyncClient;
-
- KuduClient(AsyncKuduClient asyncClient) {
- this.asyncClient = asyncClient;
- }
-
- /**
- * Create a table on the cluster with the specified name, schema, and table configurations.
- * @param name the table's name
- * @param schema the table's schema
- * @param builder a builder containing the table's configurations
- * @return an object to communicate with the created table
- * @throws KuduException if anything went wrong
- */
- public KuduTable createTable(String name, Schema schema, CreateTableOptions builder)
- throws KuduException {
- Deferred<KuduTable> d = asyncClient.createTable(name, schema, builder);
- return joinAndHandleException(d);
- }
-
- /**
- * Delete a table on the cluster with the specified name.
- * @param name the table's name
- * @return an rpc response object
- * @throws KuduException if anything went wrong
- */
- public DeleteTableResponse deleteTable(String name) throws KuduException {
- Deferred<DeleteTableResponse> d = asyncClient.deleteTable(name);
- return joinAndHandleException(d);
- }
-
- /**
- * Alter a table on the cluster as specified by the builder.
- *
- * When the method returns it only indicates that the master accepted the alter
- * command, use {@link KuduClient#isAlterTableDone(String)} to know when the alter finishes.
- * @param name the table's name, if this is a table rename then the old table name must be passed
- * @param ato the alter table builder
- * @return an rpc response object
- * @throws KuduException if anything went wrong
- */
- public AlterTableResponse alterTable(String name, AlterTableOptions ato) throws KuduException {
- Deferred<AlterTableResponse> d = asyncClient.alterTable(name, ato);
- return joinAndHandleException(d);
- }
-
- /**
- * Helper method that checks and waits until the completion of an alter command.
- * It will block until the alter command is done or the timeout is reached.
- * @param name Table's name, if the table was renamed then that name must be checked against
- * @return a boolean indicating if the table is done being altered
- * @throws KuduException for any error returned by sending RPCs to the master
- */
- public boolean isAlterTableDone(String name) throws KuduException {
- long totalSleepTime = 0;
- while (totalSleepTime < getDefaultAdminOperationTimeoutMs()) {
- long start = System.currentTimeMillis();
-
- try {
- Deferred<IsAlterTableDoneResponse> d = asyncClient.isAlterTableDone(name);
- IsAlterTableDoneResponse response;
-
- response = d.join(AsyncKuduClient.SLEEP_TIME);
- if (response.isDone()) {
- return true;
- }
-
- // Count time that was slept and see if we need to wait a little more.
- long elapsed = System.currentTimeMillis() - start;
- // Don't oversleep the deadline.
- if (totalSleepTime + AsyncKuduClient.SLEEP_TIME > getDefaultAdminOperationTimeoutMs()) {
- return false;
- }
- // elapsed can be bigger if we slept about 500ms
- if (elapsed <= AsyncKuduClient.SLEEP_TIME) {
- LOG.debug("Alter not done, sleep " + (AsyncKuduClient.SLEEP_TIME - elapsed) +
- " and slept " + totalSleepTime);
- Thread.sleep(AsyncKuduClient.SLEEP_TIME - elapsed);
- totalSleepTime += AsyncKuduClient.SLEEP_TIME;
- } else {
- totalSleepTime += elapsed;
- }
- } catch (Exception ex) {
- throw KuduException.transformException(ex);
- }
- }
- return false;
- }
-
- /**
- * Get the list of running tablet servers.
- * @return a list of tablet servers
- * @throws KuduException if anything went wrong
- */
- public ListTabletServersResponse listTabletServers() throws KuduException {
- Deferred<ListTabletServersResponse> d = asyncClient.listTabletServers();
- return joinAndHandleException(d);
- }
-
- /**
- * Get the list of all the tables.
- * @return a list of all the tables
- * @throws KuduException if anything went wrong
- */
- public ListTablesResponse getTablesList() throws KuduException {
- return getTablesList(null);
- }
-
- /**
- * Get a list of table names. Passing a null filter returns all the tables. When a filter is
- * specified, it only returns tables that satisfy a substring match.
- * @param nameFilter an optional table name filter
- * @return a deferred that contains the list of table names
- * @throws KuduException if anything went wrong
- */
- public ListTablesResponse getTablesList(String nameFilter) throws KuduException {
- Deferred<ListTablesResponse> d = asyncClient.getTablesList(nameFilter);
- return joinAndHandleException(d);
- }
-
- /**
- * Test if a table exists.
- * @param name a non-null table name
- * @return true if the table exists, else false
- * @throws KuduException if anything went wrong
- */
- public boolean tableExists(String name) throws KuduException {
- Deferred<Boolean> d = asyncClient.tableExists(name);
- return joinAndHandleException(d);
- }
-
- /**
- * Open the table with the given name. If the table was just created, this method will block until
- * all its tablets have also been created.
- * @param name table to open
- * @return a KuduTable if the table exists
- * @throws KuduException if anything went wrong
- */
- public KuduTable openTable(final String name) throws KuduException {
- Deferred<KuduTable> d = asyncClient.openTable(name);
- return joinAndHandleException(d);
- }
-
- /**
- * Create a new session for interacting with the cluster.
- * User is responsible for destroying the session object.
- * This is a fully local operation (no RPCs or blocking).
- * @return a synchronous wrapper around KuduSession.
- */
- public KuduSession newSession() {
- AsyncKuduSession session = asyncClient.newSession();
- return new KuduSession(session);
- }
-
- /**
- * Check if statistics collection is enabled for this client.
- * @return true if it is enabled, else false
- */
- public boolean isStatisticsEnabled() {
- return asyncClient.isStatisticsEnabled();
- }
-
- /**
- * Get the statistics object of this client.
- *
- * @return this client's Statistics object
- * @throws IllegalStateException thrown if statistics collection has been disabled
- */
- public Statistics getStatistics() {
- return asyncClient.getStatistics();
- }
-
- /**
- * Creates a new {@link KuduScanner.KuduScannerBuilder} for a particular table.
- * @param table the table you intend to scan.
- * The string is assumed to use the platform's default charset.
- * @return a new scanner builder for the table
- */
- public KuduScanner.KuduScannerBuilder newScannerBuilder(KuduTable table) {
- return new KuduScanner.KuduScannerBuilder(asyncClient, table);
- }
-
- /**
- * Creates a new {@link KuduScanToken.KuduScanTokenBuilder} for a particular table.
- * Used for integrations with compute frameworks.
- * @param table the table you intend to scan
- * @return a new scan token builder for the table
- */
- public KuduScanToken.KuduScanTokenBuilder newScanTokenBuilder(KuduTable table) {
- return new KuduScanToken.KuduScanTokenBuilder(asyncClient, table);
- }
-
- /**
- * Analogous to {@link #shutdown()}.
- * @throws KuduException if an error happens while closing the connections
- */
- @Override
- public void close() throws KuduException {
- try {
- asyncClient.close();
- } catch (Exception e) {
- KuduException.transformException(e);
- }
- }
-
- /**
- * Performs a graceful shutdown of this instance.
- * @throws KuduException if anything went wrong
- */
- public void shutdown() throws KuduException {
- Deferred<ArrayList<Void>> d = asyncClient.shutdown();
- joinAndHandleException(d);
- }
-
- /**
- * Get the timeout used for operations on sessions and scanners.
- * @return a timeout in milliseconds
- */
- public long getDefaultOperationTimeoutMs() {
- return asyncClient.getDefaultOperationTimeoutMs();
- }
-
- /**
- * Get the timeout used for admin operations.
- * @return a timeout in milliseconds
- */
- public long getDefaultAdminOperationTimeoutMs() {
- return asyncClient.getDefaultAdminOperationTimeoutMs();
- }
-
- // Helper method to handle joining and transforming the Exception we receive.
- private <R> R joinAndHandleException(Deferred<R> deferred) throws KuduException {
- try {
- return deferred.join(getDefaultAdminOperationTimeoutMs());
- } catch (Exception e) {
- throw KuduException.transformException(e);
- }
- }
-
- /**
- * Builder class to use in order to connect to Kudu.
- * All the parameters beyond those in the constructors are optional.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public final static class KuduClientBuilder {
- private AsyncKuduClient.AsyncKuduClientBuilder clientBuilder;
-
- /**
- * Creates a new builder for a client that will connect to the specified masters.
- * @param masterAddresses comma-separated list of "host:port" pairs of the masters
- */
- public KuduClientBuilder(String masterAddresses) {
- clientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses);
- }
-
- /**
- * Creates a new builder for a client that will connect to the specified masters.
- *
- * <p>Here are some examples of recognized formats:
- * <ul>
- * <li>example.com
- * <li>example.com:80
- * <li>192.0.2.1
- * <li>192.0.2.1:80
- * <li>[2001:db8::1]
- * <li>[2001:db8::1]:80
- * <li>2001:db8::1
- * </ul>
- *
- * @param masterAddresses list of master addresses
- */
- public KuduClientBuilder(List<String> masterAddresses) {
- clientBuilder = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses);
- }
-
- /**
- * Sets the default timeout used for administrative operations (e.g. createTable, deleteTable,
- * etc).
- * Optional.
- * If not provided, defaults to 30s.
- * A value of 0 disables the timeout.
- * @param timeoutMs a timeout in milliseconds
- * @return this builder
- */
- public KuduClientBuilder defaultAdminOperationTimeoutMs(long timeoutMs) {
- clientBuilder.defaultAdminOperationTimeoutMs(timeoutMs);
- return this;
- }
-
- /**
- * Sets the default timeout used for user operations (using sessions and scanners).
- * Optional.
- * If not provided, defaults to 30s.
- * A value of 0 disables the timeout.
- * @param timeoutMs a timeout in milliseconds
- * @return this builder
- */
- public KuduClientBuilder defaultOperationTimeoutMs(long timeoutMs) {
- clientBuilder.defaultOperationTimeoutMs(timeoutMs);
- return this;
- }
-
- /**
- * Sets the default timeout to use when waiting on data from a socket.
- * Optional.
- * If not provided, defaults to 10s.
- * A value of 0 disables the timeout.
- * @param timeoutMs a timeout in milliseconds
- * @return this builder
- */
- public KuduClientBuilder defaultSocketReadTimeoutMs(long timeoutMs) {
- clientBuilder.defaultSocketReadTimeoutMs(timeoutMs);
- return this;
- }
-
- /**
- * Disable this client's collection of statistics.
- * Statistics are enabled by default.
- * @return this builder
- */
- public KuduClientBuilder disableStatistics() {
- clientBuilder.disableStatistics();
- return this;
- }
-
- /**
- * Set the executors which will be used for the embedded Netty boss and workers.
- * Optional.
- * If not provided, uses a simple cached threadpool. If either argument is null,
- * then such a thread pool will be used in place of that argument.
- * Note: executor's max thread number must be greater or equal to corresponding
- * worker count, or netty cannot start enough threads, and client will get stuck.
- * If not sure, please just use CachedThreadPool.
- */
- public KuduClientBuilder nioExecutors(Executor bossExecutor, Executor workerExecutor) {
- clientBuilder.nioExecutors(bossExecutor, workerExecutor);
- return this;
- }
-
- /**
- * Set the maximum number of boss threads.
- * Optional.
- * If not provided, 1 is used.
- */
- public KuduClientBuilder bossCount(int bossCount) {
- clientBuilder.bossCount(bossCount);
- return this;
- }
-
- /**
- * Set the maximum number of worker threads.
- * Optional.
- * If not provided, (2 * the number of available processors) is used.
- */
- public KuduClientBuilder workerCount(int workerCount) {
- clientBuilder.workerCount(workerCount);
- return this;
- }
-
- /**
- * Creates a new client that connects to the masters.
- * Doesn't block and won't throw an exception if the masters don't exist.
- * @return a new asynchronous Kudu client
- */
- public KuduClient build() {
- AsyncKuduClient client = clientBuilder.build();
- return new KuduClient(client);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduException.java b/java/kudu-client/src/main/java/org/kududb/client/KuduException.java
deleted file mode 100644
index 4bd2eaf..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduException.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import com.stumbleupon.async.DeferredGroupException;
-import com.stumbleupon.async.TimeoutException;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.io.IOException;
-
-/**
- * The parent class of all exceptions sent by the Kudu client. This is the only exception you will
- * see if you're using the non-async API, such as {@link KuduSession} instead of
- * {@link AsyncKuduSession}.
- *
- * Each instance of this class has a {@link Status} which gives more information about the error.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-@SuppressWarnings("serial")
-public abstract class KuduException extends IOException {
-
- private final Status status;
-
- /**
- * Constructor.
- * @param status object containing the reason for the exception
- * trace.
- */
- KuduException(Status status) {
- super(status.getMessage());
- this.status = status;
- }
-
- /**
- * Constructor.
- * @param status object containing the reason for the exception
- * @param cause The exception that caused this one to be thrown.
- */
- KuduException(Status status, Throwable cause) {
- super(status.getMessage(), cause);
- this.status = status;
- }
-
- /**
- * Get the Status object for this exception.
- * @return a status object indicating the reason for the exception
- */
- public Status getStatus() {
- return status;
- }
-
- /**
- * Inspects the given exception and transforms it into a KuduException.
- * @param e generic exception we want to transform
- * @return a KuduException that's easier to handle
- */
- static KuduException transformException(Exception e) {
- if (e instanceof KuduException) {
- return (KuduException) e;
- } else if (e instanceof DeferredGroupException) {
- // TODO anything we can do to improve on that kind of exception?
- } else if (e instanceof TimeoutException) {
- Status statusTimeout = Status.TimedOut(e.getMessage());
- return new NonRecoverableException(statusTimeout, e);
- } else if (e instanceof InterruptedException) {
- // Need to reset the interrupt flag since we caught it but aren't handling it.
- Thread.currentThread().interrupt();
- Status statusAborted = Status.Aborted(e.getMessage());
- return new NonRecoverableException(statusAborted, e);
- }
- Status status = Status.IOError(e.getMessage());
- return new NonRecoverableException(status, e);
- }
-}
[07/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestFlexiblePartitioning.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestFlexiblePartitioning.java b/java/kudu-client/src/test/java/org/kududb/client/TestFlexiblePartitioning.java
deleted file mode 100644
index dafd74a..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestFlexiblePartitioning.java
+++ /dev/null
@@ -1,422 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Sets;
-import org.junit.Before;
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-public class TestFlexiblePartitioning extends BaseKuduTest {
- private String tableName;
-
- @Before
- public void setTableName() {
- tableName = TestKuduClient.class.getName() + "-" + System.currentTimeMillis();
- }
-
- private static Schema createSchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<>(3);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("a", Type.STRING).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("b", Type.STRING).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c", Type.STRING).key(true).build());
- return new Schema(columns);
- }
-
- private static Set<Row> rows() throws Exception {
- Set<Row> rows = new HashSet<>();
- for (int a = 0; a < 6; a++) {
- for (int b = 0; b < 6; b++) {
- for (int c = 0; c < 6; c++) {
- rows.add(new Row(String.format("%s", a),
- String.format("%s", b),
- String.format("%s", c)));
- }
- }
- }
- return rows;
- }
-
- private void insertRows(KuduTable table, Set<Row> rows) throws Exception {
- KuduSession session = syncClient.newSession();
- try {
- for (Row row : rows) {
- Insert insert = table.newInsert();
- PartialRow insertRow = insert.getRow();
- row.fillPartialRow(insertRow);
- session.apply(insert);
- }
- } finally {
- session.close();
- }
- }
-
- private Set<Row> collectRows(KuduScanner scanner) throws Exception {
- Set<Row> rows = new HashSet<>();
- while (scanner.hasMoreRows()) {
- for (RowResult result : scanner.nextRows()) {
- rows.add(Row.fromResult(result));
- }
- }
- return rows;
- }
-
- private void testPartitionSchema(CreateTableOptions tableBuilder) throws Exception {
- Schema schema = createSchema();
-
- syncClient.createTable(tableName, schema, tableBuilder);
-
- KuduTable table = syncClient.openTable(tableName);
-
- Set<Row> rows = rows();
- insertRows(table, rows);
-
- // Full table scan
- assertEquals(rows, collectRows(syncClient.newScannerBuilder(table).build()));
-
- { // Lower bound
- Row minRow = new Row("1", "3", "5");
- PartialRow lowerBound = schema.newPartialRow();
- minRow.fillPartialRow(lowerBound);
-
- Set<Row> expected = Sets.filter(rows, minRow.gtePred());
-
- KuduScanner scanner = syncClient.newScannerBuilder(table).lowerBound(lowerBound).build();
- Set<Row> results = collectRows(scanner);
-
- assertEquals(expected, results);
- }
-
- { // Upper bound
- Row maxRow = new Row("1", "3", "5");
- PartialRow upperBound = schema.newPartialRow();
- maxRow.fillPartialRow(upperBound);
-
- Set<Row> expected = Sets.filter(rows, maxRow.ltPred());
-
- KuduScanner scanner = syncClient.newScannerBuilder(table)
- .exclusiveUpperBound(upperBound)
- .build();
- Set<Row> results = collectRows(scanner);
-
- assertEquals(expected, results);
- }
-
- { // Lower & Upper bounds
- Row minRow = new Row("1", "3", "5");
- Row maxRow = new Row("2", "4", "");
- PartialRow lowerBound = schema.newPartialRow();
- minRow.fillPartialRow(lowerBound);
- PartialRow upperBound = schema.newPartialRow();
- maxRow.fillPartialRow(upperBound);
-
- Set<Row> expected = Sets.filter(rows, Predicates.and(minRow.gtePred(), maxRow.ltPred()));
-
- KuduScanner scanner = syncClient.newScannerBuilder(table)
- .lowerBound(lowerBound)
- .exclusiveUpperBound(upperBound)
- .build();
- Set<Row> results = collectRows(scanner);
-
- assertEquals(expected, results);
- }
-
- List<LocatedTablet> tablets = table.getTabletsLocations(TestTimeouts.DEFAULT_SLEEP);
-
- { // Per-tablet scan
- Set<Row> results = new HashSet<>();
-
- for (LocatedTablet tablet : tablets) {
- KuduScanner scanner = syncClient.newScannerBuilder(table)
- .lowerBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyStart())
- .exclusiveUpperBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyEnd())
- .build();
- Set<Row> tabletResults = collectRows(scanner);
- Set<Row> intersection = Sets.intersection(results, tabletResults);
- assertEquals(new HashSet<>(), intersection);
- results.addAll(tabletResults);
- }
-
- assertEquals(rows, results);
- }
-
- { // Per-tablet scan with lower & upper bounds
- Row minRow = new Row("1", "3", "5");
- Row maxRow = new Row("2", "4", "");
- PartialRow lowerBound = schema.newPartialRow();
- minRow.fillPartialRow(lowerBound);
- PartialRow upperBound = schema.newPartialRow();
- maxRow.fillPartialRow(upperBound);
-
- Set<Row> expected = Sets.filter(rows, Predicates.and(minRow.gtePred(), maxRow.ltPred()));
- Set<Row> results = new HashSet<>();
-
- for (LocatedTablet tablet : tablets) {
- KuduScanner scanner = syncClient.newScannerBuilder(table)
- .lowerBound(lowerBound)
- .exclusiveUpperBound(upperBound)
- .lowerBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyStart())
- .exclusiveUpperBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyEnd())
- .build();
- Set<Row> tabletResults = collectRows(scanner);
- Set<Row> intersection = Sets.intersection(results, tabletResults);
- assertEquals(new HashSet<>(), intersection);
- results.addAll(tabletResults);
- }
-
- assertEquals(expected, results);
- }
- }
-
- @Test(timeout = 100000)
- public void testHashBucketedTable() throws Exception {
- CreateTableOptions tableBuilder = new CreateTableOptions();
- tableBuilder.addHashPartitions(ImmutableList.of("a"), 3);
- tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42);
- tableBuilder.setRangePartitionColumns(ImmutableList.<String>of());
- testPartitionSchema(tableBuilder);
- }
-
- @Test(timeout = 100000)
- public void testNonDefaultRangePartitionedTable() throws Exception {
- Schema schema = createSchema();
- CreateTableOptions tableBuilder = new CreateTableOptions();
- tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b"));
-
- PartialRow split = schema.newPartialRow();
- split.addString("c", "3");
- tableBuilder.addSplitRow(split);
-
- split = schema.newPartialRow();
- split.addString("c", "3");
- split.addString("b", "3");
- tableBuilder.addSplitRow(split);
-
- testPartitionSchema(tableBuilder);
- }
-
- @Test(timeout = 100000)
- public void testHashBucketedAndRangePartitionedTable() throws Exception {
- Schema schema = createSchema();
- CreateTableOptions tableBuilder = new CreateTableOptions();
- tableBuilder.addHashPartitions(ImmutableList.of("a"), 3);
- tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42);
- tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b"));
-
- PartialRow split = schema.newPartialRow();
- split.addString("c", "3");
- tableBuilder.addSplitRow(split);
-
- split = schema.newPartialRow();
- split.addString("c", "3");
- split.addString("b", "3");
- tableBuilder.addSplitRow(split);
-
- testPartitionSchema(tableBuilder);
- }
-
- @Test(timeout = 100000)
- public void testNonCoveredRangePartitionedTable() throws Exception {
- Schema schema = createSchema();
- CreateTableOptions tableBuilder = new CreateTableOptions();
- tableBuilder.setRangePartitionColumns(ImmutableList.of("a", "b", "c"));
-
- // Create a non covered range between (3, 5, 6) and (4, 0, 0)
-
- PartialRow lowerBoundA = schema.newPartialRow();
- lowerBoundA.addString("a", "0");
- lowerBoundA.addString("b", "0");
- lowerBoundA.addString("c", "0");
- PartialRow upperBoundA = schema.newPartialRow();
- upperBoundA.addString("a", "3");
- upperBoundA.addString("b", "5");
- upperBoundA.addString("b", "6");
- tableBuilder.addRangeBound(lowerBoundA, upperBoundA);
-
- PartialRow lowerBoundB = schema.newPartialRow();
- lowerBoundB.addString("a", "4");
- lowerBoundB.addString("b", "0");
- lowerBoundB.addString("c", "0");
- PartialRow upperBoundB = schema.newPartialRow();
- upperBoundB.addString("a", "5");
- upperBoundB.addString("b", "5");
- upperBoundB.addString("b", "6");
- tableBuilder.addRangeBound(lowerBoundB, upperBoundB);
-
- testPartitionSchema(tableBuilder);
- }
-
- @Test(timeout = 100000)
- public void testHashBucketedAndNonCoveredRangePartitionedTable() throws Exception {
- Schema schema = createSchema();
- CreateTableOptions tableBuilder = new CreateTableOptions();
- tableBuilder.setRangePartitionColumns(ImmutableList.of("a", "b", "c"));
-
- // Create a non covered range between (3, 5, 6) and (4, 0, 0)
-
- PartialRow lowerBoundA = schema.newPartialRow();
- lowerBoundA.addString("a", "0");
- lowerBoundA.addString("b", "0");
- lowerBoundA.addString("c", "0");
- PartialRow upperBoundA = schema.newPartialRow();
- upperBoundA.addString("a", "3");
- upperBoundA.addString("b", "5");
- upperBoundA.addString("c", "6");
- tableBuilder.addRangeBound(lowerBoundA, upperBoundA);
-
- PartialRow lowerBoundB = schema.newPartialRow();
- lowerBoundB.addString("a", "4");
- lowerBoundB.addString("b", "0");
- lowerBoundB.addString("c", "0");
- PartialRow upperBoundB = schema.newPartialRow();
- upperBoundB.addString("a", "5");
- upperBoundB.addString("b", "5");
- upperBoundB.addString("c", "6");
- tableBuilder.addRangeBound(lowerBoundB, upperBoundB);
-
- tableBuilder.addHashPartitions(ImmutableList.of("a", "b", "c"), 4);
-
- testPartitionSchema(tableBuilder);
- }
-
- @Test(timeout = 100000)
- public void testSimplePartitionedTable() throws Exception {
- Schema schema = createSchema();
- CreateTableOptions tableBuilder =
- new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("a", "b", "c"));
-
- PartialRow split = schema.newPartialRow();
- split.addString("c", "3");
- tableBuilder.addSplitRow(split);
-
- split = schema.newPartialRow();
- split.addString("c", "3");
- split.addString("b", "3");
- tableBuilder.addSplitRow(split);
-
- testPartitionSchema(tableBuilder);
- }
-
- @Test(timeout = 100000)
- public void testUnpartitionedTable() throws Exception {
- CreateTableOptions tableBuilder =
- new CreateTableOptions().setRangePartitionColumns(ImmutableList.<String>of());
- testPartitionSchema(tableBuilder);
- }
-
- public static class Row implements Comparable<Row> {
- private final String a;
- private final String b;
- private final String c;
-
- public Row(String a, String b, String c) {
- this.a = a;
- this.b = b;
- this.c = c;
- }
-
- public String getA() {
- return a;
- }
-
- public String getB() {
- return b;
- }
-
- public String getC() {
- return c;
- }
-
- public void fillPartialRow(PartialRow row) {
- row.addString("a", a);
- row.addString("b", b);
- row.addString("c", c);
- }
-
- private static Row fromResult(RowResult result) {
- return new Row(result.getString("a"),
- result.getString("b"),
- result.getString("c"));
- }
-
- public Predicate<Row> gtePred() {
- return new Predicate<Row>() {
- @Override
- public boolean apply(Row other) {
- return other.compareTo(Row.this) >= 0;
- }
- };
- }
-
- public Predicate<Row> ltPred() {
- return new Predicate<Row>() {
- @Override
- public boolean apply(Row other) {
- return other.compareTo(Row.this) < 0;
- }
- };
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- Row row = (Row) o;
- return Objects.equals(a, row.a)
- && Objects.equals(b, row.b)
- && Objects.equals(c, row.c);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(a, b, c);
- }
-
- @Override
- public int compareTo(Row other) {
- return ComparisonChain.start()
- .compare(a, other.a)
- .compare(b, other.b)
- .compare(c, other.c)
- .result();
- }
-
- @Override
- public String toString() {
- return com.google.common.base.Objects.toStringHelper(this)
- .add("a", a)
- .add("b", b)
- .add("c", c)
- .toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestHybridTime.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestHybridTime.java b/java/kudu-client/src/test/java/org/kududb/client/TestHybridTime.java
deleted file mode 100644
index 666ac38..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestHybridTime.java
+++ /dev/null
@@ -1,163 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import com.stumbleupon.async.Deferred;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.kududb.Type.STRING;
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
-import static org.kududb.util.HybridTimeUtil.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * This only tests client propagation since it's the only thing that is client-specific.
- * All the work for commit wait is done and tested on the server-side.
- */
-public class TestHybridTime extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(TestHybridTime.class);
-
- // Generate a unique table name
- protected static final String TABLE_NAME =
- TestHybridTime.class.getName() + "-" + System.currentTimeMillis();
-
- protected static Schema schema = getSchema();
- protected static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
-
- // Using multiple tablets doesn't work with the current way this test works since we could
- // jump from one TS to another which changes the logical clock.
- CreateTableOptions builder =
- new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
- table = createTable(TABLE_NAME, schema, builder);
- }
-
- private static Schema getSchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(1);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key", STRING)
- .key(true)
- .build());
- return new Schema(columns);
- }
-
- /**
- * We write three rows. We increment the timestamp we get back from the first write
- * by some amount. The remaining writes should force an update to the server's clock and
- * only increment the logical values.
- *
- * @throws Exception
- */
- @Test(timeout = 100000)
- public void test() throws Exception {
- AsyncKuduSession session = client.newSession();
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
- session.setExternalConsistencyMode(CLIENT_PROPAGATED);
- long[] clockValues;
- long previousLogicalValue = 0;
- long previousPhysicalValue = 0;
-
- // Test timestamp propagation with single operations
- String[] keys = new String[] {"1", "2", "3"};
- for (int i = 0; i < keys.length; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addString(schema.getColumnByIndex(0).getName(), keys[i]);
- Deferred<OperationResponse> d = session.apply(insert);
- OperationResponse response = d.join(DEFAULT_SLEEP);
- assertTrue(response.getWriteTimestamp() != 0);
- clockValues = HTTimestampToPhysicalAndLogical(response.getWriteTimestamp());
- LOG.debug("Clock value after write[" + i + "]: " + new Date(clockValues[0] / 1000).toString()
- + " Logical value: " + clockValues[1]);
- // on the very first write we update the clock into the future
- // so that remaining writes only update logical values
- if (i == 0) {
- assertEquals(clockValues[1], 0);
- long toUpdateTs = clockValues[0] + 5000000;
- previousPhysicalValue = toUpdateTs;
- // After the first write we fake-update the clock into the future. Following writes
- // should force the servers to update their clocks to this value.
- client.updateLastPropagatedTimestamp(
- clockTimestampToHTTimestamp(toUpdateTs, TimeUnit.MICROSECONDS));
- } else {
- assertEquals(clockValues[0], previousPhysicalValue);
- assertTrue(clockValues[1] > previousLogicalValue);
- previousLogicalValue = clockValues[1];
- }
- }
-
- // Test timestamp propagation with Batches
- session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
- keys = new String[] {"11", "22", "33"};
- for (int i = 0; i < keys.length; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addString(schema.getColumnByIndex(0).getName(), keys[i]);
- session.apply(insert);
- Deferred<List<OperationResponse>> d = session.flush();
- List<OperationResponse> responses = d.join(DEFAULT_SLEEP);
- assertEquals("Response was not of the expected size: " + responses.size(),
- 1, responses.size());
-
- OperationResponse response = responses.get(0);
- assertTrue(response.getWriteTimestamp() != 0);
- clockValues = HTTimestampToPhysicalAndLogical(response.getWriteTimestamp());
- LOG.debug("Clock value after write[" + i + "]: " + new Date(clockValues[0] / 1000).toString()
- + " Logical value: " + clockValues[1]);
- assertEquals(clockValues[0], previousPhysicalValue);
- assertTrue(clockValues[1] > previousLogicalValue);
- previousLogicalValue = clockValues[1];
- }
-
- // Scan all rows with READ_LATEST (the default) we should get 6 rows back
- assertEquals(6, countRowsInScan(client.newScannerBuilder(table).build()));
-
- // Now scan at multiple instances with READ_AT_SNAPSHOT we should get different
- // counts depending on the scan timestamp.
- long snapTime = physicalAndLogicalToHTTimestamp(previousPhysicalValue, 0);
- assertEquals(1, scanAtSnapshot(snapTime));
- snapTime = physicalAndLogicalToHTTimestamp(previousPhysicalValue, 5);
- assertEquals(4, scanAtSnapshot(snapTime));
- // Our last snap time needs to one one into the future w.r.t. the last write's timestamp
- // for us to be able to get all rows, but the snap timestamp can't be bigger than the prop.
- // timestamp so we increase both.
- client.updateLastPropagatedTimestamp(client.getLastPropagatedTimestamp() + 1);
- snapTime = physicalAndLogicalToHTTimestamp(previousPhysicalValue, previousLogicalValue + 1);
- assertEquals(6, scanAtSnapshot(snapTime));
- }
-
- private int scanAtSnapshot(long time) throws Exception {
- AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table)
- .snapshotTimestampRaw(time)
- .readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT);
- return countRowsInScan(builder.build());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestKeyEncoding.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestKeyEncoding.java b/java/kudu-client/src/test/java/org/kududb/client/TestKeyEncoding.java
deleted file mode 100644
index e446445..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestKeyEncoding.java
+++ /dev/null
@@ -1,272 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.ImmutableList;
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.ColumnSchema.ColumnSchemaBuilder;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.PartitionSchema.HashBucketSchema;
-import org.kududb.client.PartitionSchema.RangeSchema;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class TestKeyEncoding {
-
- private static Schema buildSchema(ColumnSchemaBuilder... columns) {
- int i = 0;
- Common.SchemaPB.Builder pb = Common.SchemaPB.newBuilder();
- for (ColumnSchemaBuilder column : columns) {
- Common.ColumnSchemaPB.Builder columnPb =
- ProtobufHelper.columnToPb(column.build()).toBuilder();
- columnPb.setId(i++);
- pb.addColumns(columnPb);
- }
- return ProtobufHelper.pbToSchema(pb.build());
- }
-
- private static void assertBytesEquals(byte[] actual, byte[] expected) {
- assertTrue(String.format("expected: '%s', got '%s'",
- Bytes.pretty(expected),
- Bytes.pretty(actual)),
- Bytes.equals(expected, actual));
- }
-
- private static void assertBytesEquals(byte[] actual, String expected) {
- assertBytesEquals(actual, expected.getBytes(Charsets.UTF_8));
- }
-
- /**
- * Builds the default partition schema for a schema.
- * @param schema the schema
- * @return a default partition schema
- */
- private PartitionSchema defaultPartitionSchema(Schema schema) {
- List<Integer> columnIds = new ArrayList<>();
- for (int i = 0; i < schema.getPrimaryKeyColumnCount(); i++) {
- // Schema does not provide a way to lookup a column ID by column index,
- // so instead we assume that the IDs for the primary key columns match
- // their respective index, which holds up when the schema is created
- // with buildSchema.
- columnIds.add(i);
- }
- return new PartitionSchema(
- new PartitionSchema.RangeSchema(columnIds),
- ImmutableList.<PartitionSchema.HashBucketSchema>of(), schema);
- }
-
- @Test
- public void testPrimaryKeys() {
- Schema schemaOneString =
- buildSchema(new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true));
- KuduTable table = new KuduTable(null, "one", "one", schemaOneString,
- defaultPartitionSchema(schemaOneString));
- Insert oneKeyInsert = new Insert(table);
- PartialRow row = oneKeyInsert.getRow();
- row.addString("key", "foo");
- assertBytesEquals(row.encodePrimaryKey(), "foo");
-
- Schema schemaTwoString = buildSchema(
- new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true),
- new ColumnSchema.ColumnSchemaBuilder("key2", Type.STRING).key(true));
- KuduTable table2 = new KuduTable(null, "two", "two", schemaTwoString,
- defaultPartitionSchema(schemaTwoString));
- Insert twoKeyInsert = new Insert(table2);
- row = twoKeyInsert.getRow();
- row.addString("key", "foo");
- row.addString("key2", "bar");
- assertBytesEquals(row.encodePrimaryKey(), "foo\0\0bar");
-
- Insert twoKeyInsertWithNull = new Insert(table2);
- row = twoKeyInsertWithNull.getRow();
- row.addString("key", "xxx\0yyy");
- row.addString("key2", "bar");
- assertBytesEquals(row.encodePrimaryKey(), "xxx\0\1yyy\0\0bar");
-
- // test that we get the correct memcmp result, the bytes are in big-endian order in a key
- Schema schemaIntString = buildSchema(
- new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true),
- new ColumnSchema.ColumnSchemaBuilder("key2", Type.STRING).key(true));
- PartitionSchema partitionSchemaIntString = defaultPartitionSchema(schemaIntString);
- KuduTable table3 = new KuduTable(null, "three", "three",
- schemaIntString, partitionSchemaIntString);
- Insert small = new Insert(table3);
- row = small.getRow();
- row.addInt("key", 20);
- row.addString("key2", "data");
- byte[] smallPK = small.getRow().encodePrimaryKey();
- assertEquals(0, Bytes.memcmp(smallPK, smallPK));
-
- Insert big = new Insert(table3);
- row = big.getRow();
- row.addInt("key", 10000);
- row.addString("key2", "data");
- byte[] bigPK = big.getRow().encodePrimaryKey();
- assertTrue(Bytes.memcmp(smallPK, bigPK) < 0);
- assertTrue(Bytes.memcmp(bigPK, smallPK) > 0);
-
- // The following tests test our assumptions on unsigned data types sorting from KeyEncoder
- byte four = 4;
- byte onHundredTwentyFour = -4;
- four = Bytes.xorLeftMostBit(four);
- onHundredTwentyFour = Bytes.xorLeftMostBit(onHundredTwentyFour);
- assertTrue(four < onHundredTwentyFour);
-
- byte[] threeHundred = Bytes.fromInt(300);
- byte[] reallyBigNumber = Bytes.fromInt(-300);
- threeHundred[0] = Bytes.xorLeftMostBit(threeHundred[0]);
- reallyBigNumber[3] = Bytes.xorLeftMostBit(reallyBigNumber[3]);
- assertTrue(Bytes.memcmp(threeHundred, reallyBigNumber) < 0);
- }
-
- @Test
- public void testPrimaryKeyEncoding() {
- Schema schema = buildSchema(
- new ColumnSchemaBuilder("int8", Type.INT8).key(true),
- new ColumnSchemaBuilder("int16", Type.INT16).key(true),
- new ColumnSchemaBuilder("int32", Type.INT32).key(true),
- new ColumnSchemaBuilder("int64", Type.INT64).key(true),
- new ColumnSchemaBuilder("string", Type.STRING).key(true),
- new ColumnSchemaBuilder("binary", Type.BINARY).key(true));
-
- PartialRow rowA = schema.newPartialRow();
- rowA.addByte("int8", Byte.MIN_VALUE);
- rowA.addShort("int16", Short.MIN_VALUE);
- rowA.addInt("int32", Integer.MIN_VALUE);
- rowA.addLong("int64", Long.MIN_VALUE);
- rowA.addString("string", "");
- rowA.addBinary("binary", "".getBytes(Charsets.UTF_8));
-
- assertBytesEquals(rowA.encodePrimaryKey(),
- "\0"
- + "\0\0"
- + "\0\0\0\0"
- + "\0\0\0\0\0\0\0\0"
- + "\0\0"
- + "");
-
- PartialRow rowB = schema.newPartialRow();
- rowB.addByte("int8", Byte.MAX_VALUE);
- rowB.addShort("int16", Short.MAX_VALUE);
- rowB.addInt("int32", Integer.MAX_VALUE);
- rowB.addLong("int64", Long.MAX_VALUE);
- rowB.addString("string", "abc\1\0def");
- rowB.addBinary("binary", "\0\1binary".getBytes(Charsets.UTF_8));
-
- assertBytesEquals(rowB.encodePrimaryKey(),
- new byte[] {
- -1,
- -1, -1,
- -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1,
- 'a', 'b', 'c', 1, 0, 1, 'd', 'e', 'f', 0, 0,
- 0, 1, 'b', 'i', 'n', 'a', 'r', 'y',
- });
-
- PartialRow rowC = schema.newPartialRow();
- rowC.addByte("int8", (byte) 1);
- rowC.addShort("int16", (short) 2);
- rowC.addInt("int32", 3);
- rowC.addLong("int64", 4);
- rowC.addString("string", "abc\n123");
- rowC.addBinary("binary", "\0\1\2\3\4\5".getBytes(Charsets.UTF_8));
-
- assertBytesEquals(rowC.encodePrimaryKey(),
- new byte[] {
- (byte) 0x81,
- (byte) 0x80, 2,
- (byte) 0x80, 0, 0, 3,
- (byte) 0x80, 0, 0, 0, 0, 0, 0, 4,
- 'a', 'b', 'c', '\n', '1', '2', '3', 0, 0,
- 0, 1, 2, 3, 4, 5,
- });
- }
-
- @Test
- public void testPartitionKeyEncoding() {
- KeyEncoder encoder = new KeyEncoder();
- Schema schema = buildSchema(
- new ColumnSchemaBuilder("a", Type.INT32).key(true),
- new ColumnSchemaBuilder("b", Type.STRING).key(true),
- new ColumnSchemaBuilder("c", Type.STRING).key(true));
-
- PartitionSchema partitionSchema =
- new PartitionSchema(new RangeSchema(ImmutableList.of(0, 1, 2)),
- ImmutableList.of(
- new HashBucketSchema(ImmutableList.of(0, 1), 32, 0),
- new HashBucketSchema(ImmutableList.of(2), 32, 42)),
- schema);
-
- PartialRow rowA = schema.newPartialRow();
- rowA.addInt("a", 0);
- rowA.addString("b", "");
- rowA.addString("c", "");
- assertBytesEquals(encoder.encodePartitionKey(rowA, partitionSchema),
- new byte[]{
- 0, 0, 0, 0, // hash(0, "")
- 0, 0, 0, 0x14, // hash("")
- (byte) 0x80, 0, 0, 0, // a = 0
- 0, 0, // b = ""; c is elided
- });
-
- PartialRow rowB = schema.newPartialRow();
- rowB.addInt("a", 1);
- rowB.addString("b", "");
- rowB.addString("c", "");
- assertBytesEquals(encoder.encodePartitionKey(rowB, partitionSchema),
- new byte[]{
- 0, 0, 0, 0x5, // hash(1, "")
- 0, 0, 0, 0x14, // hash("")
- (byte) 0x80, 0, 0, 1, // a = 0
- 0, 0, // b = ""; c is elided
- });
-
- PartialRow rowC = schema.newPartialRow();
- rowC.addInt("a", 0);
- rowC.addString("b", "b");
- rowC.addString("c", "c");
- assertBytesEquals(encoder.encodePartitionKey(rowC, partitionSchema),
- new byte[]{
- 0, 0, 0, 0x1A, // hash(0, "b")
- 0, 0, 0, 0x1D, // hash("c")
- (byte) 0x80, 0, 0, 0, // a = 1
- 'b', 0, 0, // b = "b"
- 'c' // b = "c"
- });
-
- PartialRow rowD = schema.newPartialRow();
- rowD.addInt("a", 1);
- rowD.addString("b", "b");
- rowD.addString("c", "c");
- assertBytesEquals(encoder.encodePartitionKey(rowD, partitionSchema),
- new byte[]{
- 0, 0, 0, 0, // hash(1, "b")
- 0, 0, 0, 0x1D, // hash("c")
- (byte) 0x80, 0, 0, 1, // a = 0
- 'b', 0, 0, // b = "b"
- 'c' // b = "c"
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/kududb/client/TestKuduClient.java
deleted file mode 100644
index 3591b7b..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestKuduClient.java
+++ /dev/null
@@ -1,535 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
-import static org.kududb.client.RowResult.timestampToString;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestKuduClient extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(TestKuduClient.class);
- private String tableName;
-
- @Before
- public void setTableName() {
- tableName = TestKuduClient.class.getName() + "-" + System.currentTimeMillis();
- }
-
- private Schema createManyStringsSchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.STRING).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.STRING).nullable(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.STRING).nullable(true).build());
- return new Schema(columns);
- }
-
- private Schema createSchemaWithBinaryColumns() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>();
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.BINARY).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.DOUBLE).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.BINARY).nullable(true).build());
- return new Schema(columns);
- }
-
- private Schema createSchemaWithTimestampColumns() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>();
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.TIMESTAMP).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.TIMESTAMP).nullable(true).build());
- return new Schema(columns);
- }
-
- private static CreateTableOptions createTableOptions() {
- return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
- }
-
- /**
- * Test creating and deleting a table through a KuduClient.
- */
- @Test(timeout = 100000)
- public void testCreateDeleteTable() throws Exception {
- // Check that we can create a table.
- syncClient.createTable(tableName, basicSchema, getBasicCreateTableOptions());
- assertFalse(syncClient.getTablesList().getTablesList().isEmpty());
- assertTrue(syncClient.getTablesList().getTablesList().contains(tableName));
-
- // Check that we can delete it.
- syncClient.deleteTable(tableName);
- assertFalse(syncClient.getTablesList().getTablesList().contains(tableName));
-
- // Check that we can re-recreate it, with a different schema.
- List<ColumnSchema> columns = new ArrayList<>(basicSchema.getColumns());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("one more", Type.STRING).build());
- Schema newSchema = new Schema(columns);
- syncClient.createTable(tableName, newSchema, getBasicCreateTableOptions());
-
- // Check that we can open a table and see that it has the new schema.
- KuduTable table = syncClient.openTable(tableName);
- assertEquals(newSchema.getColumnCount(), table.getSchema().getColumnCount());
- assertTrue(table.getPartitionSchema().isSimpleRangePartitioning());
-
- // Check that the block size parameter we specified in the schema is respected.
- assertEquals(4096, newSchema.getColumn("column3_s").getDesiredBlockSize());
- assertEquals(ColumnSchema.Encoding.DICT_ENCODING,
- newSchema.getColumn("column3_s").getEncoding());
- assertEquals(ColumnSchema.CompressionAlgorithm.LZ4,
- newSchema.getColumn("column3_s").getCompressionAlgorithm());
- }
-
- /**
- * Test inserting and retrieving string columns.
- */
- @Test(timeout = 100000)
- public void testStrings() throws Exception {
- Schema schema = createManyStringsSchema();
- syncClient.createTable(tableName, schema, createTableOptions());
-
- KuduSession session = syncClient.newSession();
- KuduTable table = syncClient.openTable(tableName);
- for (int i = 0; i < 100; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addString("key", String.format("key_%02d", i));
- row.addString("c2", "c2_" + i);
- if (i % 2 == 1) {
- row.addString("c3", "c3_" + i);
- }
- row.addString("c4", "c4_" + i);
- // NOTE: we purposefully add the strings in a non-left-to-right
- // order to verify that we still place them in the right position in
- // the row.
- row.addString("c1", "c1_" + i);
- session.apply(insert);
- if (i % 50 == 0) {
- session.flush();
- }
- }
- session.flush();
-
- List<String> rowStrings = scanTableToStrings(table);
- assertEquals(100, rowStrings.size());
- assertEquals(
- "STRING key=key_03, STRING c1=c1_3, STRING c2=c2_3, STRING c3=c3_3, STRING c4=c4_3",
- rowStrings.get(3));
- assertEquals(
- "STRING key=key_04, STRING c1=c1_4, STRING c2=c2_4, STRING c3=NULL, STRING c4=c4_4",
- rowStrings.get(4));
-
- KuduScanner scanner = syncClient.newScannerBuilder(table).build();
-
- assertTrue("Scanner should have returned row", scanner.hasMoreRows());
-
- RowResultIterator rows = scanner.nextRows();
- final RowResult next = rows.next();
-
- // Do negative testing on string type.
- try {
- next.getInt("c2");
- fail("IllegalArgumentException was not thrown when accessing " +
- "a string column with getInt");
- } catch (IllegalArgumentException ignored) {}
- }
-
- /**
- * Test to verify that we can write in and read back UTF8.
- */
- @Test(timeout = 100000)
- public void testUTF8() throws Exception {
- Schema schema = createManyStringsSchema();
- syncClient.createTable(tableName, schema, createTableOptions());
-
- KuduSession session = syncClient.newSession();
- KuduTable table = syncClient.openTable(tableName);
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addString("key", "\u0e01\u0e02\u0e03\u0e04\u0e05\u0e06\u0e07"); // some thai
- row.addString("c1", "\u2701\u2702\u2703\u2704\u2706"); // some icons
-
- row.addString("c2", "hello"); // some normal chars
- row.addString("c4", "\U0001f431"); // supplemental plane
- session.apply(insert);
- session.flush();
-
- List<String> rowStrings = scanTableToStrings(table);
- assertEquals(1, rowStrings.size());
- assertEquals(
- "STRING key=\u0e01\u0e02\u0e03\u0e04\u0e05\u0e06\u0e07, STRING c1=\u2701\u2702\u2703\u2704\u2706, STRING c2=hello, STRING c3=NULL, STRING c4=\U0001f431",
- rowStrings.get(0));
- }
-
- /**
- * Test inserting and retrieving binary columns.
- */
- @Test(timeout = 100000)
- public void testBinaryColumns() throws Exception {
- Schema schema = createSchemaWithBinaryColumns();
- syncClient.createTable(tableName, schema, createTableOptions());
-
- byte[] testArray = new byte[] {1, 2, 3, 4, 5, 6 ,7, 8, 9};
-
- KuduSession session = syncClient.newSession();
- KuduTable table = syncClient.openTable(tableName);
- for (int i = 0; i < 100; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addBinary("key", String.format("key_%02d", i).getBytes());
- row.addString("c1", "\u2701\u2702\u2703\u2704\u2706");
- row.addDouble("c2", i);
- if (i % 2 == 1) {
- row.addBinary("c3", testArray);
- }
- session.apply(insert);
- if (i % 50 == 0) {
- session.flush();
- }
- }
- session.flush();
-
- List<String> rowStrings = scanTableToStrings(table);
- assertEquals(100, rowStrings.size());
- for (int i = 0; i < rowStrings.size(); i++) {
- StringBuilder expectedRow = new StringBuilder();
- expectedRow.append(String.format("BINARY key=\"key_%02d\", STRING c1=\u2701\u2702\u2703\u2704\u2706, DOUBLE c2=%.1f,"
- + " BINARY c3=", i, (double) i));
- if (i % 2 == 1) {
- expectedRow.append(Bytes.pretty(testArray));
- } else {
- expectedRow.append("NULL");
- }
- assertEquals(expectedRow.toString(), rowStrings.get(i));
- }
- }
-
- /**
- * Test inserting and retrieving timestamp columns.
- */
- @Test(timeout = 100000)
- public void testTimestampColumns() throws Exception {
- Schema schema = createSchemaWithTimestampColumns();
- syncClient.createTable(tableName, schema, createTableOptions());
-
- List<Long> timestamps = new ArrayList<>();
-
- KuduSession session = syncClient.newSession();
- KuduTable table = syncClient.openTable(tableName);
- long lastTimestamp = 0;
- for (int i = 0; i < 100; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- long timestamp = System.currentTimeMillis() * 1000;
- while(timestamp == lastTimestamp) {
- timestamp = System.currentTimeMillis() * 1000;
- }
- timestamps.add(timestamp);
- row.addLong("key", timestamp);
- if (i % 2 == 1) {
- row.addLong("c1", timestamp);
- }
- session.apply(insert);
- if (i % 50 == 0) {
- session.flush();
- }
- lastTimestamp = timestamp;
- }
- session.flush();
-
- List<String> rowStrings = scanTableToStrings(table);
- assertEquals(100, rowStrings.size());
- for (int i = 0; i < rowStrings.size(); i++) {
- StringBuilder expectedRow = new StringBuilder();
- expectedRow.append(String.format("TIMESTAMP key=%s, TIMESTAMP c1=",
- timestampToString(timestamps.get(i))));
- if (i % 2 == 1) {
- expectedRow.append(timestampToString(timestamps.get(i)));
- } else {
- expectedRow.append("NULL");
- }
- assertEquals(expectedRow.toString(), rowStrings.get(i));
- }
- }
-
- /**
- * Test scanning with predicates.
- */
- @Test
- public void testScanWithPredicates() throws Exception {
- Schema schema = createManyStringsSchema();
- syncClient.createTable(tableName, schema, createTableOptions());
-
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
- KuduTable table = syncClient.openTable(tableName);
- for (int i = 0; i < 100; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addString("key", String.format("key_%02d", i));
- row.addString("c1", "c1_" + i);
- row.addString("c2", "c2_" + i);
- session.apply(insert);
- }
- session.flush();
-
- assertEquals(100, scanTableToStrings(table).size());
- assertEquals(50, scanTableToStrings(table,
- KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER_EQUAL, "key_50")
- ).size());
- assertEquals(25, scanTableToStrings(table,
- KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER, "key_74")
- ).size());
- assertEquals(25, scanTableToStrings(table,
- KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER, "key_24"),
- KuduPredicate.newComparisonPredicate(schema.getColumn("c1"), LESS_EQUAL, "c1_49")
- ).size());
- assertEquals(50, scanTableToStrings(table,
- KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER, "key_24"),
- KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER_EQUAL, "key_50")
- ).size());
- assertEquals(0, scanTableToStrings(table,
- KuduPredicate.newComparisonPredicate(schema.getColumn("c1"), GREATER, "c1_30"),
- KuduPredicate.newComparisonPredicate(schema.getColumn("c2"), LESS, "c2_20")
- ).size());
- assertEquals(0, scanTableToStrings(table,
- // Short circuit scan
- KuduPredicate.newComparisonPredicate(schema.getColumn("c2"), GREATER, "c2_30"),
- KuduPredicate.newComparisonPredicate(schema.getColumn("c2"), LESS, "c2_20")
- ).size());
- }
-
- /**
- * Tests scan tokens by creating a set of scan tokens, serializing them, and
- * then executing them in parallel with separate client instances. This
- * simulates the normal usecase of scan tokens being created at a central
- * planner and distributed to remote task executors.
- */
- @Test
- public void testScanTokens() throws Exception {
- Schema schema = createManyStringsSchema();
- CreateTableOptions createOptions = new CreateTableOptions();
- createOptions.addHashPartitions(ImmutableList.of("key"), 8);
-
- PartialRow splitRow = schema.newPartialRow();
- splitRow.addString("key", "key_50");
- createOptions.addSplitRow(splitRow);
-
- syncClient.createTable(tableName, schema, createOptions);
-
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
- KuduTable table = syncClient.openTable(tableName);
- for (int i = 0; i < 100; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addString("key", String.format("key_%02d", i));
- row.addString("c1", "c1_" + i);
- row.addString("c2", "c2_" + i);
- session.apply(insert);
- }
- session.flush();
-
- KuduScanToken.KuduScanTokenBuilder tokenBuilder = syncClient.newScanTokenBuilder(table);
- tokenBuilder.setProjectedColumnIndexes(ImmutableList.<Integer>of());
- List<KuduScanToken> tokens = tokenBuilder.build();
- assertEquals(16, tokens.size());
-
- final AtomicInteger count = new AtomicInteger(0);
- List<Thread> threads = new ArrayList<>();
- for (final KuduScanToken token : tokens) {
- final byte[] serializedToken = token.serialize();
- Thread thread = new Thread(new Runnable() {
- @Override
- public void run() {
- try (KuduClient contextClient = new KuduClient.KuduClientBuilder(masterAddresses)
- .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
- .build()) {
- KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, contextClient);
- try {
- int localCount = 0;
- while (scanner.hasMoreRows()) {
- localCount += Iterators.size(scanner.nextRows());
- }
- assertTrue(localCount > 0);
- count.addAndGet(localCount);
- } finally {
- scanner.close();
- }
- } catch (Exception e) {
- LOG.error("exception in parallel token scanner", e);
- }
- }
- });
- thread.run();
- threads.add(thread);
- }
-
- for (Thread thread : threads) {
- thread.join();
- }
- assertEquals(100, count.get());
- }
-
- /**
- * Counts the rows in a table between two optional bounds.
- * @param table the table to scan, must have the basic schema
- * @param lowerBound an optional lower bound key
- * @param upperBound an optional upper bound key
- * @return the row count
- * @throws Exception on error
- */
- private int countRowsForTestScanNonCoveredTable(KuduTable table,
- Integer lowerBound,
- Integer upperBound) throws Exception {
-
- KuduScanner.KuduScannerBuilder scanBuilder = syncClient.newScannerBuilder(table);
- if (lowerBound != null) {
- PartialRow bound = basicSchema.newPartialRow();
- bound.addInt(0, lowerBound);
- scanBuilder.lowerBound(bound);
- }
- if (upperBound != null) {
- PartialRow bound = basicSchema.newPartialRow();
- bound.addInt(0, upperBound);
- scanBuilder.exclusiveUpperBound(bound);
- }
-
- KuduScanner scanner = scanBuilder.build();
- int count = 0;
- while (scanner.hasMoreRows()) {
- count += scanner.nextRows().getNumRows();
- }
- return count;
- }
-
- /**
- * Tests scanning a table with non-covering range partitions.
- */
- @Test(timeout = 100000)
- public void testScanNonCoveredTable() throws Exception {
-
- Schema schema = basicSchema;
- syncClient.createTable(tableName, schema, getBasicTableOptionsWithNonCoveredRange());
-
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
- KuduTable table = syncClient.openTable(tableName);
-
- for (int key = 0; key < 100; key++) {
- session.apply(createBasicSchemaInsert(table, key));
- }
- for (int key = 200; key < 300; key++) {
- session.apply(createBasicSchemaInsert(table, key));
- }
- session.flush();
- assertEquals(0, session.countPendingErrors());
-
- assertEquals(200, countRowsForTestScanNonCoveredTable(table, null, null));
- assertEquals(100, countRowsForTestScanNonCoveredTable(table, null, 200));
- assertEquals(0, countRowsForTestScanNonCoveredTable(table, null, -1));
- assertEquals(0, countRowsForTestScanNonCoveredTable(table, 120, 180));
- assertEquals(0, countRowsForTestScanNonCoveredTable(table, 300, null));
- }
-
- /**
- * Creates a local client that we auto-close while buffering one row, then makes sure that after
- * closing that we can read the row.
- */
- @Test(timeout = 100000)
- public void testAutoClose() throws Exception {
- try (KuduClient localClient = new KuduClient.KuduClientBuilder(masterAddresses).build()) {
- localClient.createTable(tableName, basicSchema, getBasicCreateTableOptions());
- KuduTable table = localClient.openTable(tableName);
- KuduSession session = localClient.newSession();
-
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- Insert insert = createBasicSchemaInsert(table, 0);
- session.apply(insert);
- }
-
- KuduTable table = syncClient.openTable(tableName);
- AsyncKuduScanner scanner = new AsyncKuduScanner.AsyncKuduScannerBuilder(client, table).build();
- assertEquals(1, countRowsInScan(scanner));
- }
-
- @Test(timeout = 100000)
- public void testCustomNioExecutor() throws Exception {
- long startTime = System.nanoTime();
- final KuduClient localClient = new KuduClient.KuduClientBuilder(masterAddresses)
- .nioExecutors(Executors.newFixedThreadPool(1), Executors.newFixedThreadPool(2))
- .bossCount(1)
- .workerCount(2)
- .build();
- long buildTime = (System.nanoTime() - startTime) / 1000000000L;
- assertTrue("Building KuduClient is slow, maybe netty get stuck", buildTime < 3);
- localClient.createTable(tableName, basicSchema, getBasicCreateTableOptions());
- Thread[] threads = new Thread[4];
- for (int t = 0; t < 4; t++) {
- final int id = t;
- threads[t] = new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- KuduTable table = localClient.openTable(tableName);
- KuduSession session = localClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
- for (int i = 0; i < 100; i++) {
- Insert insert = createBasicSchemaInsert(table, id * 100 + i);
- session.apply(insert);
- }
- session.close();
- } catch (Exception e) {
- fail("insert thread should not throw exception: " + e);
- }
- }
- });
- threads[t].start();
- }
- for (int t = 0; t< 4;t++) {
- threads[t].join();
- }
- localClient.shutdown();
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testNoDefaultPartitioning() throws Exception {
- syncClient.createTable(tableName, basicSchema, new CreateTableOptions());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestKuduPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestKuduPredicate.java b/java/kudu-client/src/test/java/org/kududb/client/TestKuduPredicate.java
deleted file mode 100644
index 4915a18..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestKuduPredicate.java
+++ /dev/null
@@ -1,628 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.kududb.client;
-
-import com.google.common.base.Preconditions;
-import org.junit.Assert;
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-
-import static org.kududb.client.KuduPredicate.ComparisonOp.EQUAL;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER;
-import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS;
-import static org.kududb.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
-import static org.kududb.client.KuduPredicate.PredicateType.RANGE;
-
-public class TestKuduPredicate {
-
- private static final ColumnSchema boolCol =
- new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build();
-
- private static final ColumnSchema byteCol =
- new ColumnSchema.ColumnSchemaBuilder("byte", Type.INT8).build();
-
- private static final ColumnSchema shortCol =
- new ColumnSchema.ColumnSchemaBuilder("short", Type.INT16).build();
-
- private static final ColumnSchema intCol =
- new ColumnSchema.ColumnSchemaBuilder("int", Type.INT32).build();
-
- private static final ColumnSchema longCol =
- new ColumnSchema.ColumnSchemaBuilder("long", Type.INT64).build();
-
- private static final ColumnSchema floatCol =
- new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build();
-
- private static final ColumnSchema doubleCol =
- new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build();
-
- private static final ColumnSchema stringCol =
- new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).build();
-
- private static final ColumnSchema binaryCol =
- new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).build();
-
- private static KuduPredicate intRange(int lower, int upper) {
- Preconditions.checkArgument(lower < upper);
- return new KuduPredicate(RANGE, intCol, Bytes.fromInt(lower), Bytes.fromInt(upper));
- }
-
- private void testMerge(KuduPredicate a,
- KuduPredicate b,
- KuduPredicate expected) {
-
- Assert.assertEquals(expected, a.merge(b));
- Assert.assertEquals(expected, b.merge(a));
- }
-
- /**
- * Tests merges on all types of integer predicates.
- */
- @Test
- public void testMergeInt() {
-
- // Equality + Equality
-
- // |
- // |
- // =
- // |
- testMerge(KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0));
- // |
- // |
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 1),
- KuduPredicate.none(intCol));
-
- // Range + Equality
-
- // [-------->
- // |
- // =
- // |
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 10),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 10));
-
- // [-------->
- // |
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 10),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
- KuduPredicate.none(intCol));
-
- // <--------)
- // |
- // =
- // |
- testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5));
-
- // <--------)
- // |
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 10),
- KuduPredicate.none(intCol));
-
- // Unbounded Range + Unbounded Range
-
- // [--------> AND
- // [-------->
- // =
- // [-------->
-
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0));
-
- // [--------> AND
- // [----->
- // =
- // [----->
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
- KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5));
-
- // <--------) AND
- // <--------)
- // =
- // <--------)
-
- testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 0));
-
- // <--------) AND
- // <----)
- // =
- // <----)
-
- testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
- KuduPredicate.newComparisonPredicate(intCol, LESS, -10),
- KuduPredicate.newComparisonPredicate(intCol, LESS, -10));
-
- // [--------> AND
- // <-------)
- // =
- // [----)
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
- intRange(0, 10));
-
- // [-----> AND
- // <----)
- // =
- // |
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 6),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5));
-
- // [-----> AND
- // <---)
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 5),
- KuduPredicate.none(intCol));
-
- // [-----> AND
- // <---)
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 3),
- KuduPredicate.none(intCol));
-
- // Range + Range
-
- // [--------) AND
- // [--------)
- // =
- // [--------)
-
- testMerge(intRange(0, 10),
- intRange(0, 10),
- intRange(0, 10));
-
- // [--------) AND
- // [----)
- // =
- // [----)
- testMerge(intRange(0, 10),
- intRange(0, 5),
- intRange(0, 5));
-
- // [--------) AND
- // [----)
- // =
- // [----)
- testMerge(intRange(0, 10),
- intRange(3, 8),
- intRange(3, 8));
-
- // [-----) AND
- // [------)
- // =
- // [---)
- testMerge(intRange(0, 8),
- intRange(3, 10),
- intRange(3, 8));
- // [--) AND
- // [---)
- // =
- // None
- testMerge(intRange(0, 5),
- intRange(5, 10),
- KuduPredicate.none(intCol));
-
- // [--) AND
- // [---)
- // =
- // None
- testMerge(intRange(0, 3),
- intRange(5, 10),
- KuduPredicate.none(intCol));
-
- // Lower Bound + Range
-
- // [------------>
- // [---)
- // =
- // [---)
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
- intRange(5, 10),
- intRange(5, 10));
-
- // [------------>
- // [--------)
- // =
- // [--------)
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
- intRange(5, 10),
- intRange(5, 10));
-
- // [------------>
- // [--------)
- // =
- // [---)
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 5),
- intRange(0, 10),
- intRange(5, 10));
-
- // [------->
- // [-----)
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 10),
- intRange(0, 5),
- KuduPredicate.none(intCol));
-
- // Upper Bound + Range
-
- // <------------)
- // [---)
- // =
- // [---)
- testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
- intRange(3, 8),
- intRange(3, 8));
-
- // <------------)
- // [--------)
- // =
- // [--------)
- testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 10),
- intRange(5, 10),
- intRange(5, 10));
-
-
- // <------------)
- // [--------)
- // =
- // [----)
- testMerge(KuduPredicate.newComparisonPredicate(intCol, LESS, 5),
- intRange(0, 10),
- intRange(0, 5));
-
- // Range + Equality
-
- // [---) AND
- // |
- // =
- // None
- testMerge(intRange(3, 5),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 1),
- KuduPredicate.none(intCol));
-
- // [---) AND
- // |
- // =
- // |
- testMerge(intRange(0, 5),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 0));
-
- // [---) AND
- // |
- // =
- // |
- testMerge(intRange(0, 5),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 3),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 3));
-
- // [---) AND
- // |
- // =
- // None
- testMerge(intRange(0, 5),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5),
- KuduPredicate.none(intCol));
-
- // [---) AND
- // |
- // =
- // None
- testMerge(intRange(0, 5),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 7),
- KuduPredicate.none(intCol));
-
- // None
-
- // None AND
- // [---->
- // =
- // None
- testMerge(KuduPredicate.none(intCol),
- KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 0),
- KuduPredicate.none(intCol));
- // None AND
- // <----)
- // =
- // None
- testMerge(KuduPredicate.none(intCol),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 0),
- KuduPredicate.none(intCol));
-
- // None AND
- // [----)
- // =
- // None
- testMerge(KuduPredicate.none(intCol),
- intRange(3, 7),
- KuduPredicate.none(intCol));
-
- // None AND
- // |
- // =
- // None
- testMerge(KuduPredicate.none(intCol),
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, 5),
- KuduPredicate.none(intCol));
-
- // None AND
- // None
- // =
- // None
- testMerge(KuduPredicate.none(intCol),
- KuduPredicate.none(intCol),
- KuduPredicate.none(intCol));
- }
-
- /**
- * Tests tricky merges on a var length type.
- */
- @Test
- public void testMergeString() {
-
- // [----->
- // <-----)
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "b\0"),
- KuduPredicate.newComparisonPredicate(stringCol, LESS, "b"),
- KuduPredicate.none(stringCol));
-
- // [----->
- // <-----)
- // =
- // None
- testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "b"),
- KuduPredicate.newComparisonPredicate(stringCol, LESS, "b"),
- KuduPredicate.none(stringCol));
-
- // [----->
- // <----)
- // =
- // |
- testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "b"),
- KuduPredicate.newComparisonPredicate(stringCol, LESS, "b\0"),
- KuduPredicate.newComparisonPredicate(stringCol, EQUAL, "b"));
-
- // [----->
- // <-----)
- // =
- // [--)
- testMerge(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "a"),
- KuduPredicate.newComparisonPredicate(stringCol, LESS, "a\0\0"),
- new KuduPredicate(RANGE, stringCol,
- Bytes.fromString("a"), Bytes.fromString("a\0\0")));
- }
-
- @Test
- public void testBoolean() {
-
- // b >= false
- Assert.assertEquals(KuduPredicate.newIsNotNullPredicate(boolCol),
- KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, false));
- // b > false
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true),
- KuduPredicate.newComparisonPredicate(boolCol, GREATER, false));
- // b = false
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false),
- KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false));
- // b < false
- Assert.assertEquals(KuduPredicate.none(boolCol),
- KuduPredicate.newComparisonPredicate(boolCol, LESS, false));
- // b <= false
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false),
- KuduPredicate.newComparisonPredicate(boolCol, LESS_EQUAL, false));
-
- // b >= true
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true),
- KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, true));
- // b > true
- Assert.assertEquals(KuduPredicate.none(boolCol),
- KuduPredicate.newComparisonPredicate(boolCol, GREATER, true));
- // b = true
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true),
- KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true));
- // b < true
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(boolCol, EQUAL, false),
- KuduPredicate.newComparisonPredicate(boolCol, LESS, true));
- // b <= true
- Assert.assertEquals(KuduPredicate.newIsNotNullPredicate(boolCol),
- KuduPredicate.newComparisonPredicate(boolCol, LESS_EQUAL, true));
- }
-
- /**
- * Tests basic predicate merges across all types.
- */
- @Test
- public void testAllTypesMerge() {
-
- testMerge(KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, false),
- KuduPredicate.newComparisonPredicate(boolCol, LESS, true),
- new KuduPredicate(KuduPredicate.PredicateType.EQUALITY,
- boolCol,
- Bytes.fromBoolean(false),
- null));
-
- testMerge(KuduPredicate.newComparisonPredicate(boolCol, GREATER_EQUAL, false),
- KuduPredicate.newComparisonPredicate(boolCol, LESS_EQUAL, true),
- KuduPredicate.newIsNotNullPredicate(boolCol));
-
- testMerge(KuduPredicate.newComparisonPredicate(byteCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(byteCol, LESS, 10),
- new KuduPredicate(RANGE,
- byteCol,
- new byte[] { (byte) 0 },
- new byte[] { (byte) 10 }));
-
- testMerge(KuduPredicate.newComparisonPredicate(shortCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(shortCol, LESS, 10),
- new KuduPredicate(RANGE,
- shortCol,
- Bytes.fromShort((short) 0),
- Bytes.fromShort((short) 10)));
-
- testMerge(KuduPredicate.newComparisonPredicate(longCol, GREATER_EQUAL, 0),
- KuduPredicate.newComparisonPredicate(longCol, LESS, 10),
- new KuduPredicate(RANGE,
- longCol,
- Bytes.fromLong(0),
- Bytes.fromLong(10)));
-
- testMerge(KuduPredicate.newComparisonPredicate(floatCol, GREATER_EQUAL, 123.45f),
- KuduPredicate.newComparisonPredicate(floatCol, LESS, 678.90f),
- new KuduPredicate(RANGE,
- floatCol,
- Bytes.fromFloat(123.45f),
- Bytes.fromFloat(678.90f)));
-
- testMerge(KuduPredicate.newComparisonPredicate(doubleCol, GREATER_EQUAL, 123.45),
- KuduPredicate.newComparisonPredicate(doubleCol, LESS, 678.90),
- new KuduPredicate(RANGE,
- doubleCol,
- Bytes.fromDouble(123.45),
- Bytes.fromDouble(678.90)));
-
- testMerge(KuduPredicate.newComparisonPredicate(binaryCol, GREATER_EQUAL,
- new byte[] { 0, 1, 2, 3, 4, 5, 6 }),
- KuduPredicate.newComparisonPredicate(binaryCol, LESS, new byte[] { 10 }),
- new KuduPredicate(RANGE,
- binaryCol,
- new byte[] { 0, 1, 2, 3, 4, 5, 6 },
- new byte[] { 10 }));
- }
-
- @Test
- public void testLessEqual() {
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(byteCol, LESS_EQUAL, 10),
- KuduPredicate.newComparisonPredicate(byteCol, LESS, 11));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(shortCol, LESS_EQUAL, 10),
- KuduPredicate.newComparisonPredicate(shortCol, LESS, 11));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(intCol, LESS_EQUAL, 10),
- KuduPredicate.newComparisonPredicate(intCol, LESS, 11));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(longCol, LESS_EQUAL, 10),
- KuduPredicate.newComparisonPredicate(longCol, LESS, 11));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, LESS_EQUAL, 12.345f),
- KuduPredicate.newComparisonPredicate(floatCol, LESS, Math.nextAfter(12.345f, Float.POSITIVE_INFINITY)));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, LESS_EQUAL, 12.345),
- KuduPredicate.newComparisonPredicate(doubleCol, LESS, Math.nextAfter(12.345, Float.POSITIVE_INFINITY)));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(stringCol, LESS_EQUAL, "a"),
- KuduPredicate.newComparisonPredicate(stringCol, LESS, "a\0"));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(binaryCol, LESS_EQUAL, new byte[] { (byte) 10 }),
- KuduPredicate.newComparisonPredicate(binaryCol, LESS, new byte[] { (byte) 10, (byte) 0 }));
-
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(byteCol, LESS_EQUAL, Byte.MAX_VALUE),
- KuduPredicate.newIsNotNullPredicate(byteCol));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(shortCol, LESS_EQUAL, Short.MAX_VALUE),
- KuduPredicate.newIsNotNullPredicate(shortCol));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(intCol, LESS_EQUAL, Integer.MAX_VALUE),
- KuduPredicate.newIsNotNullPredicate(intCol));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(longCol, LESS_EQUAL, Long.MAX_VALUE),
- KuduPredicate.newIsNotNullPredicate(longCol));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, LESS_EQUAL, Float.MAX_VALUE),
- KuduPredicate.newComparisonPredicate(floatCol, LESS, Float.POSITIVE_INFINITY));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, LESS_EQUAL, Float.POSITIVE_INFINITY),
- KuduPredicate.newIsNotNullPredicate(floatCol));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, LESS_EQUAL, Double.MAX_VALUE),
- KuduPredicate.newComparisonPredicate(doubleCol, LESS, Double.POSITIVE_INFINITY));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, LESS_EQUAL, Double.POSITIVE_INFINITY),
- KuduPredicate.newIsNotNullPredicate(doubleCol));
- }
-
- @Test
- public void testGreater() {
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(byteCol, GREATER_EQUAL, 11),
- KuduPredicate.newComparisonPredicate(byteCol, GREATER, 10));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(shortCol, GREATER_EQUAL, 11),
- KuduPredicate.newComparisonPredicate(shortCol, GREATER, 10));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(intCol, GREATER_EQUAL, 11),
- KuduPredicate.newComparisonPredicate(intCol, GREATER, 10));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(longCol, GREATER_EQUAL, 11),
- KuduPredicate.newComparisonPredicate(longCol, GREATER, 10));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, GREATER_EQUAL, Math.nextAfter(12.345f, Float.MAX_VALUE)),
- KuduPredicate.newComparisonPredicate(floatCol, GREATER, 12.345f));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, GREATER_EQUAL, Math.nextAfter(12.345, Float.MAX_VALUE)),
- KuduPredicate.newComparisonPredicate(doubleCol, GREATER, 12.345));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(stringCol, GREATER_EQUAL, "a\0"),
- KuduPredicate.newComparisonPredicate(stringCol, GREATER, "a"));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(binaryCol, GREATER_EQUAL, new byte[] { (byte) 10, (byte) 0 }),
- KuduPredicate.newComparisonPredicate(binaryCol, GREATER, new byte[] { (byte) 10 }));
-
- Assert.assertEquals(KuduPredicate.none(byteCol),
- KuduPredicate.newComparisonPredicate(byteCol, GREATER, Byte.MAX_VALUE));
- Assert.assertEquals(KuduPredicate.none(shortCol),
- KuduPredicate.newComparisonPredicate(shortCol, GREATER, Short.MAX_VALUE));
- Assert.assertEquals(KuduPredicate.none(intCol),
- KuduPredicate.newComparisonPredicate(intCol, GREATER, Integer.MAX_VALUE));
- Assert.assertEquals(KuduPredicate.none(longCol),
- KuduPredicate.newComparisonPredicate(longCol, GREATER, Long.MAX_VALUE));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(floatCol, GREATER_EQUAL, Float.POSITIVE_INFINITY),
- KuduPredicate.newComparisonPredicate(floatCol, GREATER, Float.MAX_VALUE));
- Assert.assertEquals(KuduPredicate.none(floatCol),
- KuduPredicate.newComparisonPredicate(floatCol, GREATER, Float.POSITIVE_INFINITY));
- Assert.assertEquals(KuduPredicate.newComparisonPredicate(doubleCol, GREATER_EQUAL, Double.POSITIVE_INFINITY),
- KuduPredicate.newComparisonPredicate(doubleCol, GREATER, Double.MAX_VALUE));
- Assert.assertEquals(KuduPredicate.none(doubleCol),
- KuduPredicate.newComparisonPredicate(doubleCol, GREATER, Double.POSITIVE_INFINITY));
- }
-
- @Test
- public void testToString() {
- Assert.assertEquals("`bool` = true",
- KuduPredicate.newComparisonPredicate(boolCol, EQUAL, true).toString());
- Assert.assertEquals("`byte` = 11",
- KuduPredicate.newComparisonPredicate(byteCol, EQUAL, 11).toString());
- Assert.assertEquals("`short` = 11",
- KuduPredicate.newComparisonPredicate(shortCol, EQUAL, 11).toString());
- Assert.assertEquals("`int` = -123",
- KuduPredicate.newComparisonPredicate(intCol, EQUAL, -123).toString());
- Assert.assertEquals("`long` = 5454",
- KuduPredicate.newComparisonPredicate(longCol, EQUAL, 5454).toString());
- Assert.assertEquals("`float` = 123.456",
- KuduPredicate.newComparisonPredicate(floatCol, EQUAL, 123.456f).toString());
- Assert.assertEquals("`double` = 123.456",
- KuduPredicate.newComparisonPredicate(doubleCol, EQUAL, 123.456).toString());
- Assert.assertEquals("`string` = \"my string\"",
- KuduPredicate.newComparisonPredicate(stringCol, EQUAL, "my string").toString());
- Assert.assertEquals("`binary` = 0xAB01CD", KuduPredicate.newComparisonPredicate(
- binaryCol, EQUAL, new byte[] { (byte) 0xAB, (byte) 0x01, (byte) 0xCD }).toString());
- }
-}
[11/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java
new file mode 100644
index 0000000..bf13f2d
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestColumnRangePredicate.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.Type;
+import org.kududb.tserver.Tserver;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class TestColumnRangePredicate {
+
+ @Test
+ public void testRawLists() {
+ ColumnSchema col1 = new ColumnSchema.ColumnSchemaBuilder("col1", Type.INT32).build();
+ ColumnSchema col2 = new ColumnSchema.ColumnSchemaBuilder("col2", Type.STRING).build();
+
+ ColumnRangePredicate pred1 = new ColumnRangePredicate(col1);
+ pred1.setLowerBound(1);
+
+ ColumnRangePredicate pred2 = new ColumnRangePredicate(col1);
+ pred2.setUpperBound(2);
+
+ ColumnRangePredicate pred3 = new ColumnRangePredicate(col2);
+ pred3.setLowerBound("aaa");
+ pred3.setUpperBound("bbb");
+
+ List<ColumnRangePredicate> preds = Lists.newArrayList(pred1, pred2, pred3);
+
+ byte[] rawPreds = ColumnRangePredicate.toByteArray(preds);
+
+ List<Tserver.ColumnRangePredicatePB> decodedPreds = null;
+ try {
+ decodedPreds = ColumnRangePredicate.fromByteArray(rawPreds);
+ } catch (IllegalArgumentException e) {
+ fail("Couldn't decode: " + e.getMessage());
+ }
+
+ assertEquals(3, decodedPreds.size());
+
+ assertEquals(col1.getName(), decodedPreds.get(0).getColumn().getName());
+ assertEquals(1, Bytes.getInt(Bytes.get(decodedPreds.get(0).getLowerBound())));
+ assertFalse(decodedPreds.get(0).hasInclusiveUpperBound());
+
+ assertEquals(col1.getName(), decodedPreds.get(1).getColumn().getName());
+ assertEquals(2, Bytes.getInt(Bytes.get(decodedPreds.get(1).getInclusiveUpperBound())));
+ assertFalse(decodedPreds.get(1).hasLowerBound());
+
+ assertEquals(col2.getName(), decodedPreds.get(2).getColumn().getName());
+ assertEquals("aaa", Bytes.getString(Bytes.get(decodedPreds.get(2).getLowerBound())));
+ assertEquals("bbb", Bytes.getString(Bytes.get(decodedPreds.get(2).getInclusiveUpperBound())));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java
new file mode 100644
index 0000000..20ee06a
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestDeadlineTracker.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import static org.junit.Assert.*;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Ticker;
+import org.junit.Test;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class TestDeadlineTracker {
+
+ @Test
+ public void testTimeout() {
+ final AtomicLong timeToReturn = new AtomicLong();
+ Ticker ticker = new Ticker() {
+ @Override
+ public long read() {
+ return timeToReturn.get();
+ }
+ };
+ Stopwatch stopwatch = Stopwatch.createUnstarted(ticker);
+
+ // no timeout set
+ DeadlineTracker tracker = new DeadlineTracker(stopwatch);
+ tracker.setDeadline(0);
+ assertFalse(tracker.hasDeadline());
+ assertFalse(tracker.timedOut());
+
+ // 500ms timeout set
+ tracker.reset();
+ tracker.setDeadline(500);
+ assertTrue(tracker.hasDeadline());
+ assertFalse(tracker.timedOut());
+ assertFalse(tracker.wouldSleepingTimeout(499));
+ assertTrue(tracker.wouldSleepingTimeout(500));
+ assertTrue(tracker.wouldSleepingTimeout(501));
+ assertEquals(500, tracker.getMillisBeforeDeadline());
+
+ // fast forward 200ms
+ timeToReturn.set(200 * 1000000);
+ assertTrue(tracker.hasDeadline());
+ assertFalse(tracker.timedOut());
+ assertFalse(tracker.wouldSleepingTimeout(299));
+ assertTrue(tracker.wouldSleepingTimeout(300));
+ assertTrue(tracker.wouldSleepingTimeout(301));
+ assertEquals(300, tracker.getMillisBeforeDeadline());
+
+ // fast forward another 400ms, so the RPC timed out
+ timeToReturn.set(600 * 1000000);
+ assertTrue(tracker.hasDeadline());
+ assertTrue(tracker.timedOut());
+ assertTrue(tracker.wouldSleepingTimeout(299));
+ assertTrue(tracker.wouldSleepingTimeout(300));
+ assertTrue(tracker.wouldSleepingTimeout(301));
+ assertEquals(1, tracker.getMillisBeforeDeadline());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java
new file mode 100644
index 0000000..883b2e1
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestErrorCollector.java
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestErrorCollector {
+
+ @Test
+ public void testErrorCollector() {
+ int maxErrors = 10;
+ ErrorCollector collector = new ErrorCollector(maxErrors);
+
+ // Test with no errors.
+ int countToTest = 0;
+ Assert.assertEquals(countToTest, collector.countErrors());
+ RowErrorsAndOverflowStatus reos = collector.getErrors();
+ Assert.assertEquals(0, collector.countErrors());
+ Assert.assertFalse(reos.isOverflowed());
+ Assert.assertEquals(countToTest, reos.getRowErrors().length);
+
+ // Test a single row error.
+ countToTest = 1;
+ collector.addError(createRowError(countToTest));
+ Assert.assertEquals(countToTest, collector.countErrors());
+ reos = collector.getErrors();
+ Assert.assertEquals(0, collector.countErrors());
+ Assert.assertFalse(reos.isOverflowed());
+ Assert.assertEquals(countToTest, reos.getRowErrors().length);
+ Assert.assertEquals(countToTest, reos.getRowErrors()[0].getErrorStatus().getPosixCode());
+
+ // Test filling the collector to the max.
+ countToTest = maxErrors;
+ fillCollectorWith(collector, countToTest);
+ Assert.assertEquals(countToTest, collector.countErrors());
+ reos = collector.getErrors();
+ Assert.assertEquals(0, collector.countErrors());
+ Assert.assertFalse(reos.isOverflowed());
+ Assert.assertEquals(countToTest, reos.getRowErrors().length);
+ Assert.assertEquals(countToTest - 1, reos.getRowErrors()[9].getErrorStatus().getPosixCode());
+
+ // Test overflowing.
+ countToTest = 95;
+ fillCollectorWith(collector, countToTest);
+ Assert.assertEquals(maxErrors, collector.countErrors());
+ reos = collector.getErrors();
+ Assert.assertEquals(0, collector.countErrors());
+ Assert.assertTrue(reos.isOverflowed());
+ Assert.assertEquals(maxErrors, reos.getRowErrors().length);
+ Assert.assertEquals(countToTest - 1, reos.getRowErrors()[9].getErrorStatus().getPosixCode());
+
+ // Test overflowing on a newly created collector.
+ countToTest = 95;
+ collector = new ErrorCollector(maxErrors);
+ fillCollectorWith(collector, countToTest);
+ Assert.assertEquals(maxErrors, collector.countErrors());
+ reos = collector.getErrors();
+ Assert.assertEquals(0, collector.countErrors());
+ Assert.assertTrue(reos.isOverflowed());
+ Assert.assertEquals(maxErrors, reos.getRowErrors().length);
+ Assert.assertEquals(countToTest - 1, reos.getRowErrors()[9].getErrorStatus().getPosixCode());
+ }
+
+ private void fillCollectorWith(ErrorCollector collector, int errorsToAdd) {
+ for (int i = 0; i < errorsToAdd; i++) {
+ collector.addError(createRowError(i));
+ }
+ }
+
+ private RowError createRowError(int id) {
+ // Use the error status as a way to message pass and so that we can test we're getting the right
+ // messages on the other end.
+ return new RowError(Status.NotAuthorized("test", id), null, "test");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
new file mode 100644
index 0000000..dafd74a
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestFlexiblePartitioning.java
@@ -0,0 +1,422 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import org.junit.Before;
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class TestFlexiblePartitioning extends BaseKuduTest {
+ private String tableName;
+
+ @Before
+ public void setTableName() {
+ tableName = TestKuduClient.class.getName() + "-" + System.currentTimeMillis();
+ }
+
+ private static Schema createSchema() {
+ ArrayList<ColumnSchema> columns = new ArrayList<>(3);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("a", Type.STRING).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("b", Type.STRING).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c", Type.STRING).key(true).build());
+ return new Schema(columns);
+ }
+
+ private static Set<Row> rows() throws Exception {
+ Set<Row> rows = new HashSet<>();
+ for (int a = 0; a < 6; a++) {
+ for (int b = 0; b < 6; b++) {
+ for (int c = 0; c < 6; c++) {
+ rows.add(new Row(String.format("%s", a),
+ String.format("%s", b),
+ String.format("%s", c)));
+ }
+ }
+ }
+ return rows;
+ }
+
+ private void insertRows(KuduTable table, Set<Row> rows) throws Exception {
+ KuduSession session = syncClient.newSession();
+ try {
+ for (Row row : rows) {
+ Insert insert = table.newInsert();
+ PartialRow insertRow = insert.getRow();
+ row.fillPartialRow(insertRow);
+ session.apply(insert);
+ }
+ } finally {
+ session.close();
+ }
+ }
+
+ private Set<Row> collectRows(KuduScanner scanner) throws Exception {
+ Set<Row> rows = new HashSet<>();
+ while (scanner.hasMoreRows()) {
+ for (RowResult result : scanner.nextRows()) {
+ rows.add(Row.fromResult(result));
+ }
+ }
+ return rows;
+ }
+
+ private void testPartitionSchema(CreateTableOptions tableBuilder) throws Exception {
+ Schema schema = createSchema();
+
+ syncClient.createTable(tableName, schema, tableBuilder);
+
+ KuduTable table = syncClient.openTable(tableName);
+
+ Set<Row> rows = rows();
+ insertRows(table, rows);
+
+ // Full table scan
+ assertEquals(rows, collectRows(syncClient.newScannerBuilder(table).build()));
+
+ { // Lower bound
+ Row minRow = new Row("1", "3", "5");
+ PartialRow lowerBound = schema.newPartialRow();
+ minRow.fillPartialRow(lowerBound);
+
+ Set<Row> expected = Sets.filter(rows, minRow.gtePred());
+
+ KuduScanner scanner = syncClient.newScannerBuilder(table).lowerBound(lowerBound).build();
+ Set<Row> results = collectRows(scanner);
+
+ assertEquals(expected, results);
+ }
+
+ { // Upper bound
+ Row maxRow = new Row("1", "3", "5");
+ PartialRow upperBound = schema.newPartialRow();
+ maxRow.fillPartialRow(upperBound);
+
+ Set<Row> expected = Sets.filter(rows, maxRow.ltPred());
+
+ KuduScanner scanner = syncClient.newScannerBuilder(table)
+ .exclusiveUpperBound(upperBound)
+ .build();
+ Set<Row> results = collectRows(scanner);
+
+ assertEquals(expected, results);
+ }
+
+ { // Lower & Upper bounds
+ Row minRow = new Row("1", "3", "5");
+ Row maxRow = new Row("2", "4", "");
+ PartialRow lowerBound = schema.newPartialRow();
+ minRow.fillPartialRow(lowerBound);
+ PartialRow upperBound = schema.newPartialRow();
+ maxRow.fillPartialRow(upperBound);
+
+ Set<Row> expected = Sets.filter(rows, Predicates.and(minRow.gtePred(), maxRow.ltPred()));
+
+ KuduScanner scanner = syncClient.newScannerBuilder(table)
+ .lowerBound(lowerBound)
+ .exclusiveUpperBound(upperBound)
+ .build();
+ Set<Row> results = collectRows(scanner);
+
+ assertEquals(expected, results);
+ }
+
+ List<LocatedTablet> tablets = table.getTabletsLocations(TestTimeouts.DEFAULT_SLEEP);
+
+ { // Per-tablet scan
+ Set<Row> results = new HashSet<>();
+
+ for (LocatedTablet tablet : tablets) {
+ KuduScanner scanner = syncClient.newScannerBuilder(table)
+ .lowerBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyStart())
+ .exclusiveUpperBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyEnd())
+ .build();
+ Set<Row> tabletResults = collectRows(scanner);
+ Set<Row> intersection = Sets.intersection(results, tabletResults);
+ assertEquals(new HashSet<>(), intersection);
+ results.addAll(tabletResults);
+ }
+
+ assertEquals(rows, results);
+ }
+
+ { // Per-tablet scan with lower & upper bounds
+ Row minRow = new Row("1", "3", "5");
+ Row maxRow = new Row("2", "4", "");
+ PartialRow lowerBound = schema.newPartialRow();
+ minRow.fillPartialRow(lowerBound);
+ PartialRow upperBound = schema.newPartialRow();
+ maxRow.fillPartialRow(upperBound);
+
+ Set<Row> expected = Sets.filter(rows, Predicates.and(minRow.gtePred(), maxRow.ltPred()));
+ Set<Row> results = new HashSet<>();
+
+ for (LocatedTablet tablet : tablets) {
+ KuduScanner scanner = syncClient.newScannerBuilder(table)
+ .lowerBound(lowerBound)
+ .exclusiveUpperBound(upperBound)
+ .lowerBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyStart())
+ .exclusiveUpperBoundPartitionKeyRaw(tablet.getPartition().getPartitionKeyEnd())
+ .build();
+ Set<Row> tabletResults = collectRows(scanner);
+ Set<Row> intersection = Sets.intersection(results, tabletResults);
+ assertEquals(new HashSet<>(), intersection);
+ results.addAll(tabletResults);
+ }
+
+ assertEquals(expected, results);
+ }
+ }
+
+ @Test(timeout = 100000)
+ public void testHashBucketedTable() throws Exception {
+ CreateTableOptions tableBuilder = new CreateTableOptions();
+ tableBuilder.addHashPartitions(ImmutableList.of("a"), 3);
+ tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42);
+ tableBuilder.setRangePartitionColumns(ImmutableList.<String>of());
+ testPartitionSchema(tableBuilder);
+ }
+
+ @Test(timeout = 100000)
+ public void testNonDefaultRangePartitionedTable() throws Exception {
+ Schema schema = createSchema();
+ CreateTableOptions tableBuilder = new CreateTableOptions();
+ tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b"));
+
+ PartialRow split = schema.newPartialRow();
+ split.addString("c", "3");
+ tableBuilder.addSplitRow(split);
+
+ split = schema.newPartialRow();
+ split.addString("c", "3");
+ split.addString("b", "3");
+ tableBuilder.addSplitRow(split);
+
+ testPartitionSchema(tableBuilder);
+ }
+
+ @Test(timeout = 100000)
+ public void testHashBucketedAndRangePartitionedTable() throws Exception {
+ Schema schema = createSchema();
+ CreateTableOptions tableBuilder = new CreateTableOptions();
+ tableBuilder.addHashPartitions(ImmutableList.of("a"), 3);
+ tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42);
+ tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b"));
+
+ PartialRow split = schema.newPartialRow();
+ split.addString("c", "3");
+ tableBuilder.addSplitRow(split);
+
+ split = schema.newPartialRow();
+ split.addString("c", "3");
+ split.addString("b", "3");
+ tableBuilder.addSplitRow(split);
+
+ testPartitionSchema(tableBuilder);
+ }
+
+ @Test(timeout = 100000)
+ public void testNonCoveredRangePartitionedTable() throws Exception {
+ Schema schema = createSchema();
+ CreateTableOptions tableBuilder = new CreateTableOptions();
+ tableBuilder.setRangePartitionColumns(ImmutableList.of("a", "b", "c"));
+
+ // Create a non covered range between (3, 5, 6) and (4, 0, 0)
+
+ PartialRow lowerBoundA = schema.newPartialRow();
+ lowerBoundA.addString("a", "0");
+ lowerBoundA.addString("b", "0");
+ lowerBoundA.addString("c", "0");
+ PartialRow upperBoundA = schema.newPartialRow();
+ upperBoundA.addString("a", "3");
+ upperBoundA.addString("b", "5");
+ upperBoundA.addString("b", "6");
+ tableBuilder.addRangeBound(lowerBoundA, upperBoundA);
+
+ PartialRow lowerBoundB = schema.newPartialRow();
+ lowerBoundB.addString("a", "4");
+ lowerBoundB.addString("b", "0");
+ lowerBoundB.addString("c", "0");
+ PartialRow upperBoundB = schema.newPartialRow();
+ upperBoundB.addString("a", "5");
+ upperBoundB.addString("b", "5");
+ upperBoundB.addString("b", "6");
+ tableBuilder.addRangeBound(lowerBoundB, upperBoundB);
+
+ testPartitionSchema(tableBuilder);
+ }
+
+ @Test(timeout = 100000)
+ public void testHashBucketedAndNonCoveredRangePartitionedTable() throws Exception {
+ Schema schema = createSchema();
+ CreateTableOptions tableBuilder = new CreateTableOptions();
+ tableBuilder.setRangePartitionColumns(ImmutableList.of("a", "b", "c"));
+
+ // Create a non covered range between (3, 5, 6) and (4, 0, 0)
+
+ PartialRow lowerBoundA = schema.newPartialRow();
+ lowerBoundA.addString("a", "0");
+ lowerBoundA.addString("b", "0");
+ lowerBoundA.addString("c", "0");
+ PartialRow upperBoundA = schema.newPartialRow();
+ upperBoundA.addString("a", "3");
+ upperBoundA.addString("b", "5");
+ upperBoundA.addString("c", "6");
+ tableBuilder.addRangeBound(lowerBoundA, upperBoundA);
+
+ PartialRow lowerBoundB = schema.newPartialRow();
+ lowerBoundB.addString("a", "4");
+ lowerBoundB.addString("b", "0");
+ lowerBoundB.addString("c", "0");
+ PartialRow upperBoundB = schema.newPartialRow();
+ upperBoundB.addString("a", "5");
+ upperBoundB.addString("b", "5");
+ upperBoundB.addString("c", "6");
+ tableBuilder.addRangeBound(lowerBoundB, upperBoundB);
+
+ tableBuilder.addHashPartitions(ImmutableList.of("a", "b", "c"), 4);
+
+ testPartitionSchema(tableBuilder);
+ }
+
+ @Test(timeout = 100000)
+ public void testSimplePartitionedTable() throws Exception {
+ Schema schema = createSchema();
+ CreateTableOptions tableBuilder =
+ new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("a", "b", "c"));
+
+ PartialRow split = schema.newPartialRow();
+ split.addString("c", "3");
+ tableBuilder.addSplitRow(split);
+
+ split = schema.newPartialRow();
+ split.addString("c", "3");
+ split.addString("b", "3");
+ tableBuilder.addSplitRow(split);
+
+ testPartitionSchema(tableBuilder);
+ }
+
+ @Test(timeout = 100000)
+ public void testUnpartitionedTable() throws Exception {
+ CreateTableOptions tableBuilder =
+ new CreateTableOptions().setRangePartitionColumns(ImmutableList.<String>of());
+ testPartitionSchema(tableBuilder);
+ }
+
+ public static class Row implements Comparable<Row> {
+ private final String a;
+ private final String b;
+ private final String c;
+
+ public Row(String a, String b, String c) {
+ this.a = a;
+ this.b = b;
+ this.c = c;
+ }
+
+ public String getA() {
+ return a;
+ }
+
+ public String getB() {
+ return b;
+ }
+
+ public String getC() {
+ return c;
+ }
+
+ public void fillPartialRow(PartialRow row) {
+ row.addString("a", a);
+ row.addString("b", b);
+ row.addString("c", c);
+ }
+
+ private static Row fromResult(RowResult result) {
+ return new Row(result.getString("a"),
+ result.getString("b"),
+ result.getString("c"));
+ }
+
+ public Predicate<Row> gtePred() {
+ return new Predicate<Row>() {
+ @Override
+ public boolean apply(Row other) {
+ return other.compareTo(Row.this) >= 0;
+ }
+ };
+ }
+
+ public Predicate<Row> ltPred() {
+ return new Predicate<Row>() {
+ @Override
+ public boolean apply(Row other) {
+ return other.compareTo(Row.this) < 0;
+ }
+ };
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Row row = (Row) o;
+ return Objects.equals(a, row.a)
+ && Objects.equals(b, row.b)
+ && Objects.equals(c, row.c);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(a, b, c);
+ }
+
+ @Override
+ public int compareTo(Row other) {
+ return ComparisonChain.start()
+ .compare(a, other.a)
+ .compare(b, other.b)
+ .compare(c, other.c)
+ .result();
+ }
+
+ @Override
+ public String toString() {
+ return com.google.common.base.Objects.toStringHelper(this)
+ .add("a", a)
+ .add("b", b)
+ .add("c", c)
+ .toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java
new file mode 100644
index 0000000..666ac38
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestHybridTime.java
@@ -0,0 +1,163 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import com.stumbleupon.async.Deferred;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.kududb.Type.STRING;
+import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+import static org.kududb.util.HybridTimeUtil.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This only tests client propagation since it's the only thing that is client-specific.
+ * All the work for commit wait is done and tested on the server-side.
+ */
+public class TestHybridTime extends BaseKuduTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TestHybridTime.class);
+
+ // Generate a unique table name
+ protected static final String TABLE_NAME =
+ TestHybridTime.class.getName() + "-" + System.currentTimeMillis();
+
+ protected static Schema schema = getSchema();
+ protected static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+
+ // Using multiple tablets doesn't work with the current way this test works since we could
+ // jump from one TS to another which changes the logical clock.
+ CreateTableOptions builder =
+ new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
+ table = createTable(TABLE_NAME, schema, builder);
+ }
+
+ private static Schema getSchema() {
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(1);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key", STRING)
+ .key(true)
+ .build());
+ return new Schema(columns);
+ }
+
+ /**
+ * We write three rows. We increment the timestamp we get back from the first write
+ * by some amount. The remaining writes should force an update to the server's clock and
+ * only increment the logical values.
+ *
+ * @throws Exception
+ */
+ @Test(timeout = 100000)
+ public void test() throws Exception {
+ AsyncKuduSession session = client.newSession();
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
+ session.setExternalConsistencyMode(CLIENT_PROPAGATED);
+ long[] clockValues;
+ long previousLogicalValue = 0;
+ long previousPhysicalValue = 0;
+
+ // Test timestamp propagation with single operations
+ String[] keys = new String[] {"1", "2", "3"};
+ for (int i = 0; i < keys.length; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addString(schema.getColumnByIndex(0).getName(), keys[i]);
+ Deferred<OperationResponse> d = session.apply(insert);
+ OperationResponse response = d.join(DEFAULT_SLEEP);
+ assertTrue(response.getWriteTimestamp() != 0);
+ clockValues = HTTimestampToPhysicalAndLogical(response.getWriteTimestamp());
+ LOG.debug("Clock value after write[" + i + "]: " + new Date(clockValues[0] / 1000).toString()
+ + " Logical value: " + clockValues[1]);
+ // on the very first write we update the clock into the future
+ // so that remaining writes only update logical values
+ if (i == 0) {
+ assertEquals(clockValues[1], 0);
+ long toUpdateTs = clockValues[0] + 5000000;
+ previousPhysicalValue = toUpdateTs;
+ // After the first write we fake-update the clock into the future. Following writes
+ // should force the servers to update their clocks to this value.
+ client.updateLastPropagatedTimestamp(
+ clockTimestampToHTTimestamp(toUpdateTs, TimeUnit.MICROSECONDS));
+ } else {
+ assertEquals(clockValues[0], previousPhysicalValue);
+ assertTrue(clockValues[1] > previousLogicalValue);
+ previousLogicalValue = clockValues[1];
+ }
+ }
+
+ // Test timestamp propagation with Batches
+ session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
+ keys = new String[] {"11", "22", "33"};
+ for (int i = 0; i < keys.length; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addString(schema.getColumnByIndex(0).getName(), keys[i]);
+ session.apply(insert);
+ Deferred<List<OperationResponse>> d = session.flush();
+ List<OperationResponse> responses = d.join(DEFAULT_SLEEP);
+ assertEquals("Response was not of the expected size: " + responses.size(),
+ 1, responses.size());
+
+ OperationResponse response = responses.get(0);
+ assertTrue(response.getWriteTimestamp() != 0);
+ clockValues = HTTimestampToPhysicalAndLogical(response.getWriteTimestamp());
+ LOG.debug("Clock value after write[" + i + "]: " + new Date(clockValues[0] / 1000).toString()
+ + " Logical value: " + clockValues[1]);
+ assertEquals(clockValues[0], previousPhysicalValue);
+ assertTrue(clockValues[1] > previousLogicalValue);
+ previousLogicalValue = clockValues[1];
+ }
+
+ // Scan all rows with READ_LATEST (the default) we should get 6 rows back
+ assertEquals(6, countRowsInScan(client.newScannerBuilder(table).build()));
+
+ // Now scan at multiple instances with READ_AT_SNAPSHOT we should get different
+ // counts depending on the scan timestamp.
+ long snapTime = physicalAndLogicalToHTTimestamp(previousPhysicalValue, 0);
+ assertEquals(1, scanAtSnapshot(snapTime));
+ snapTime = physicalAndLogicalToHTTimestamp(previousPhysicalValue, 5);
+ assertEquals(4, scanAtSnapshot(snapTime));
+ // Our last snap time needs to one one into the future w.r.t. the last write's timestamp
+ // for us to be able to get all rows, but the snap timestamp can't be bigger than the prop.
+ // timestamp so we increase both.
+ client.updateLastPropagatedTimestamp(client.getLastPropagatedTimestamp() + 1);
+ snapTime = physicalAndLogicalToHTTimestamp(previousPhysicalValue, previousLogicalValue + 1);
+ assertEquals(6, scanAtSnapshot(snapTime));
+ }
+
+ private int scanAtSnapshot(long time) throws Exception {
+ AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table)
+ .snapshotTimestampRaw(time)
+ .readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT);
+ return countRowsInScan(builder.build());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
new file mode 100644
index 0000000..e446445
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKeyEncoding.java
@@ -0,0 +1,272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.ColumnSchema.ColumnSchemaBuilder;
+import org.kududb.Common;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.client.PartitionSchema.HashBucketSchema;
+import org.kududb.client.PartitionSchema.RangeSchema;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestKeyEncoding {
+
+ private static Schema buildSchema(ColumnSchemaBuilder... columns) {
+ int i = 0;
+ Common.SchemaPB.Builder pb = Common.SchemaPB.newBuilder();
+ for (ColumnSchemaBuilder column : columns) {
+ Common.ColumnSchemaPB.Builder columnPb =
+ ProtobufHelper.columnToPb(column.build()).toBuilder();
+ columnPb.setId(i++);
+ pb.addColumns(columnPb);
+ }
+ return ProtobufHelper.pbToSchema(pb.build());
+ }
+
+ private static void assertBytesEquals(byte[] actual, byte[] expected) {
+ assertTrue(String.format("expected: '%s', got '%s'",
+ Bytes.pretty(expected),
+ Bytes.pretty(actual)),
+ Bytes.equals(expected, actual));
+ }
+
+ private static void assertBytesEquals(byte[] actual, String expected) {
+ assertBytesEquals(actual, expected.getBytes(Charsets.UTF_8));
+ }
+
+ /**
+ * Builds the default partition schema for a schema.
+ * @param schema the schema
+ * @return a default partition schema
+ */
+ private PartitionSchema defaultPartitionSchema(Schema schema) {
+ List<Integer> columnIds = new ArrayList<>();
+ for (int i = 0; i < schema.getPrimaryKeyColumnCount(); i++) {
+ // Schema does not provide a way to lookup a column ID by column index,
+ // so instead we assume that the IDs for the primary key columns match
+ // their respective index, which holds up when the schema is created
+ // with buildSchema.
+ columnIds.add(i);
+ }
+ return new PartitionSchema(
+ new PartitionSchema.RangeSchema(columnIds),
+ ImmutableList.<PartitionSchema.HashBucketSchema>of(), schema);
+ }
+
+ @Test
+ public void testPrimaryKeys() {
+ Schema schemaOneString =
+ buildSchema(new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true));
+ KuduTable table = new KuduTable(null, "one", "one", schemaOneString,
+ defaultPartitionSchema(schemaOneString));
+ Insert oneKeyInsert = new Insert(table);
+ PartialRow row = oneKeyInsert.getRow();
+ row.addString("key", "foo");
+ assertBytesEquals(row.encodePrimaryKey(), "foo");
+
+ Schema schemaTwoString = buildSchema(
+ new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true),
+ new ColumnSchema.ColumnSchemaBuilder("key2", Type.STRING).key(true));
+ KuduTable table2 = new KuduTable(null, "two", "two", schemaTwoString,
+ defaultPartitionSchema(schemaTwoString));
+ Insert twoKeyInsert = new Insert(table2);
+ row = twoKeyInsert.getRow();
+ row.addString("key", "foo");
+ row.addString("key2", "bar");
+ assertBytesEquals(row.encodePrimaryKey(), "foo\0\0bar");
+
+ Insert twoKeyInsertWithNull = new Insert(table2);
+ row = twoKeyInsertWithNull.getRow();
+ row.addString("key", "xxx\0yyy");
+ row.addString("key2", "bar");
+ assertBytesEquals(row.encodePrimaryKey(), "xxx\0\1yyy\0\0bar");
+
+ // test that we get the correct memcmp result, the bytes are in big-endian order in a key
+ Schema schemaIntString = buildSchema(
+ new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true),
+ new ColumnSchema.ColumnSchemaBuilder("key2", Type.STRING).key(true));
+ PartitionSchema partitionSchemaIntString = defaultPartitionSchema(schemaIntString);
+ KuduTable table3 = new KuduTable(null, "three", "three",
+ schemaIntString, partitionSchemaIntString);
+ Insert small = new Insert(table3);
+ row = small.getRow();
+ row.addInt("key", 20);
+ row.addString("key2", "data");
+ byte[] smallPK = small.getRow().encodePrimaryKey();
+ assertEquals(0, Bytes.memcmp(smallPK, smallPK));
+
+ Insert big = new Insert(table3);
+ row = big.getRow();
+ row.addInt("key", 10000);
+ row.addString("key2", "data");
+ byte[] bigPK = big.getRow().encodePrimaryKey();
+ assertTrue(Bytes.memcmp(smallPK, bigPK) < 0);
+ assertTrue(Bytes.memcmp(bigPK, smallPK) > 0);
+
+ // The following tests test our assumptions on unsigned data types sorting from KeyEncoder
+ byte four = 4;
+ byte onHundredTwentyFour = -4;
+ four = Bytes.xorLeftMostBit(four);
+ onHundredTwentyFour = Bytes.xorLeftMostBit(onHundredTwentyFour);
+ assertTrue(four < onHundredTwentyFour);
+
+ byte[] threeHundred = Bytes.fromInt(300);
+ byte[] reallyBigNumber = Bytes.fromInt(-300);
+ threeHundred[0] = Bytes.xorLeftMostBit(threeHundred[0]);
+ reallyBigNumber[3] = Bytes.xorLeftMostBit(reallyBigNumber[3]);
+ assertTrue(Bytes.memcmp(threeHundred, reallyBigNumber) < 0);
+ }
+
+ @Test
+ public void testPrimaryKeyEncoding() {
+ Schema schema = buildSchema(
+ new ColumnSchemaBuilder("int8", Type.INT8).key(true),
+ new ColumnSchemaBuilder("int16", Type.INT16).key(true),
+ new ColumnSchemaBuilder("int32", Type.INT32).key(true),
+ new ColumnSchemaBuilder("int64", Type.INT64).key(true),
+ new ColumnSchemaBuilder("string", Type.STRING).key(true),
+ new ColumnSchemaBuilder("binary", Type.BINARY).key(true));
+
+ PartialRow rowA = schema.newPartialRow();
+ rowA.addByte("int8", Byte.MIN_VALUE);
+ rowA.addShort("int16", Short.MIN_VALUE);
+ rowA.addInt("int32", Integer.MIN_VALUE);
+ rowA.addLong("int64", Long.MIN_VALUE);
+ rowA.addString("string", "");
+ rowA.addBinary("binary", "".getBytes(Charsets.UTF_8));
+
+ assertBytesEquals(rowA.encodePrimaryKey(),
+ "\0"
+ + "\0\0"
+ + "\0\0\0\0"
+ + "\0\0\0\0\0\0\0\0"
+ + "\0\0"
+ + "");
+
+ PartialRow rowB = schema.newPartialRow();
+ rowB.addByte("int8", Byte.MAX_VALUE);
+ rowB.addShort("int16", Short.MAX_VALUE);
+ rowB.addInt("int32", Integer.MAX_VALUE);
+ rowB.addLong("int64", Long.MAX_VALUE);
+ rowB.addString("string", "abc\1\0def");
+ rowB.addBinary("binary", "\0\1binary".getBytes(Charsets.UTF_8));
+
+ assertBytesEquals(rowB.encodePrimaryKey(),
+ new byte[] {
+ -1,
+ -1, -1,
+ -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ 'a', 'b', 'c', 1, 0, 1, 'd', 'e', 'f', 0, 0,
+ 0, 1, 'b', 'i', 'n', 'a', 'r', 'y',
+ });
+
+ PartialRow rowC = schema.newPartialRow();
+ rowC.addByte("int8", (byte) 1);
+ rowC.addShort("int16", (short) 2);
+ rowC.addInt("int32", 3);
+ rowC.addLong("int64", 4);
+ rowC.addString("string", "abc\n123");
+ rowC.addBinary("binary", "\0\1\2\3\4\5".getBytes(Charsets.UTF_8));
+
+ assertBytesEquals(rowC.encodePrimaryKey(),
+ new byte[] {
+ (byte) 0x81,
+ (byte) 0x80, 2,
+ (byte) 0x80, 0, 0, 3,
+ (byte) 0x80, 0, 0, 0, 0, 0, 0, 4,
+ 'a', 'b', 'c', '\n', '1', '2', '3', 0, 0,
+ 0, 1, 2, 3, 4, 5,
+ });
+ }
+
+ @Test
+ public void testPartitionKeyEncoding() {
+ KeyEncoder encoder = new KeyEncoder();
+ Schema schema = buildSchema(
+ new ColumnSchemaBuilder("a", Type.INT32).key(true),
+ new ColumnSchemaBuilder("b", Type.STRING).key(true),
+ new ColumnSchemaBuilder("c", Type.STRING).key(true));
+
+ PartitionSchema partitionSchema =
+ new PartitionSchema(new RangeSchema(ImmutableList.of(0, 1, 2)),
+ ImmutableList.of(
+ new HashBucketSchema(ImmutableList.of(0, 1), 32, 0),
+ new HashBucketSchema(ImmutableList.of(2), 32, 42)),
+ schema);
+
+ PartialRow rowA = schema.newPartialRow();
+ rowA.addInt("a", 0);
+ rowA.addString("b", "");
+ rowA.addString("c", "");
+ assertBytesEquals(encoder.encodePartitionKey(rowA, partitionSchema),
+ new byte[]{
+ 0, 0, 0, 0, // hash(0, "")
+ 0, 0, 0, 0x14, // hash("")
+ (byte) 0x80, 0, 0, 0, // a = 0
+ 0, 0, // b = ""; c is elided
+ });
+
+ PartialRow rowB = schema.newPartialRow();
+ rowB.addInt("a", 1);
+ rowB.addString("b", "");
+ rowB.addString("c", "");
+ assertBytesEquals(encoder.encodePartitionKey(rowB, partitionSchema),
+ new byte[]{
+ 0, 0, 0, 0x5, // hash(1, "")
+ 0, 0, 0, 0x14, // hash("")
+ (byte) 0x80, 0, 0, 1, // a = 0
+ 0, 0, // b = ""; c is elided
+ });
+
+ PartialRow rowC = schema.newPartialRow();
+ rowC.addInt("a", 0);
+ rowC.addString("b", "b");
+ rowC.addString("c", "c");
+ assertBytesEquals(encoder.encodePartitionKey(rowC, partitionSchema),
+ new byte[]{
+ 0, 0, 0, 0x1A, // hash(0, "b")
+ 0, 0, 0, 0x1D, // hash("c")
+ (byte) 0x80, 0, 0, 0, // a = 1
+ 'b', 0, 0, // b = "b"
+ 'c' // b = "c"
+ });
+
+ PartialRow rowD = schema.newPartialRow();
+ rowD.addInt("a", 1);
+ rowD.addString("b", "b");
+ rowD.addString("c", "c");
+ assertBytesEquals(encoder.encodePartitionKey(rowD, partitionSchema),
+ new byte[]{
+ 0, 0, 0, 0, // hash(1, "b")
+ 0, 0, 0, 0x1D, // hash("c")
+ (byte) 0x80, 0, 0, 1, // a = 0
+ 'b', 0, 0, // b = "b"
+ 'c' // b = "c"
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
new file mode 100644
index 0000000..3591b7b
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -0,0 +1,535 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER;
+import static org.kududb.client.KuduPredicate.ComparisonOp.GREATER_EQUAL;
+import static org.kududb.client.KuduPredicate.ComparisonOp.LESS;
+import static org.kududb.client.KuduPredicate.ComparisonOp.LESS_EQUAL;
+import static org.kududb.client.RowResult.timestampToString;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestKuduClient extends BaseKuduTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TestKuduClient.class);
+ private String tableName;
+
+ @Before
+ public void setTableName() {
+ tableName = TestKuduClient.class.getName() + "-" + System.currentTimeMillis();
+ }
+
+ private Schema createManyStringsSchema() {
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.STRING).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.STRING).nullable(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.STRING).nullable(true).build());
+ return new Schema(columns);
+ }
+
+ private Schema createSchemaWithBinaryColumns() {
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>();
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.BINARY).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.DOUBLE).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.BINARY).nullable(true).build());
+ return new Schema(columns);
+ }
+
+ private Schema createSchemaWithTimestampColumns() {
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>();
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.TIMESTAMP).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.TIMESTAMP).nullable(true).build());
+ return new Schema(columns);
+ }
+
+ private static CreateTableOptions createTableOptions() {
+ return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
+ }
+
+ /**
+ * Test creating and deleting a table through a KuduClient.
+ */
+ @Test(timeout = 100000)
+ public void testCreateDeleteTable() throws Exception {
+ // Check that we can create a table.
+ syncClient.createTable(tableName, basicSchema, getBasicCreateTableOptions());
+ assertFalse(syncClient.getTablesList().getTablesList().isEmpty());
+ assertTrue(syncClient.getTablesList().getTablesList().contains(tableName));
+
+ // Check that we can delete it.
+ syncClient.deleteTable(tableName);
+ assertFalse(syncClient.getTablesList().getTablesList().contains(tableName));
+
+ // Check that we can re-recreate it, with a different schema.
+ List<ColumnSchema> columns = new ArrayList<>(basicSchema.getColumns());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("one more", Type.STRING).build());
+ Schema newSchema = new Schema(columns);
+ syncClient.createTable(tableName, newSchema, getBasicCreateTableOptions());
+
+ // Check that we can open a table and see that it has the new schema.
+ KuduTable table = syncClient.openTable(tableName);
+ assertEquals(newSchema.getColumnCount(), table.getSchema().getColumnCount());
+ assertTrue(table.getPartitionSchema().isSimpleRangePartitioning());
+
+ // Check that the block size parameter we specified in the schema is respected.
+ assertEquals(4096, newSchema.getColumn("column3_s").getDesiredBlockSize());
+ assertEquals(ColumnSchema.Encoding.DICT_ENCODING,
+ newSchema.getColumn("column3_s").getEncoding());
+ assertEquals(ColumnSchema.CompressionAlgorithm.LZ4,
+ newSchema.getColumn("column3_s").getCompressionAlgorithm());
+ }
+
+ /**
+ * Test inserting and retrieving string columns.
+ */
+ @Test(timeout = 100000)
+ public void testStrings() throws Exception {
+ Schema schema = createManyStringsSchema();
+ syncClient.createTable(tableName, schema, createTableOptions());
+
+ KuduSession session = syncClient.newSession();
+ KuduTable table = syncClient.openTable(tableName);
+ for (int i = 0; i < 100; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addString("key", String.format("key_%02d", i));
+ row.addString("c2", "c2_" + i);
+ if (i % 2 == 1) {
+ row.addString("c3", "c3_" + i);
+ }
+ row.addString("c4", "c4_" + i);
+ // NOTE: we purposefully add the strings in a non-left-to-right
+ // order to verify that we still place them in the right position in
+ // the row.
+ row.addString("c1", "c1_" + i);
+ session.apply(insert);
+ if (i % 50 == 0) {
+ session.flush();
+ }
+ }
+ session.flush();
+
+ List<String> rowStrings = scanTableToStrings(table);
+ assertEquals(100, rowStrings.size());
+ assertEquals(
+ "STRING key=key_03, STRING c1=c1_3, STRING c2=c2_3, STRING c3=c3_3, STRING c4=c4_3",
+ rowStrings.get(3));
+ assertEquals(
+ "STRING key=key_04, STRING c1=c1_4, STRING c2=c2_4, STRING c3=NULL, STRING c4=c4_4",
+ rowStrings.get(4));
+
+ KuduScanner scanner = syncClient.newScannerBuilder(table).build();
+
+ assertTrue("Scanner should have returned row", scanner.hasMoreRows());
+
+ RowResultIterator rows = scanner.nextRows();
+ final RowResult next = rows.next();
+
+ // Do negative testing on string type.
+ try {
+ next.getInt("c2");
+ fail("IllegalArgumentException was not thrown when accessing " +
+ "a string column with getInt");
+ } catch (IllegalArgumentException ignored) {}
+ }
+
+ /**
+ * Test to verify that we can write in and read back UTF8.
+ */
+ @Test(timeout = 100000)
+ public void testUTF8() throws Exception {
+ Schema schema = createManyStringsSchema();
+ syncClient.createTable(tableName, schema, createTableOptions());
+
+ KuduSession session = syncClient.newSession();
+ KuduTable table = syncClient.openTable(tableName);
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addString("key", "\u0e01\u0e02\u0e03\u0e04\u0e05\u0e06\u0e07"); // some thai
+ row.addString("c1", "\u2701\u2702\u2703\u2704\u2706"); // some icons
+
+ row.addString("c2", "hello"); // some normal chars
+ row.addString("c4", "\U0001f431"); // supplemental plane
+ session.apply(insert);
+ session.flush();
+
+ List<String> rowStrings = scanTableToStrings(table);
+ assertEquals(1, rowStrings.size());
+ assertEquals(
+ "STRING key=\u0e01\u0e02\u0e03\u0e04\u0e05\u0e06\u0e07, STRING c1=\u2701\u2702\u2703\u2704\u2706, STRING c2=hello, STRING c3=NULL, STRING c4=\U0001f431",
+ rowStrings.get(0));
+ }
+
+ /**
+ * Test inserting and retrieving binary columns.
+ */
+ @Test(timeout = 100000)
+ public void testBinaryColumns() throws Exception {
+ Schema schema = createSchemaWithBinaryColumns();
+ syncClient.createTable(tableName, schema, createTableOptions());
+
+ byte[] testArray = new byte[] {1, 2, 3, 4, 5, 6 ,7, 8, 9};
+
+ KuduSession session = syncClient.newSession();
+ KuduTable table = syncClient.openTable(tableName);
+ for (int i = 0; i < 100; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addBinary("key", String.format("key_%02d", i).getBytes());
+ row.addString("c1", "\u2701\u2702\u2703\u2704\u2706");
+ row.addDouble("c2", i);
+ if (i % 2 == 1) {
+ row.addBinary("c3", testArray);
+ }
+ session.apply(insert);
+ if (i % 50 == 0) {
+ session.flush();
+ }
+ }
+ session.flush();
+
+ List<String> rowStrings = scanTableToStrings(table);
+ assertEquals(100, rowStrings.size());
+ for (int i = 0; i < rowStrings.size(); i++) {
+ StringBuilder expectedRow = new StringBuilder();
+ expectedRow.append(String.format("BINARY key=\"key_%02d\", STRING c1=\u2701\u2702\u2703\u2704\u2706, DOUBLE c2=%.1f,"
+ + " BINARY c3=", i, (double) i));
+ if (i % 2 == 1) {
+ expectedRow.append(Bytes.pretty(testArray));
+ } else {
+ expectedRow.append("NULL");
+ }
+ assertEquals(expectedRow.toString(), rowStrings.get(i));
+ }
+ }
+
+ /**
+ * Test inserting and retrieving timestamp columns.
+ */
+ @Test(timeout = 100000)
+ public void testTimestampColumns() throws Exception {
+ Schema schema = createSchemaWithTimestampColumns();
+ syncClient.createTable(tableName, schema, createTableOptions());
+
+ List<Long> timestamps = new ArrayList<>();
+
+ KuduSession session = syncClient.newSession();
+ KuduTable table = syncClient.openTable(tableName);
+ long lastTimestamp = 0;
+ for (int i = 0; i < 100; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ long timestamp = System.currentTimeMillis() * 1000;
+ while(timestamp == lastTimestamp) {
+ timestamp = System.currentTimeMillis() * 1000;
+ }
+ timestamps.add(timestamp);
+ row.addLong("key", timestamp);
+ if (i % 2 == 1) {
+ row.addLong("c1", timestamp);
+ }
+ session.apply(insert);
+ if (i % 50 == 0) {
+ session.flush();
+ }
+ lastTimestamp = timestamp;
+ }
+ session.flush();
+
+ List<String> rowStrings = scanTableToStrings(table);
+ assertEquals(100, rowStrings.size());
+ for (int i = 0; i < rowStrings.size(); i++) {
+ StringBuilder expectedRow = new StringBuilder();
+ expectedRow.append(String.format("TIMESTAMP key=%s, TIMESTAMP c1=",
+ timestampToString(timestamps.get(i))));
+ if (i % 2 == 1) {
+ expectedRow.append(timestampToString(timestamps.get(i)));
+ } else {
+ expectedRow.append("NULL");
+ }
+ assertEquals(expectedRow.toString(), rowStrings.get(i));
+ }
+ }
+
+ /**
+ * Test scanning with predicates.
+ */
+ @Test
+ public void testScanWithPredicates() throws Exception {
+ Schema schema = createManyStringsSchema();
+ syncClient.createTable(tableName, schema, createTableOptions());
+
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+ KuduTable table = syncClient.openTable(tableName);
+ for (int i = 0; i < 100; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addString("key", String.format("key_%02d", i));
+ row.addString("c1", "c1_" + i);
+ row.addString("c2", "c2_" + i);
+ session.apply(insert);
+ }
+ session.flush();
+
+ assertEquals(100, scanTableToStrings(table).size());
+ assertEquals(50, scanTableToStrings(table,
+ KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER_EQUAL, "key_50")
+ ).size());
+ assertEquals(25, scanTableToStrings(table,
+ KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER, "key_74")
+ ).size());
+ assertEquals(25, scanTableToStrings(table,
+ KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER, "key_24"),
+ KuduPredicate.newComparisonPredicate(schema.getColumn("c1"), LESS_EQUAL, "c1_49")
+ ).size());
+ assertEquals(50, scanTableToStrings(table,
+ KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER, "key_24"),
+ KuduPredicate.newComparisonPredicate(schema.getColumn("key"), GREATER_EQUAL, "key_50")
+ ).size());
+ assertEquals(0, scanTableToStrings(table,
+ KuduPredicate.newComparisonPredicate(schema.getColumn("c1"), GREATER, "c1_30"),
+ KuduPredicate.newComparisonPredicate(schema.getColumn("c2"), LESS, "c2_20")
+ ).size());
+ assertEquals(0, scanTableToStrings(table,
+ // Short circuit scan
+ KuduPredicate.newComparisonPredicate(schema.getColumn("c2"), GREATER, "c2_30"),
+ KuduPredicate.newComparisonPredicate(schema.getColumn("c2"), LESS, "c2_20")
+ ).size());
+ }
+
+ /**
+ * Tests scan tokens by creating a set of scan tokens, serializing them, and
+ * then executing them in parallel with separate client instances. This
+ * simulates the normal usecase of scan tokens being created at a central
+ * planner and distributed to remote task executors.
+ */
+ @Test
+ public void testScanTokens() throws Exception {
+ Schema schema = createManyStringsSchema();
+ CreateTableOptions createOptions = new CreateTableOptions();
+ createOptions.addHashPartitions(ImmutableList.of("key"), 8);
+
+ PartialRow splitRow = schema.newPartialRow();
+ splitRow.addString("key", "key_50");
+ createOptions.addSplitRow(splitRow);
+
+ syncClient.createTable(tableName, schema, createOptions);
+
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+ KuduTable table = syncClient.openTable(tableName);
+ for (int i = 0; i < 100; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addString("key", String.format("key_%02d", i));
+ row.addString("c1", "c1_" + i);
+ row.addString("c2", "c2_" + i);
+ session.apply(insert);
+ }
+ session.flush();
+
+ KuduScanToken.KuduScanTokenBuilder tokenBuilder = syncClient.newScanTokenBuilder(table);
+ tokenBuilder.setProjectedColumnIndexes(ImmutableList.<Integer>of());
+ List<KuduScanToken> tokens = tokenBuilder.build();
+ assertEquals(16, tokens.size());
+
+ final AtomicInteger count = new AtomicInteger(0);
+ List<Thread> threads = new ArrayList<>();
+ for (final KuduScanToken token : tokens) {
+ final byte[] serializedToken = token.serialize();
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try (KuduClient contextClient = new KuduClient.KuduClientBuilder(masterAddresses)
+ .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
+ .build()) {
+ KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, contextClient);
+ try {
+ int localCount = 0;
+ while (scanner.hasMoreRows()) {
+ localCount += Iterators.size(scanner.nextRows());
+ }
+ assertTrue(localCount > 0);
+ count.addAndGet(localCount);
+ } finally {
+ scanner.close();
+ }
+ } catch (Exception e) {
+ LOG.error("exception in parallel token scanner", e);
+ }
+ }
+ });
+ thread.run();
+ threads.add(thread);
+ }
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+ assertEquals(100, count.get());
+ }
+
+ /**
+ * Counts the rows in a table between two optional bounds.
+ * @param table the table to scan, must have the basic schema
+ * @param lowerBound an optional lower bound key
+ * @param upperBound an optional upper bound key
+ * @return the row count
+ * @throws Exception on error
+ */
+ private int countRowsForTestScanNonCoveredTable(KuduTable table,
+ Integer lowerBound,
+ Integer upperBound) throws Exception {
+
+ KuduScanner.KuduScannerBuilder scanBuilder = syncClient.newScannerBuilder(table);
+ if (lowerBound != null) {
+ PartialRow bound = basicSchema.newPartialRow();
+ bound.addInt(0, lowerBound);
+ scanBuilder.lowerBound(bound);
+ }
+ if (upperBound != null) {
+ PartialRow bound = basicSchema.newPartialRow();
+ bound.addInt(0, upperBound);
+ scanBuilder.exclusiveUpperBound(bound);
+ }
+
+ KuduScanner scanner = scanBuilder.build();
+ int count = 0;
+ while (scanner.hasMoreRows()) {
+ count += scanner.nextRows().getNumRows();
+ }
+ return count;
+ }
+
+ /**
+ * Tests scanning a table with non-covering range partitions.
+ */
+ @Test(timeout = 100000)
+ public void testScanNonCoveredTable() throws Exception {
+
+ Schema schema = basicSchema;
+ syncClient.createTable(tableName, schema, getBasicTableOptionsWithNonCoveredRange());
+
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+ KuduTable table = syncClient.openTable(tableName);
+
+ for (int key = 0; key < 100; key++) {
+ session.apply(createBasicSchemaInsert(table, key));
+ }
+ for (int key = 200; key < 300; key++) {
+ session.apply(createBasicSchemaInsert(table, key));
+ }
+ session.flush();
+ assertEquals(0, session.countPendingErrors());
+
+ assertEquals(200, countRowsForTestScanNonCoveredTable(table, null, null));
+ assertEquals(100, countRowsForTestScanNonCoveredTable(table, null, 200));
+ assertEquals(0, countRowsForTestScanNonCoveredTable(table, null, -1));
+ assertEquals(0, countRowsForTestScanNonCoveredTable(table, 120, 180));
+ assertEquals(0, countRowsForTestScanNonCoveredTable(table, 300, null));
+ }
+
+ /**
+ * Creates a local client that we auto-close while buffering one row, then makes sure that after
+ * closing that we can read the row.
+ */
+ @Test(timeout = 100000)
+ public void testAutoClose() throws Exception {
+ try (KuduClient localClient = new KuduClient.KuduClientBuilder(masterAddresses).build()) {
+ localClient.createTable(tableName, basicSchema, getBasicCreateTableOptions());
+ KuduTable table = localClient.openTable(tableName);
+ KuduSession session = localClient.newSession();
+
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ Insert insert = createBasicSchemaInsert(table, 0);
+ session.apply(insert);
+ }
+
+ KuduTable table = syncClient.openTable(tableName);
+ AsyncKuduScanner scanner = new AsyncKuduScanner.AsyncKuduScannerBuilder(client, table).build();
+ assertEquals(1, countRowsInScan(scanner));
+ }
+
+ @Test(timeout = 100000)
+ public void testCustomNioExecutor() throws Exception {
+ long startTime = System.nanoTime();
+ final KuduClient localClient = new KuduClient.KuduClientBuilder(masterAddresses)
+ .nioExecutors(Executors.newFixedThreadPool(1), Executors.newFixedThreadPool(2))
+ .bossCount(1)
+ .workerCount(2)
+ .build();
+ long buildTime = (System.nanoTime() - startTime) / 1000000000L;
+ assertTrue("Building KuduClient is slow, maybe netty get stuck", buildTime < 3);
+ localClient.createTable(tableName, basicSchema, getBasicCreateTableOptions());
+ Thread[] threads = new Thread[4];
+ for (int t = 0; t < 4; t++) {
+ final int id = t;
+ threads[t] = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ KuduTable table = localClient.openTable(tableName);
+ KuduSession session = localClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
+ for (int i = 0; i < 100; i++) {
+ Insert insert = createBasicSchemaInsert(table, id * 100 + i);
+ session.apply(insert);
+ }
+ session.close();
+ } catch (Exception e) {
+ fail("insert thread should not throw exception: " + e);
+ }
+ }
+ });
+ threads[t].start();
+ }
+ for (int t = 0; t< 4;t++) {
+ threads[t].join();
+ }
+ localClient.shutdown();
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testNoDefaultPartitioning() throws Exception {
+ syncClient.createTable(tableName, basicSchema, new CreateTableOptions());
+ }
+}
[22/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java
new file mode 100644
index 0000000..cd0a17d
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Status.java
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.WireProtocol;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.master.Master;
+import org.kududb.tserver.Tserver;
+
+/**
+ * Representation of an error code and message.
+ * See also {@code src/kudu/util/status.h} in the C++ codebase.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Status {
+
+ // Keep a single OK status object else we'll end up instantiating tons of them.
+ private static final Status STATIC_OK = new Status(WireProtocol.AppStatusPB.ErrorCode.OK);
+
+ private final WireProtocol.AppStatusPB appStatusPB;
+
+ private Status(WireProtocol.AppStatusPB appStatusPB) {
+ this.appStatusPB = appStatusPB;
+ }
+
+ private Status(WireProtocol.AppStatusPB.ErrorCode code, String msg, int posixCode) {
+ this.appStatusPB =
+ WireProtocol.AppStatusPB.newBuilder()
+ .setCode(code)
+ .setMessage(msg)
+ .setPosixCode(posixCode)
+ .build();
+ }
+
+ private Status(WireProtocol.AppStatusPB.ErrorCode code, String msg) {
+ this(code, msg, -1);
+ }
+
+ private Status(WireProtocol.AppStatusPB.ErrorCode code) {
+ this(code, "", -1);
+ }
+
+ // Factory methods.
+
+ /**
+ * Create a status object from a master error.
+ * @param masterErrorPB pb object received via RPC from the master
+ * @return status object equivalent to the pb
+ */
+ static Status fromMasterErrorPB(Master.MasterErrorPB masterErrorPB) {
+ if (masterErrorPB == Master.MasterErrorPB.getDefaultInstance()) {
+ return Status.OK();
+ } else {
+ return new Status(masterErrorPB.getStatus());
+ }
+ }
+
+ /**
+ * Create a status object from a tablet server error.
+ * @param tserverErrorPB pb object received via RPC from the TS
+ * @return status object equivalent to the pb
+ */
+ static Status fromTabletServerErrorPB(Tserver.TabletServerErrorPB tserverErrorPB) {
+ if (tserverErrorPB == Tserver.TabletServerErrorPB.getDefaultInstance()) {
+ return Status.OK();
+ } else {
+ return new Status(tserverErrorPB.getStatus());
+ }
+ }
+
+ /**
+ * Create a Status object from a {@link WireProtocol.AppStatusPB} protobuf object.
+ * Package-private because we shade Protobuf and this is not usable outside this package.
+ */
+ static Status fromPB(WireProtocol.AppStatusPB pb) {
+ return new Status(pb);
+ }
+
+ public static Status OK() {
+ return STATIC_OK;
+ }
+
+ public static Status NotFound(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND, msg);
+ }
+ public static Status NotFound(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND, msg, posixCode);
+ }
+
+ public static Status Corruption(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.CORRUPTION, msg);
+ }
+ public static Status Corruption(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.CORRUPTION, msg, posixCode);
+ }
+
+ public static Status NotSupported(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED, msg);
+ }
+ public static Status NotSupported(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED, msg, posixCode);
+ }
+
+ public static Status InvalidArgument(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT, msg);
+ }
+ public static Status InvalidArgument(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT, msg, posixCode);
+ }
+
+ public static Status IOError(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.IO_ERROR, msg);
+ }
+ public static Status IOError(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.IO_ERROR, msg, posixCode);
+ }
+
+ public static Status AlreadyPresent(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT, msg);
+ }
+ public static Status AlreadyPresent(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT, msg, posixCode);
+ }
+
+ public static Status RuntimeError(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR, msg);
+ }
+ public static Status RuntimeError(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR, msg, posixCode);
+ }
+
+ public static Status NetworkError(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR, msg);
+ }
+ public static Status NetworkError(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR, msg, posixCode);
+ }
+
+ public static Status IllegalState(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE, msg);
+ }
+ public static Status IllegalState(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE, msg, posixCode);
+ }
+
+ public static Status NotAuthorized(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED, msg);
+ }
+ public static Status NotAuthorized(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED, msg, posixCode);
+ }
+
+ public static Status Aborted(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.ABORTED, msg);
+ }
+ public static Status Aborted(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.ABORTED, msg, posixCode);
+ }
+
+ public static Status RemoteError(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR, msg);
+ }
+ public static Status RemoteError(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR, msg, posixCode);
+ }
+
+ public static Status ServiceUnavailable(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE, msg);
+ }
+ public static Status ServiceUnavailable(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE, msg, posixCode);
+ }
+
+ public static Status TimedOut(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT, msg);
+ }
+ public static Status TimedOut(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT, msg, posixCode);
+ }
+
+ public static Status Uninitialized(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED, msg);
+ }
+ public static Status Uninitialized(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED, msg, posixCode);
+ }
+
+ public static Status ConfigurationError(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR, msg);
+ }
+ public static Status ConfigurationError(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR, msg, posixCode);
+ }
+
+ public static Status Incomplete(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE, msg);
+ }
+ public static Status Incomplete(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE, msg, posixCode);
+ }
+
+ public static Status EndOfFile(String msg) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE, msg);
+ }
+ public static Status EndOfFile(String msg, int posixCode) {
+ return new Status(WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE, msg, posixCode);
+ }
+
+ // Boolean status checks.
+
+ public boolean ok() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.OK;
+ }
+ public boolean isCorruption() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.CORRUPTION;
+ }
+ public boolean isNotFound() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND;
+ }
+ public boolean isNotSupported() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED;
+ }
+ public boolean isInvalidArgument() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT;
+ }
+ public boolean isIOError() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.IO_ERROR;
+ }
+ public boolean isAlreadyPresent() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT;
+ }
+ public boolean isRuntimeError() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR;
+ }
+ public boolean isNetworkError() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR;
+ }
+ public boolean isIllegalState() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE;
+ }
+ public boolean isNotAuthorized() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED;
+ }
+ public boolean isAborted() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.ABORTED;
+ }
+ public boolean isRemoteError() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR;
+ }
+ public boolean isServiceUnavailable() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE;
+ }
+ public boolean isTimedOut() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT;
+ }
+ public boolean isUninitialized() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED;
+ }
+ public boolean isConfigurationError() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR;
+ }
+ public boolean isIncomplete() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE;
+ }
+ public boolean isEndOfFile() {
+ return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE;
+ }
+
+ /**
+ * Return a human-readable version of the status code.
+ * See also status.cc in the C++ codebase.
+ */
+ private String getCodeAsString() {
+ switch (appStatusPB.getCode().getNumber()) {
+ case WireProtocol.AppStatusPB.ErrorCode.OK_VALUE:
+ return "OK";
+ case WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND_VALUE:
+ return "Not found";
+ case WireProtocol.AppStatusPB.ErrorCode.CORRUPTION_VALUE:
+ return "Corruption";
+ case WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED_VALUE:
+ return "Not implemented";
+ case WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT_VALUE:
+ return "Invalid argument";
+ case WireProtocol.AppStatusPB.ErrorCode.IO_ERROR_VALUE:
+ return "IO error";
+ case WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT_VALUE:
+ return "Already present";
+ case WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR_VALUE:
+ return "Runtime error";
+ case WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR_VALUE:
+ return "Network error";
+ case WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE_VALUE:
+ return "Illegal state";
+ case WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED_VALUE:
+ return "Not authorized";
+ case WireProtocol.AppStatusPB.ErrorCode.ABORTED_VALUE:
+ return "Aborted";
+ case WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR_VALUE:
+ return "Remote error";
+ case WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE_VALUE:
+ return "Service unavailable";
+ case WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT_VALUE:
+ return "Timed out";
+ case WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED_VALUE:
+ return "Uninitialized";
+ case WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR_VALUE:
+ return "Configuration error";
+ case WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE_VALUE:
+ return "Incomplete";
+ case WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE_VALUE:
+ return "End of file";
+ default:
+ return "Unknown error (" + appStatusPB.getCode().getNumber() + ")";
+ }
+ }
+
+ /**
+ * Get the posix code associated with the error.
+ * @return {@code -1} if no posix code is set. Otherwise, returns the posix code.
+ */
+ public int getPosixCode() {
+ return appStatusPB.getPosixCode();
+ }
+
+ /**
+ * Get enum code name.
+ * Intended for internal use only.
+ */
+ String getCodeName() {
+ return appStatusPB.getCode().name();
+ }
+
+ /**
+ * Returns string error message.
+ * Intended for internal use only.
+ */
+ String getMessage() {
+ return appStatusPB.getMessage();
+ }
+
+ /**
+ * Get a human-readable version of the Status message fit for logging or display.
+ */
+ @Override
+ public String toString() {
+ String str = getCodeAsString();
+ if (appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.OK) {
+ return str;
+ }
+ str = String.format("%s: %s", str, appStatusPB.getMessage());
+ if (appStatusPB.getPosixCode() != -1) {
+ str = String.format("%s (error %d)", str, appStatusPB.getPosixCode());
+ }
+ return str;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java
new file mode 100644
index 0000000..dafb6fc
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/TabletClient.java
@@ -0,0 +1,879 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.kududb.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.stumbleupon.async.Deferred;
+
+import org.jboss.netty.handler.timeout.ReadTimeoutException;
+import org.kududb.WireProtocol;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.master.Master;
+import org.kududb.rpc.RpcHeader;
+import org.kududb.tserver.Tserver;
+import org.kududb.util.Pair;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelEvent;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.handler.codec.replay.ReplayingDecoder;
+import org.jboss.netty.handler.codec.replay.VoidEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.sasl.SaslException;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Stateful handler that manages a connection to a specific TabletServer.
+ * <p>
+ * This handler manages the RPC IDs, the serialization and de-serialization of
+ * RPC requests and responses, and keeps track of the RPC in flights for which
+ * a response is currently awaited, as well as temporarily buffered RPCs that
+ * are awaiting to be sent to the network.
+ * <p>
+ * This class needs careful synchronization. It's a non-sharable handler,
+ * meaning there is one instance of it per Netty {@link Channel} and each
+ * instance is only used by one Netty IO thread at a time. At the same time,
+ * {@link AsyncKuduClient} calls methods of this class from random threads at
+ * random times. The bottom line is that any data only used in the Netty IO
+ * threads doesn't require synchronization, everything else does.
+ * <p>
+ * Acquiring the monitor on an object of this class will prevent it from
+ * accepting write requests as well as buffering requests if the underlying
+ * channel isn't connected.
+ */
+@InterfaceAudience.Private
+public class TabletClient extends ReplayingDecoder<VoidEnum> {
+
+ public static final Logger LOG = LoggerFactory.getLogger(TabletClient.class);
+
+ private ArrayList<KuduRpc<?>> pending_rpcs;
+
+ public static final byte RPC_CURRENT_VERSION = 9;
+ /** Initial part of the header for 0.95 and up. */
+ private static final byte[] RPC_HEADER = new byte[] { 'h', 'r', 'p', 'c',
+ RPC_CURRENT_VERSION, // RPC version.
+ 0,
+ 0
+ };
+ public static final int CONNECTION_CTX_CALL_ID = -3;
+
+ /**
+ * A monotonically increasing counter for RPC IDs.
+ * RPCs can be sent out from any thread, so we need an atomic integer.
+ * RPC IDs can be arbitrary. So it's fine if this integer wraps around and
+ * becomes negative. They don't even have to start at 0, but we do it for
+ * simplicity and ease of debugging.
+ */
+ private final AtomicInteger rpcid = new AtomicInteger(-1);
+
+ /**
+ * The channel we're connected to.
+ * This will be {@code null} while we're not connected to the TabletServer.
+ * This attribute is volatile because {@link #shutdown} may access it from a
+ * different thread, and because while we connect various user threads will
+ * test whether it's {@code null}. Once we're connected and we know what
+ * protocol version the server speaks, we'll set this reference.
+ */
+ private volatile Channel chan;
+
+ /**
+ * Set to {@code true} once we've disconnected from the server.
+ * This way, if any thread is still trying to use this client after it's
+ * been removed from the caches in the {@link AsyncKuduClient}, we will
+ * immediately fail / reschedule its requests.
+ * <p>
+ * Manipulating this value requires synchronizing on `this'.
+ */
+ private boolean dead = false;
+
+ /**
+ * Maps an RPC ID to the in-flight RPC that was given this ID.
+ * RPCs can be sent out from any thread, so we need a concurrent map.
+ */
+ private final ConcurrentHashMap<Integer, KuduRpc<?>> rpcs_inflight = new ConcurrentHashMap<>();
+
+ private final AsyncKuduClient kuduClient;
+
+ private final String uuid;
+
+ private final String host;
+
+ private final int port;
+
+ private final long socketReadTimeoutMs;
+
+ private SecureRpcHelper secureRpcHelper;
+
+ private final RequestTracker requestTracker;
+
+ public TabletClient(AsyncKuduClient client, String uuid, String host, int port) {
+ this.kuduClient = client;
+ this.uuid = uuid;
+ this.socketReadTimeoutMs = client.getDefaultSocketReadTimeoutMs();
+ this.host = host;
+ this.port = port;
+ this.requestTracker = client.getRequestTracker();
+ }
+
+ <R> void sendRpc(KuduRpc<R> rpc) {
+ if (!rpc.deadlineTracker.hasDeadline()) {
+ LOG.warn(getPeerUuidLoggingString() + " sending an rpc without a timeout " + rpc);
+ }
+ Pair<ChannelBuffer, Integer> encodedRpcAndId = null;
+ if (chan != null) {
+ if (!rpc.getRequiredFeatures().isEmpty() &&
+ !secureRpcHelper.getServerFeatures().contains(
+ RpcHeader.RpcFeatureFlag.APPLICATION_FEATURE_FLAGS)) {
+ Status statusNotSupported = Status.NotSupported("the server does not support the" +
+ "APPLICATION_FEATURE_FLAGS RPC feature");
+ rpc.errback(new NonRecoverableException(statusNotSupported));
+ }
+
+ encodedRpcAndId = encode(rpc);
+ if (encodedRpcAndId == null) { // Error during encoding.
+ return; // Stop here. RPC has been failed already.
+ }
+
+ final Channel chan = this.chan; // Volatile read.
+ if (chan != null) { // Double check if we disconnected during encode().
+ Channels.write(chan, encodedRpcAndId.getFirst());
+ return;
+ }
+ }
+ boolean tryAgain = false; // True when we notice we are about to get connected to the TS.
+ boolean failRpc = false; // True when the connection was closed while encoding.
+ synchronized (this) {
+ // Check if we got connected while entering this synchronized block.
+ if (chan != null) {
+ tryAgain = true;
+ // Check if we got disconnected.
+ } else if (dead) {
+ // We got disconnected during the process of encoding this rpc, but we need to check if
+ // cleanup() already took care of calling failOrRetryRpc() for us. If it did, the entry we
+ // added in rpcs_inflight will be missing. If not, we have to call failOrRetryRpc()
+ // ourselves after this synchronized block.
+ // `encodedRpcAndId` is null iff `chan` is null.
+ if (encodedRpcAndId == null || rpcs_inflight.containsKey(encodedRpcAndId.getSecond())) {
+ failRpc = true;
+ }
+ } else {
+ if (pending_rpcs == null) {
+ pending_rpcs = new ArrayList<>();
+ }
+ pending_rpcs.add(rpc);
+ }
+ }
+
+ if (failRpc) {
+ Status statusNetworkError =
+ Status.NetworkError(getPeerUuidLoggingString() + "Connection reset on " + chan);
+ failOrRetryRpc(rpc, new RecoverableException(statusNetworkError));
+ } else if (tryAgain) {
+ // This recursion will not lead to a loop because we only get here if we
+ // connected while entering the synchronized block above. So when trying
+ // a second time, we will either succeed to send the RPC if we're still
+ // connected, or fail through to the code below if we got disconnected
+ // in the mean time.
+ sendRpc(rpc);
+ }
+ }
+
+ private <R> Pair<ChannelBuffer, Integer> encode(final KuduRpc<R> rpc) {
+ final int rpcid = this.rpcid.incrementAndGet();
+ ChannelBuffer payload;
+ final String service = rpc.serviceName();
+ final String method = rpc.method();
+ try {
+ final RpcHeader.RequestHeader.Builder headerBuilder = RpcHeader.RequestHeader.newBuilder()
+ .setCallId(rpcid)
+ .addAllRequiredFeatureFlags(rpc.getRequiredFeatures())
+ .setRemoteMethod(
+ RpcHeader.RemoteMethodPB.newBuilder().setServiceName(service).setMethodName(method));
+
+ // If any timeout is set, find the lowest non-zero one, since this will be the deadline that
+ // the server must respect.
+ if (rpc.deadlineTracker.hasDeadline() || socketReadTimeoutMs > 0) {
+ long millisBeforeDeadline = Long.MAX_VALUE;
+ if (rpc.deadlineTracker.hasDeadline()) {
+ millisBeforeDeadline = rpc.deadlineTracker.getMillisBeforeDeadline();
+ }
+
+ long localRpcTimeoutMs = Long.MAX_VALUE;
+ if (socketReadTimeoutMs > 0) {
+ localRpcTimeoutMs = socketReadTimeoutMs;
+ }
+
+ headerBuilder.setTimeoutMillis((int) Math.min(millisBeforeDeadline, localRpcTimeoutMs));
+ }
+
+ if (rpc.isRequestTracked()) {
+ RpcHeader.RequestIdPB.Builder requestIdBuilder = RpcHeader.RequestIdPB.newBuilder();
+ if (rpc.getSequenceId() == RequestTracker.NO_SEQ_NO) {
+ rpc.setSequenceId(requestTracker.newSeqNo());
+ }
+ requestIdBuilder.setClientId(requestTracker.getClientId());
+ requestIdBuilder.setSeqNo(rpc.getSequenceId());
+ requestIdBuilder.setAttemptNo(rpc.attempt);
+ requestIdBuilder.setFirstIncompleteSeqNo(requestTracker.firstIncomplete());
+ headerBuilder.setRequestId(requestIdBuilder);
+ }
+
+ payload = rpc.serialize(headerBuilder.build());
+ } catch (Exception e) {
+ LOG.error("Uncaught exception while serializing RPC: " + rpc, e);
+ rpc.errback(e); // Make the RPC fail with the exception.
+ return null;
+ }
+ final KuduRpc<?> oldrpc = rpcs_inflight.put(rpcid, rpc);
+ if (oldrpc != null) {
+ final String wtf = getPeerUuidLoggingString() +
+ "WTF? There was already an RPC in flight with"
+ + " rpcid=" + rpcid + ": " + oldrpc
+ + ". This happened when sending out: " + rpc;
+ LOG.error(wtf);
+ Status statusIllegalState = Status.IllegalState(wtf);
+ // Make it fail. This isn't an expected failure mode.
+ oldrpc.errback(new NonRecoverableException(statusIllegalState));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(getPeerUuidLoggingString() + chan + " Sending RPC #" + rpcid
+ + ", payload=" + payload + ' ' + Bytes.pretty(payload));
+ }
+
+ payload = secureRpcHelper.wrap(payload);
+
+ return new Pair<>(payload, rpcid);
+ }
+
+ /**
+ * Quick and dirty way to close a connection to a tablet server, if it wasn't already closed.
+ */
+ @VisibleForTesting
+ void disconnect() {
+ Channel chancopy = chan;
+ if (chancopy != null && chancopy.isConnected()) {
+ Channels.disconnect(chancopy);
+ }
+ }
+
+ /**
+ * Forcefully shuts down the connection to this tablet server and fails all the outstanding RPCs.
+ * Only use when shutting down a client.
+ * @return deferred object to use to track the shutting down of this connection
+ */
+ public Deferred<Void> shutdown() {
+ Status statusNetworkError =
+ Status.NetworkError(getPeerUuidLoggingString() + "Client is shutting down");
+ NonRecoverableException exception = new NonRecoverableException(statusNetworkError);
+ // First, check whether we have RPCs in flight and cancel them.
+ for (Iterator<KuduRpc<?>> ite = rpcs_inflight.values().iterator(); ite
+ .hasNext();) {
+ ite.next().errback(exception);
+ ite.remove();
+ }
+
+ // Same for the pending RPCs.
+ synchronized (this) {
+ if (pending_rpcs != null) {
+ for (Iterator<KuduRpc<?>> ite = pending_rpcs.iterator(); ite.hasNext();) {
+ ite.next().errback(exception);
+ ite.remove();
+ }
+ }
+ }
+
+ final Channel chancopy = chan;
+ if (chancopy == null) {
+ return Deferred.fromResult(null);
+ }
+ if (chancopy.isConnected()) {
+ Channels.disconnect(chancopy); // ... this is going to set it to null.
+ // At this point, all in-flight RPCs are going to be failed.
+ }
+ if (chancopy.isBound()) {
+ Channels.unbind(chancopy);
+ }
+ // It's OK to call close() on a Channel if it's already closed.
+ final ChannelFuture future = Channels.close(chancopy);
+ // Now wrap the ChannelFuture in a Deferred.
+ final Deferred<Void> d = new Deferred<Void>();
+ // Opportunistically check if it's already completed successfully.
+ if (future.isSuccess()) {
+ d.callback(null);
+ } else {
+ // If we get here, either the future failed (yeah, that sounds weird)
+ // or the future hasn't completed yet (heh).
+ future.addListener(new ChannelFutureListener() {
+ public void operationComplete(final ChannelFuture future) {
+ if (future.isSuccess()) {
+ d.callback(null);
+ return;
+ }
+ final Throwable t = future.getCause();
+ if (t instanceof Exception) {
+ d.callback(t);
+ } else {
+ // Wrap the Throwable because Deferred doesn't handle Throwables,
+ // it only uses Exception.
+ Status statusIllegalState = Status.IllegalState("Failed to shutdown: " +
+ TabletClient.this);
+ d.callback(new NonRecoverableException(statusIllegalState, t));
+ }
+ }
+ });
+ }
+ return d;
+ }
+
+ /**
+ * The reason we are suppressing the unchecked conversions is because the KuduRpc is coming
+ * from a collection that has RPCs with different generics, and there's no way to get "decoded"
+ * casted correctly. The best we can do is to rely on the RPC to decode correctly,
+ * and to not pass an Exception in the callback.
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ protected Object decode(ChannelHandlerContext ctx, Channel chan, ChannelBuffer buf,
+ VoidEnum voidEnum) throws NonRecoverableException {
+ final long start = System.nanoTime();
+ final int rdx = buf.readerIndex();
+ LOG.debug("------------------>> ENTERING DECODE >>------------------");
+
+ try {
+ buf = secureRpcHelper.handleResponse(buf, chan);
+ } catch (SaslException e) {
+ String message = getPeerUuidLoggingString() + "Couldn't complete the SASL handshake";
+ LOG.error(message);
+ Status statusIOE = Status.IOError(message);
+ throw new NonRecoverableException(statusIOE, e);
+ }
+ if (buf == null) {
+ return null;
+ }
+
+ CallResponse response = new CallResponse(buf);
+
+ RpcHeader.ResponseHeader header = response.getHeader();
+ if (!header.hasCallId()) {
+ final int size = response.getTotalResponseSize();
+ final String msg = getPeerUuidLoggingString() + "RPC response (size: " + size + ") doesn't"
+ + " have a call ID: " + header + ", buf=" + Bytes.pretty(buf);
+ LOG.error(msg);
+ Status statusIncomplete = Status.Incomplete(msg);
+ throw new NonRecoverableException(statusIncomplete);
+ }
+ final int rpcid = header.getCallId();
+
+ @SuppressWarnings("rawtypes")
+ final KuduRpc rpc = rpcs_inflight.get(rpcid);
+
+ if (rpc == null) {
+ final String msg = getPeerUuidLoggingString() + "Invalid rpcid: " + rpcid + " found in "
+ + buf + '=' + Bytes.pretty(buf);
+ LOG.error(msg);
+ Status statusIllegalState = Status.IllegalState(msg);
+ // The problem here is that we don't know which Deferred corresponds to
+ // this RPC, since we don't have a valid ID. So we're hopeless, we'll
+ // never be able to recover because responses are not framed, we don't
+ // know where the next response will start... We have to give up here
+ // and throw this outside of our Netty handler, so Netty will call our
+ // exception handler where we'll close this channel, which will cause
+ // all RPCs in flight to be failed.
+ throw new NonRecoverableException(statusIllegalState);
+ }
+
+ Pair<Object, Object> decoded = null;
+ Exception exception = null;
+ Status retryableHeaderError = Status.OK();
+ if (header.hasIsError() && header.getIsError()) {
+ RpcHeader.ErrorStatusPB.Builder errorBuilder = RpcHeader.ErrorStatusPB.newBuilder();
+ KuduRpc.readProtobuf(response.getPBMessage(), errorBuilder);
+ RpcHeader.ErrorStatusPB error = errorBuilder.build();
+ if (error.getCode().equals(RpcHeader.ErrorStatusPB.RpcErrorCodePB.ERROR_SERVER_TOO_BUSY)) {
+ // We can't return right away, we still need to remove ourselves from 'rpcs_inflight', so we
+ // populate 'retryableHeaderError'.
+ retryableHeaderError = Status.ServiceUnavailable(error.getMessage());
+ } else {
+ String message = getPeerUuidLoggingString() +
+ "Tablet server sent error " + error.getMessage();
+ Status status = Status.RemoteError(message);
+ exception = new NonRecoverableException(status);
+ LOG.error(message); // can be useful
+ }
+ } else {
+ try {
+ decoded = rpc.deserialize(response, this.uuid);
+ } catch (Exception ex) {
+ exception = ex;
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(getPeerUuidLoggingString() + "rpcid=" + rpcid
+ + ", response size=" + (buf.readerIndex() - rdx) + " bytes"
+ + ", " + actualReadableBytes() + " readable bytes left"
+ + ", rpc=" + rpc);
+ }
+
+ {
+ final KuduRpc<?> removed = rpcs_inflight.remove(rpcid);
+ if (removed == null) {
+ // The RPC we were decoding was cleaned up already, give up.
+ Status statusIllegalState = Status.IllegalState("RPC not found");
+ throw new NonRecoverableException(statusIllegalState);
+ }
+ }
+
+ // This check is specifically for the ERROR_SERVER_TOO_BUSY case above.
+ if (!retryableHeaderError.ok()) {
+ kuduClient.handleRetryableError(rpc, new RecoverableException(retryableHeaderError));
+ return null;
+ }
+
+ // We can get this Message from within the RPC's expected type,
+ // so convert it into an exception and nullify decoded so that we use the errback route.
+ // Have to do it for both TS and Master errors.
+ if (decoded != null) {
+ if (decoded.getSecond() instanceof Tserver.TabletServerErrorPB) {
+ Tserver.TabletServerErrorPB error = (Tserver.TabletServerErrorPB) decoded.getSecond();
+ exception = dispatchTSErrorOrReturnException(rpc, error);
+ if (exception == null) {
+ // It was taken care of.
+ return null;
+ } else {
+ // We're going to errback.
+ decoded = null;
+ }
+
+ } else if (decoded.getSecond() instanceof Master.MasterErrorPB) {
+ Master.MasterErrorPB error = (Master.MasterErrorPB) decoded.getSecond();
+ exception = dispatchMasterErrorOrReturnException(rpc, error);
+ if (exception == null) {
+ // Exception was taken care of.
+ return null;
+ } else {
+ decoded = null;
+ }
+ }
+ }
+
+ try {
+ if (decoded != null) {
+ assert !(decoded.getFirst() instanceof Exception);
+ if (kuduClient.isStatisticsEnabled()) {
+ rpc.updateStatistics(kuduClient.getStatistics(), decoded.getFirst());
+ }
+ rpc.callback(decoded.getFirst());
+ } else {
+ if (kuduClient.isStatisticsEnabled()) {
+ rpc.updateStatistics(kuduClient.getStatistics(), null);
+ }
+ rpc.errback(exception);
+ }
+ } catch (Exception e) {
+ LOG.debug(getPeerUuidLoggingString() + "Unexpected exception while handling RPC #" + rpcid
+ + ", rpc=" + rpc + ", buf=" + Bytes.pretty(buf), e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("------------------<< LEAVING DECODE <<------------------"
+ + " time elapsed: " + ((System.nanoTime() - start) / 1000) + "us");
+ }
+ return null; // Stop processing here. The Deferred does everything else.
+ }
+
+ /**
+ * Takes care of a few kinds of TS errors that we handle differently, like tablets or leaders
+ * moving. Builds and returns an exception if we don't know what to do with it.
+ * @param rpc The original RPC call that triggered the error.
+ * @param error The error the TS sent.
+ * @return An exception if we couldn't dispatch the error, or null.
+ */
+ private Exception dispatchTSErrorOrReturnException(KuduRpc rpc,
+ Tserver.TabletServerErrorPB error) {
+ WireProtocol.AppStatusPB.ErrorCode code = error.getStatus().getCode();
+ Status status = Status.fromTabletServerErrorPB(error);
+ if (error.getCode() == Tserver.TabletServerErrorPB.Code.TABLET_NOT_FOUND) {
+ kuduClient.handleTabletNotFound(rpc, new RecoverableException(status), this);
+ // we're not calling rpc.callback() so we rely on the client to retry that RPC
+ } else if (code == WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE) {
+ kuduClient.handleRetryableError(rpc, new RecoverableException(status));
+ // The following two error codes are an indication that the tablet isn't a leader.
+ } else if (code == WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE ||
+ code == WireProtocol.AppStatusPB.ErrorCode.ABORTED) {
+ kuduClient.handleNotLeader(rpc, new RecoverableException(status), this);
+ } else {
+ return new NonRecoverableException(status);
+ }
+ return null;
+ }
+
+ /**
+ * Provides different handling for various kinds of master errors: re-uses the
+ * mechanisms already in place for handling tablet server errors as much as possible.
+ * @param rpc The original RPC call that triggered the error.
+ * @param error The error the master sent.
+ * @return An exception if we couldn't dispatch the error, or null.
+ */
+ private Exception dispatchMasterErrorOrReturnException(KuduRpc rpc,
+ Master.MasterErrorPB error) {
+ WireProtocol.AppStatusPB.ErrorCode code = error.getStatus().getCode();
+ Status status = Status.fromMasterErrorPB(error);
+ if (error.getCode() == Master.MasterErrorPB.Code.NOT_THE_LEADER) {
+ kuduClient.handleNotLeader(rpc, new RecoverableException(status), this);
+ } else if (code == WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE) {
+ if (rpc instanceof GetMasterRegistrationRequest) {
+ // Special case:
+ // We never want to retry this RPC, we only use it to poke masters to learn where the leader
+ // is. If the error is truly non recoverable, it'll be handled later.
+ return new RecoverableException(status);
+ } else {
+ // TODO: This is a crutch until we either don't have to retry RPCs going to the
+ // same server or use retry policies.
+ kuduClient.handleRetryableError(rpc, new RecoverableException(status));
+ }
+ } else {
+ return new NonRecoverableException(status);
+ }
+ return null;
+ }
+
+ /**
+ * Decodes the response of an RPC and triggers its {@link Deferred}.
+ * <p>
+ * This method is used by FrameDecoder when the channel gets
+ * disconnected. The buffer for that channel is passed to this method in
+ * case there's anything left in it.
+ * @param ctx Unused.
+ * @param chan The channel on which the response came.
+ * @param buf The buffer containing the raw RPC response.
+ * @return {@code null}, always.
+ */
+ @Override
+ protected Object decodeLast(final ChannelHandlerContext ctx,
+ final Channel chan,
+ final ChannelBuffer buf,
+ final VoidEnum unused) throws NonRecoverableException {
+ // When we disconnect, decodeLast is called instead of decode.
+ // We simply check whether there's any data left in the buffer, in which
+ // case we attempt to process it. But if there's no data left, then we
+ // don't even bother calling decode() as it'll complain that the buffer
+ // doesn't contain enough data, which unnecessarily pollutes the logs.
+ if (buf.readable()) {
+ try {
+ return decode(ctx, chan, buf, unused);
+ } finally {
+ if (buf.readable()) {
+ LOG.error(getPeerUuidLoggingString() + "After decoding the last message on " + chan
+ + ", there was still some undecoded bytes in the channel's"
+ + " buffer (which are going to be lost): "
+ + buf + '=' + Bytes.pretty(buf));
+ }
+ }
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Tells whether or not this handler should be used.
+ * <p>
+ * This method is not synchronized. You need to synchronize on this
+ * instance if you need a memory visibility guarantee. You may not need
+ * this guarantee if you're OK with the RPC finding out that the connection
+ * has been reset "the hard way" and you can retry the RPC. In this case,
+ * you can call this method as a hint. After getting the initial exception
+ * back, this thread is guaranteed to see this method return {@code false}
+ * without synchronization needed.
+ * @return {@code false} if this handler is known to have been disconnected
+ * from the server and sending an RPC (via {@link #sendRpc} or any other
+ * indirect means such as {@code GetTableLocations()}) will fail immediately
+ * by having the RPC's {@link Deferred} called back immediately with a
+ * {@link RecoverableException}. This typically means that you got a
+ * stale reference (or that the reference to this instance is just about to
+ * be invalidated) and that you shouldn't use this instance.
+ */
+ public boolean isAlive() {
+ return !dead;
+ }
+
+ /**
+ * Ensures that at least a {@code nbytes} are readable from the given buffer.
+ * If there aren't enough bytes in the buffer this will raise an exception
+ * and cause the {@link ReplayingDecoder} to undo whatever we did thus far
+ * so we can wait until we read more from the socket.
+ * @param buf Buffer to check.
+ * @param nbytes Number of bytes desired.
+ */
+ static void ensureReadable(final ChannelBuffer buf, final int nbytes) {
+ buf.markReaderIndex();
+ buf.skipBytes(nbytes); // can puke with Throwable
+ buf.resetReaderIndex();
+ }
+
+ @Override
+ public void channelConnected(final ChannelHandlerContext ctx,
+ final ChannelStateEvent e) {
+ final Channel chan = e.getChannel();
+ ChannelBuffer header = connectionHeaderPreamble();
+ header.writerIndex(RPC_HEADER.length);
+ Channels.write(chan, header);
+
+ secureRpcHelper = new SecureRpcHelper(this);
+ secureRpcHelper.sendHello(chan);
+ }
+
+ @Override
+ public void handleUpstream(final ChannelHandlerContext ctx,
+ final ChannelEvent e) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(getPeerUuidLoggingString() + e.toString());
+ }
+ super.handleUpstream(ctx, e);
+ }
+
+ @Override
+ public void channelDisconnected(final ChannelHandlerContext ctx,
+ final ChannelStateEvent e) throws Exception {
+ chan = null;
+ super.channelDisconnected(ctx, e); // Let the ReplayingDecoder cleanup.
+ cleanup(e.getChannel());
+ }
+
+ @Override
+ public void channelClosed(final ChannelHandlerContext ctx,
+ final ChannelStateEvent e) {
+ chan = null;
+ // No need to call super.channelClosed() because we already called
+ // super.channelDisconnected(). If we get here without getting a
+ // DISCONNECTED event, then we were never connected in the first place so
+ // the ReplayingDecoder has nothing to cleanup.
+ cleanup(e.getChannel());
+ }
+
+ /**
+ * Cleans up any outstanding or lingering RPC (used when shutting down).
+ * <p>
+ * All RPCs in flight will fail with a {@link RecoverableException} and
+ * all edits buffered will be re-scheduled.
+ */
+ private void cleanup(final Channel chan) {
+ final ArrayList<KuduRpc<?>> rpcs;
+
+ // The timing of this block is critical. If this TabletClient is 'dead' then it means that
+ // rpcs_inflight was emptied and that anything added to it after won't be handled and needs
+ // to be sent to failOrRetryRpc.
+ synchronized (this) {
+ // Cleanup can be called multiple times, but we only want to run it once so that we don't
+ // clear up rpcs_inflight multiple times.
+ if (dead) {
+ return;
+ }
+ dead = true;
+ rpcs = pending_rpcs == null ? new ArrayList<KuduRpc<?>>(rpcs_inflight.size()) : pending_rpcs;
+
+ for (Iterator<KuduRpc<?>> iterator = rpcs_inflight.values().iterator(); iterator.hasNext();) {
+ KuduRpc<?> rpc = iterator.next();
+ rpcs.add(rpc);
+ iterator.remove();
+ }
+ // After this, rpcs_inflight might still have entries since they could have been added
+ // concurrently, and those RPCs will be handled by their caller in sendRpc.
+
+ pending_rpcs = null;
+ }
+ Status statusNetworkError =
+ Status.NetworkError(getPeerUuidLoggingString() + "Connection reset on " + chan);
+ RecoverableException exception = new RecoverableException(statusNetworkError);
+
+ failOrRetryRpcs(rpcs, exception);
+ }
+
+ /**
+ * Retry all the given RPCs.
+ * @param rpcs a possibly empty but non-{@code null} collection of RPCs to retry or fail
+ * @param exception an exception to propagate with the RPCs
+ */
+ private void failOrRetryRpcs(final Collection<KuduRpc<?>> rpcs,
+ final RecoverableException exception) {
+ for (final KuduRpc<?> rpc : rpcs) {
+ failOrRetryRpc(rpc, exception);
+ }
+ }
+
+ /**
+ * Retry the given RPC.
+ * @param rpc an RPC to retry or fail
+ * @param exception an exception to propagate with the RPC
+ */
+ private void failOrRetryRpc(final KuduRpc<?> rpc,
+ final RecoverableException exception) {
+ AsyncKuduClient.RemoteTablet tablet = rpc.getTablet();
+ // Note As of the time of writing (03/11/16), a null tablet doesn't make sense, if we see a null
+ // tablet it's because we didn't set it properly before calling sendRpc().
+ if (tablet == null) { // Can't retry, dunno where this RPC should go.
+ rpc.errback(exception);
+ } else {
+ kuduClient.handleRetryableError(rpc, exception);
+ }
+ }
+
+
+ @Override
+ public void exceptionCaught(final ChannelHandlerContext ctx,
+ final ExceptionEvent event) {
+ final Throwable e = event.getCause();
+ final Channel c = event.getChannel();
+
+ if (e instanceof RejectedExecutionException) {
+ LOG.warn(getPeerUuidLoggingString() + "RPC rejected by the executor,"
+ + " ignore this if we're shutting down", e);
+ } else if (e instanceof ReadTimeoutException) {
+ LOG.debug(getPeerUuidLoggingString() + "Encountered a read timeout, will close the channel");
+ } else {
+ LOG.error(getPeerUuidLoggingString() + "Unexpected exception from downstream on " + c, e);
+ // For any other exception, likely a connection error, we clear the leader state
+ // for those tablets that this TS is the cached leader of.
+ kuduClient.demoteAsLeaderForAllTablets(this);
+ }
+ if (c.isOpen()) {
+ Channels.close(c); // Will trigger channelClosed(), which will cleanup()
+ } else { // else: presumably a connection timeout.
+ cleanup(c); // => need to cleanup() from here directly.
+ }
+ }
+
+
+ private ChannelBuffer connectionHeaderPreamble() {
+ return ChannelBuffers.wrappedBuffer(RPC_HEADER);
+ }
+
+ public void becomeReady(Channel chan) {
+ this.chan = chan;
+ sendQueuedRpcs();
+ }
+
+ /**
+ * Sends the queued RPCs to the server, once we're connected to it.
+ * This gets called after {@link #channelConnected}, once we were able to
+ * handshake with the server
+ */
+ private void sendQueuedRpcs() {
+ ArrayList<KuduRpc<?>> rpcs;
+ synchronized (this) {
+ rpcs = pending_rpcs;
+ pending_rpcs = null;
+ }
+ if (rpcs != null) {
+ for (final KuduRpc<?> rpc : rpcs) {
+ LOG.debug(getPeerUuidLoggingString() + "Executing RPC queued: " + rpc);
+ sendRpc(rpc);
+ }
+ }
+ }
+
+ void sendContext(Channel channel) {
+ Channels.write(channel, header());
+ becomeReady(channel);
+ }
+
+ private ChannelBuffer header() {
+ RpcHeader.ConnectionContextPB.Builder builder = RpcHeader.ConnectionContextPB.newBuilder();
+ RpcHeader.UserInformationPB.Builder userBuilder = RpcHeader.UserInformationPB.newBuilder();
+ userBuilder.setEffectiveUser(SecureRpcHelper.USER_AND_PASSWORD); // TODO set real user
+ userBuilder.setRealUser(SecureRpcHelper.USER_AND_PASSWORD);
+ builder.setUserInfo(userBuilder.build());
+ RpcHeader.ConnectionContextPB pb = builder.build();
+ RpcHeader.RequestHeader header = RpcHeader.RequestHeader.newBuilder().setCallId
+ (CONNECTION_CTX_CALL_ID).build();
+ return KuduRpc.toChannelBuffer(header, pb);
+ }
+
+ private String getPeerUuidLoggingString() {
+ return "[Peer " + uuid + "] ";
+ }
+
+ /**
+ * Returns this tablet server's uuid.
+ * @return a string that contains this tablet server's uuid
+ */
+ String getUuid() {
+ return uuid;
+ }
+
+ /**
+ * Returns this tablet server's port.
+ * @return a port number that this tablet server is bound to
+ */
+ int getPort() {
+ return port;
+ }
+
+ /**
+ * Returns this tablet server's hostname. We might get many hostnames from the master for a single
+ * TS, and this is the one we picked to connect to originally.
+ * @returna string that contains this tablet server's hostname
+ */
+ String getHost() {
+ return host;
+ }
+
+ public String toString() {
+ final StringBuilder buf = new StringBuilder(13 + 10 + 6 + 64 + 7 + 32 + 16 + 1 + 17 + 2 + 1);
+ buf.append("TabletClient@") // =13
+ .append(hashCode()) // ~10
+ .append("(chan=") // = 6
+ .append(chan) // ~64 (up to 66 when using IPv4)
+ .append(", uuid=") // = 7
+ .append(uuid) // = 32
+ .append(", #pending_rpcs="); // =16
+ int npending_rpcs;
+ synchronized (this) {
+ npending_rpcs = pending_rpcs == null ? 0 : pending_rpcs.size();
+ }
+ buf.append(npending_rpcs); // = 1
+ buf.append(", #rpcs_inflight=") // =17
+ .append(rpcs_inflight.size()) // ~ 2
+ .append(')'); // = 1
+ return buf.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java
new file mode 100644
index 0000000..3db2026
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Update.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Operation to update columns on an existing row. Instances of this class should not be reused.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Update extends Operation {
+
+ Update(KuduTable table) {
+ super(table);
+ }
+
+ @Override
+ ChangeType getChangeType() {
+ return ChangeType.UPDATE;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java
new file mode 100644
index 0000000..4ba2635
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Upsert.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Represents a single row upsert. Instances of this class should not be reused.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Upsert extends Operation {
+
+ Upsert(KuduTable table) {
+ super(table);
+ }
+
+ @Override
+ ChangeType getChangeType() {
+ return ChangeType.UPSERT;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java
new file mode 100644
index 0000000..a93d1b9
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/AsyncUtil.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.util;
+
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+
+import org.kududb.annotations.InterfaceAudience;
+
+/**
+ * Utility methods for various parts of async, such as Deferred.
+ * TODO (KUDU-602): Some of these methods could eventually be contributed back to async or to a
+ * custom fork/derivative of async.
+ */
+@InterfaceAudience.Private
+public class AsyncUtil {
+
+ /**
+ * Register a callback and an "errback".
+ * <p>
+ * This has the exact same effect as {@link Deferred#addCallbacks(Callback, Callback)}
+ * keeps the type information "correct" when the callback and errback return a
+ * {@code Deferred}.
+ * @param d The {@code Deferred} we want to add the callback and errback to.
+ * @param cb The callback to register.
+ * @param eb The errback to register.
+ * @return {@code d} with an "updated" type.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, R, D extends Deferred<R>, E>
+ Deferred<R> addCallbacksDeferring(final Deferred<T> d,
+ final Callback<D, T> cb,
+ final Callback<D, E> eb) {
+ return d.addCallbacks((Callback<R, T>) cb, eb);
+ }
+
+ /**
+ * Workaround for {@link Deferred#addBoth}'s failure to use generics correctly. Allows callers
+ * to provide a {@link Callback} which takes an {@link Object} instead of the type of the deferred
+ * it is applied to, which avoids a runtime {@link ClassCastException} when the deferred fails.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, U> Deferred<U> addBoth(final Deferred<T> deferred,
+ final Callback<? extends U, Object> callback) {
+ return ((Deferred) deferred).addBoth(callback);
+ }
+
+ /**
+ * Workaround for {@link Deferred#addBothDeferring}'s failure to use generics correctly. Allows
+ * callers to provide a {@link Callback} which takes an {@link Object} instead of the type of the
+ * deferred it is applied to, which avoids a runtime {@link ClassCastException} when the deferred
+ * fails.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, U> Deferred<U> addBothDeferring(final Deferred<T> deferred,
+ final Callback<Deferred<U>, Object> callback) {
+ return ((Deferred) deferred).addBothDeferring(callback);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java
new file mode 100644
index 0000000..31436e7
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/HybridTimeUtil.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.util;
+
+import org.kududb.annotations.InterfaceAudience;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Set of common utility methods to handle HybridTime and related timestamps.
+ */
+@InterfaceAudience.Private
+public class HybridTimeUtil {
+
+ public static final int hybridTimeNumBitsToShift = 12;
+ public static final int hybridTimeLogicalBitsMask = (1 << hybridTimeNumBitsToShift) - 1;
+
+ /**
+ * Converts the provided timestamp, in the provided unit, to the HybridTime timestamp
+ * format. Logical bits are set to 0.
+ *
+ * @param timestamp the value of the timestamp, must be greater than 0
+ * @param timeUnit the time unit of the timestamp
+ * @throws IllegalArgumentException if the timestamp is less than 0
+ */
+ public static long clockTimestampToHTTimestamp(long timestamp, TimeUnit timeUnit) {
+ if (timestamp < 0) {
+ throw new IllegalArgumentException("Timestamp cannot be less than 0");
+ }
+ long timestampInMicros = TimeUnit.MICROSECONDS.convert(timestamp, timeUnit);
+ return timestampInMicros << hybridTimeNumBitsToShift;
+ }
+
+ /**
+ * Extracts the physical and logical values from an HT timestamp.
+ *
+ * @param htTimestamp the encoded HT timestamp
+ * @return a pair of {physical, logical} long values in an array
+ */
+ public static long[] HTTimestampToPhysicalAndLogical(long htTimestamp) {
+ long timestampInMicros = htTimestamp >> hybridTimeNumBitsToShift;
+ long logicalValues = htTimestamp & hybridTimeLogicalBitsMask;
+ return new long[] {timestampInMicros, logicalValues};
+ }
+
+ /**
+ * Encodes separate physical and logical components into a single HT timestamp
+ *
+ * @param physical the physical component, in microseconds
+ * @param logical the logical component
+ * @return an encoded HT timestamp
+ */
+ public static long physicalAndLogicalToHTTimestamp(long physical, long logical) {
+ return (physical << hybridTimeNumBitsToShift) + logical;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java
new file mode 100644
index 0000000..1ff77a2
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/NetUtil.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.util;
+
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.net.HostAndPort;
+import org.kududb.annotations.InterfaceAudience;
+
+import java.util.List;
+
+/**
+ * Networking related methods.
+ */
+@InterfaceAudience.Private
+public class NetUtil {
+
+ /**
+ * Convert a list of {@link HostAndPort} objects to a comma separate string.
+ * The inverse of {@link #parseStrings(String, int)}.
+ *
+ * @param hostsAndPorts A list of {@link HostAndPort} objects.
+ * @return Comma separate list of "host:port" pairs.
+ */
+ public static String hostsAndPortsToString(List<HostAndPort> hostsAndPorts) {
+ return Joiner.on(",").join(Lists.transform(hostsAndPorts, Functions.toStringFunction()));
+ }
+
+ /**
+ * Parse a "host:port" pair into a {@link HostAndPort} object. If there is no
+ * port specified in the string, then 'defaultPort' is used.
+ *
+ * @param addrString A host or a "host:port" pair.
+ * @param defaultPort Default port to use if no port is specified in addrString.
+ * @return The HostAndPort object constructed from addrString.
+ */
+ public static HostAndPort parseString(String addrString, int defaultPort) {
+ return addrString.indexOf(':') == -1 ? HostAndPort.fromParts(addrString, defaultPort) :
+ HostAndPort.fromString(addrString);
+ }
+
+ /**
+ * Parse a comma separated list of "host:port" pairs into a list of
+ * {@link HostAndPort} objects. If no port is specified for an entry in
+ * the comma separated list, then a default port is used.
+ * The inverse of {@link #hostsAndPortsToString(List)}.
+ *
+ * @param commaSepAddrs The comma separated list of "host:port" pairs.
+ * @param defaultPort The default port to use if no port is specified.
+ * @return A list of HostAndPort objects constructed from commaSepAddrs.
+ */
+ public static List<HostAndPort> parseStrings(final String commaSepAddrs, int defaultPort) {
+ Iterable<String> addrStrings = Splitter.on(',').trimResults().split(commaSepAddrs);
+ List<HostAndPort> hostsAndPorts = Lists.newArrayListWithCapacity(Iterables.size(addrStrings));
+ for (String addrString : addrStrings) {
+ HostAndPort hostAndPort = parseString(addrString, defaultPort);
+ hostsAndPorts.add(hostAndPort);
+ }
+ return hostsAndPorts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java b/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java
new file mode 100644
index 0000000..341ec10
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/Pair.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.util;
+
+import com.google.common.base.Objects;
+import org.kududb.annotations.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class Pair<A, B> {
+ private final A first;
+ private final B second;
+
+ public Pair(A first, B second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ public A getFirst() {
+ return first;
+ }
+
+ public B getSecond() {
+ return second;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Pair<?, ?> pair = (Pair<?, ?>) o;
+
+ if (first != null ? !first.equals(pair.first) : pair.first != null) return false;
+ if (second != null ? !second.equals(pair.second) : pair.second != null) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(first, second);
+ }
+}
[14/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/RecoverableException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/RecoverableException.java b/java/kudu-client/src/main/java/org/kududb/client/RecoverableException.java
deleted file mode 100644
index 25c0fe0..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/RecoverableException.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * An exception that's possible to retry.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-@SuppressWarnings("serial")
-class RecoverableException extends KuduException {
-
- /**
- * Constructor.
- * @param status status object containing the reason for the exception
- * trace.
- */
- RecoverableException(Status status) {
- super(status);
- }
-
- /**
- * Constructor.
- * @param status status object containing the reason for the exception
- * @param cause The exception that caused this one to be thrown.
- */
- RecoverableException(Status status, Throwable cause) {
- super(status, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/RequestTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/RequestTracker.java b/java/kudu-client/src/main/java/org/kududb/client/RequestTracker.java
deleted file mode 100644
index 229b64f..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/RequestTracker.java
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-
-import java.util.Queue;
-import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This is the same class as src/kudu/rpc/request_tracker.h.
- */
-@InterfaceAudience.Private
-public class RequestTracker {
- private final AtomicLong sequenceIdTracker = new AtomicLong();
- private final Queue<Long> incompleteRpcs = new PriorityBlockingQueue<>();
-
- static final long NO_SEQ_NO = -1;
-
- private final String clientId;
-
- /**
- * Create a new request tracker for the given client id.
- * @param clientId identifier for the client this tracker belongs to
- */
- public RequestTracker(String clientId) {
- this.clientId = clientId;
- }
-
- /**
- * Generates a new sequence number and tracks it.
- * @return a new sequence number
- */
- public long newSeqNo() {
- Long next = sequenceIdTracker.incrementAndGet();
- incompleteRpcs.add(next);
- return next;
- }
-
- /**
- * Returns the oldest sequence number that wasn't marked as completed. If there is no incomplete
- * RPC then {@link RequestTracker#NO_SEQ_NO} is returned.
- * @return the first incomplete sequence number
- */
- public long firstIncomplete() {
- Long peek = incompleteRpcs.peek();
- return peek == null ? NO_SEQ_NO : peek;
- }
-
- /**
- * Marks the given sequence id as complete. This operation is idempotent.
- * @param sequenceId the sequence id to mark as complete
- */
- public void rpcCompleted(long sequenceId) {
- incompleteRpcs.remove(sequenceId);
- }
-
- public String getClientId() {
- return clientId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/RowError.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/RowError.java b/java/kudu-client/src/main/java/org/kududb/client/RowError.java
deleted file mode 100644
index b4c8f36..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/RowError.java
+++ /dev/null
@@ -1,116 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
-
-/**
- * Wrapper class for a single row error.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class RowError {
- private final Status status;
- private final Operation operation;
- private final String tsUUID;
-
- /**
- * Creates a new {@code RowError} with the provided status, operation, and tablet server UUID.
- */
- RowError(Status status, Operation operation, String tsUUID) {
- this.status = status;
- this.operation = operation;
- this.tsUUID = tsUUID;
- }
-
- /**
- * Creates a new {@code RowError} with the provided status, and operation.
- *
- * This constructor should be used when the operation fails before the tablet
- * lookup is complete.
- */
- RowError(Status status, Operation operation) {
- this(status, operation, null);
- }
-
- /**
- * Get the status code and message of the row error.
- */
- public Status getErrorStatus() {
- return status;
- }
-
- /**
- * Get the string-representation of the error code that the tablet server returned.
- * @return A short string representation of the error.
- * @deprecated Please use getErrorStatus() instead. Will be removed in a future version.
- */
- public String getStatus() {
- return status.getCodeName();
- }
-
- /**
- * Get the error message the tablet server sent.
- * @return The error message.
- * @deprecated Please use getErrorStatus() instead. Will be removed in a future version.
- */
- public String getMessage() {
- return status.getMessage();
- }
-
- /**
- * Get the Operation that failed.
- * @return The same Operation instance that failed
- */
- public Operation getOperation() {
- return operation;
- }
-
- /**
- * Get the identifier of the tablet server that sent the error.
- * The UUID may be {@code null} if the failure occurred before sending the row
- * to a tablet server (for instance, if the row falls in a non-covered range partition).
- * @return A string containing a UUID
- */
- public String getTsUUID() {
- return tsUUID;
- }
-
- @Override
- public String toString() {
- return "Row error for primary key=" + Bytes.pretty(operation.getRow().encodePrimaryKey()) +
- ", tablet=" + operation.getTablet() +
- ", server=" + tsUUID +
- ", status=" + status.toString();
- }
-
- /**
- * Converts a PerRowErrorPB into a RowError.
- * @param errorPB a row error in its pb format
- * @param operation the original operation
- * @param tsUUID a string containing the originating TS's UUID
- * @return a row error
- */
- static RowError fromRowErrorPb(Tserver.WriteResponsePB.PerRowErrorPB errorPB,
- Operation operation, String tsUUID) {
- WireProtocol.AppStatusPB statusPB = errorPB.getError();
- return new RowError(Status.fromPB(statusPB), operation, tsUUID);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/RowErrorsAndOverflowStatus.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/RowErrorsAndOverflowStatus.java b/java/kudu-client/src/main/java/org/kududb/client/RowErrorsAndOverflowStatus.java
deleted file mode 100644
index 17a4778..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/RowErrorsAndOverflowStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Container class used as a response when retrieving pending row errors.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class RowErrorsAndOverflowStatus {
- private final RowError[] rowErrors;
- private final boolean overflowed;
-
- RowErrorsAndOverflowStatus(RowError[] rowErrors, boolean overflowed) {
- this.rowErrors = rowErrors;
- this.overflowed = overflowed;
- }
-
- /**
- * Get the collected row errors.
- * @return an array of row errors, may be empty
- */
- public RowError[] getRowErrors() {
- return rowErrors;
- }
-
- /**
- * Check if the error collector had an overflow and had to discard row errors.
- * @return true if row errors were discarded, false otherwise
- */
- public boolean isOverflowed() {
- return overflowed;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/RowResult.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/RowResult.java b/java/kudu-client/src/main/java/org/kududb/client/RowResult.java
deleted file mode 100644
index 7692c53..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/RowResult.java
+++ /dev/null
@@ -1,570 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.Slice;
-
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.BitSet;
-import java.util.Date;
-import java.util.TimeZone;
-
-/**
- * RowResult represents one row from a scanner. Do not reuse or store the objects.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class RowResult {
-
- private static final int INDEX_RESET_LOCATION = -1;
- private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- {
- DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT"));
- }
- private static final long MS_IN_S = 1000L;
- private static final long US_IN_S = 1000L * 1000L;
- private int index = INDEX_RESET_LOCATION;
- private int offset;
- private BitSet nullsBitSet;
- private final int rowSize;
- private final int[] columnOffsets;
- private final Schema schema;
- private final Slice rowData;
- private final Slice indirectData;
-
- /**
- * Prepares the row representation using the provided data. Doesn't copy data
- * out of the byte arrays. Package private.
- * @param schema Schema used to build the rowData
- * @param rowData The Slice of data returned by the tablet server
- * @param indirectData The full indirect data that contains the strings
- */
- RowResult(Schema schema, Slice rowData, Slice indirectData) {
- this.schema = schema;
- this.rowData = rowData;
- this.indirectData = indirectData;
- int columnOffsetsSize = schema.getColumnCount();
- if (schema.hasNullableColumns()) {
- columnOffsetsSize++;
- }
- this.rowSize = this.schema.getRowSize();
- columnOffsets = new int[columnOffsetsSize];
- // Empty projection, usually used for quick row counting
- if (columnOffsetsSize == 0) {
- return;
- }
- int currentOffset = 0;
- columnOffsets[0] = currentOffset;
- // Pre-compute the columns offsets in rowData for easier lookups later
- // If the schema has nullables, we also add the offset for the null bitmap at the end
- for (int i = 1; i < columnOffsetsSize; i++) {
- int previousSize = schema.getColumnByIndex(i - 1).getType().getSize();
- columnOffsets[i] = previousSize + currentOffset;
- currentOffset += previousSize;
- }
- }
-
- /**
- * Package-protected, only meant to be used by the RowResultIterator
- */
- void advancePointer() {
- advancePointerTo(this.index + 1);
- }
-
- void resetPointer() {
- advancePointerTo(INDEX_RESET_LOCATION);
- }
-
- void advancePointerTo(int rowIndex) {
- this.index = rowIndex;
- this.offset = this.rowSize * this.index;
- if (schema.hasNullableColumns() && this.index != INDEX_RESET_LOCATION) {
- this.nullsBitSet = Bytes.toBitSet(
- this.rowData.getRawArray(),
- this.rowData.getRawOffset()
- + getCurrentRowDataOffsetForColumn(schema.getColumnCount()),
- schema.getColumnCount());
- }
- }
-
- int getCurrentRowDataOffsetForColumn(int columnIndex) {
- return this.offset + this.columnOffsets[columnIndex];
- }
-
- /**
- * Get the specified column's integer
- * @param columnName name of the column to get data for
- * @return An integer
- * @throws IllegalArgumentException if the column is null
- */
- public int getInt(String columnName) {
- return getInt(this.schema.getColumnIndex(columnName));
- }
-
- /**
- * Get the specified column's integer
- * @param columnIndex Column index in the schema
- * @return An integer
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public int getInt(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.INT32);
- return Bytes.getInt(this.rowData.getRawArray(),
- this.rowData.getRawOffset() + getCurrentRowDataOffsetForColumn(columnIndex));
- }
-
- /**
- * Get the specified column's short
- * @param columnName name of the column to get data for
- * @return A short
- * @throws IllegalArgumentException if the column is null
- */
- public short getShort(String columnName) {
- return getShort(this.schema.getColumnIndex(columnName));
- }
-
- /**
- * Get the specified column's short
- * @param columnIndex Column index in the schema
- * @return A short
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public short getShort(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.INT16);
- return Bytes.getShort(this.rowData.getRawArray(),
- this.rowData.getRawOffset() + getCurrentRowDataOffsetForColumn(columnIndex));
- }
-
- /**
- * Get the specified column's boolean
- * @param columnName name of the column to get data for
- * @return A boolean
- * @throws IllegalArgumentException if the column is null
- */
- public boolean getBoolean(String columnName) {
- return getBoolean(this.schema.getColumnIndex(columnName));
- }
-
- /**
- * Get the specified column's boolean
- * @param columnIndex Column index in the schema
- * @return A boolean
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public boolean getBoolean(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.BOOL);
- byte b = Bytes.getByte(this.rowData.getRawArray(),
- this.rowData.getRawOffset()
- + getCurrentRowDataOffsetForColumn(columnIndex));
- return b == 1;
- }
-
- /**
- * Get the specified column's byte
- * @param columnName name of the column to get data for
- * @return A byte
- * @throws IllegalArgumentException if the column is null
- */
- public byte getByte(String columnName) {
- return getByte(this.schema.getColumnIndex(columnName));
-
- }
-
- /**
- * Get the specified column's byte
- * @param columnIndex Column index in the schema
- * @return A byte
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public byte getByte(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.INT8);
- return Bytes.getByte(this.rowData.getRawArray(),
- this.rowData.getRawOffset() + getCurrentRowDataOffsetForColumn(columnIndex));
- }
-
- /**
- * Get the specified column's long
- *
- * If this is a TIMESTAMP column, the long value corresponds to a number of microseconds
- * since midnight, January 1, 1970 UTC.
- *
- * @param columnName name of the column to get data for
- * @return A positive long
- * @throws IllegalArgumentException if the column is null\
- */
- public long getLong(String columnName) {
- return getLong(this.schema.getColumnIndex(columnName));
- }
-
- /**
- * Get the specified column's long
- *
- * If this is a TIMESTAMP column, the long value corresponds to a number of microseconds
- * since midnight, January 1, 1970 UTC.
- *
- * @param columnIndex Column index in the schema
- * @return A positive long
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public long getLong(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- // Can't check type because this could be a long, string, or Timestamp
- return Bytes.getLong(this.rowData.getRawArray(),
- this.rowData.getRawOffset()
- + getCurrentRowDataOffsetForColumn(columnIndex));
- }
-
- /**
- * Get the specified column's float
- * @param columnName name of the column to get data for
- * @return A float
- */
- public float getFloat(String columnName) {
- return getFloat(this.schema.getColumnIndex(columnName));
- }
-
- /**
- * Get the specified column's float
- * @param columnIndex Column index in the schema
- * @return A float
- */
- public float getFloat(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.FLOAT);
- return Bytes.getFloat(this.rowData.getRawArray(),
- this.rowData.getRawOffset()
- + getCurrentRowDataOffsetForColumn(columnIndex));
- }
-
- /**
- * Get the specified column's double
- * @param columnName name of the column to get data for
- * @return A double
- */
- public double getDouble(String columnName) {
- return getDouble(this.schema.getColumnIndex(columnName));
-
- }
-
- /**
- * Get the specified column's double
- * @param columnIndex Column index in the schema
- * @return A double
- */
- public double getDouble(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.DOUBLE);
- return Bytes.getDouble(this.rowData.getRawArray(),
- this.rowData.getRawOffset()
- + getCurrentRowDataOffsetForColumn(columnIndex));
- }
-
- /**
- * Get the schema used for this scanner's column projection.
- * @return A column projection as a schema.
- */
- public Schema getColumnProjection() {
- return this.schema;
- }
-
- /**
- * Get the specified column's string.
- * @param columnName name of the column to get data for
- * @return A string
- * @throws IllegalArgumentException if the column is null
- */
- public String getString(String columnName) {
- return getString(this.schema.getColumnIndex(columnName));
-
- }
-
- /**
- * Get the specified column's string.
- * @param columnIndex Column index in the schema
- * @return A string
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public String getString(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.STRING);
- // C++ puts a Slice in rowData which is 16 bytes long for simplity, but we only support ints
- long offset = getLong(columnIndex);
- long length = rowData.getLong(getCurrentRowDataOffsetForColumn(columnIndex) + 8);
- assert offset < Integer.MAX_VALUE;
- assert length < Integer.MAX_VALUE;
- return Bytes.getString(indirectData.getRawArray(),
- indirectData.getRawOffset() + (int)offset,
- (int)length);
- }
-
- /**
- * Get a copy of the specified column's binary data.
- * @param columnName name of the column to get data for
- * @return a byte[] with the binary data.
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public byte[] getBinaryCopy(String columnName) {
- return getBinaryCopy(this.schema.getColumnIndex(columnName));
-
- }
-
- /**
- * Get a copy of the specified column's binary data.
- * @param columnIndex Column index in the schema
- * @return a byte[] with the binary data.
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public byte[] getBinaryCopy(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- // C++ puts a Slice in rowData which is 16 bytes long for simplicity,
- // but we only support ints
- long offset = getLong(columnIndex);
- long length = rowData.getLong(getCurrentRowDataOffsetForColumn(columnIndex) + 8);
- assert offset < Integer.MAX_VALUE;
- assert length < Integer.MAX_VALUE;
- byte[] ret = new byte[(int)length];
- System.arraycopy(indirectData.getRawArray(), indirectData.getRawOffset() + (int) offset,
- ret, 0, (int) length);
- return ret;
- }
-
- /**
- * Get the specified column's binary data.
- *
- * This doesn't copy the data and instead returns a ByteBuffer that wraps it.
- *
- * @param columnName name of the column to get data for
- * @return a byte[] with the binary data.
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public ByteBuffer getBinary(String columnName) {
- return getBinary(this.schema.getColumnIndex(columnName));
- }
-
- /**
- * Get the specified column's binary data.
- *
- * This doesn't copy the data and instead returns a ByteBuffer that wraps it.
- *
- * @param columnIndex Column index in the schema
- * @return a byte[] with the binary data.
- * @throws IllegalArgumentException if the column is null
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public ByteBuffer getBinary(int columnIndex) {
- checkValidColumn(columnIndex);
- checkNull(columnIndex);
- checkType(columnIndex, Type.BINARY);
- // C++ puts a Slice in rowData which is 16 bytes long for simplicity,
- // but we only support ints
- long offset = getLong(columnIndex);
- long length = rowData.getLong(getCurrentRowDataOffsetForColumn(columnIndex) + 8);
- assert offset < Integer.MAX_VALUE;
- assert length < Integer.MAX_VALUE;
- return ByteBuffer.wrap(indirectData.getRawArray(), indirectData.getRawOffset() + (int) offset,
- (int) length);
- }
-
- /**
- * Get if the specified column is NULL
- * @param columnName name of the column to get data for
- * @return true if the column cell is null and the column is nullable,
- * false otherwise
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public boolean isNull(String columnName) {
- return isNull(this.schema.getColumnIndex(columnName));
- }
-
- /**
- * Get if the specified column is NULL
- * @param columnIndex Column index in the schema
- * @return true if the column cell is null and the column is nullable,
- * false otherwise
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public boolean isNull(int columnIndex) {
- checkValidColumn(columnIndex);
- if (nullsBitSet == null) {
- return false;
- }
- return schema.getColumnByIndex(columnIndex).isNullable()
- && nullsBitSet.get(columnIndex);
- }
-
- /**
- * Get the type of a column in this result.
- * @param columnName name of the column
- * @return a type
- */
- public Type getColumnType(String columnName) {
- return this.schema.getColumn(columnName).getType();
- }
-
- /**
- * Get the type of a column in this result.
- * @param columnIndex column index in the schema
- * @return a type
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- public Type getColumnType(int columnIndex) {
- return this.schema.getColumnByIndex(columnIndex).getType();
- }
-
- /**
- * Get the schema associated with this result.
- * @return a schema
- */
- public Schema getSchema() {
- return schema;
- }
-
- /**
- * @throws IndexOutOfBoundsException if the column doesn't exist
- */
- private void checkValidColumn(int columnIndex) {
- if (columnIndex >= schema.getColumnCount()) {
- throw new IndexOutOfBoundsException("Requested column is out of range, " +
- columnIndex + " out of " + schema.getColumnCount());
- }
- }
-
- /**
- * @throws IllegalArgumentException if the column is null
- */
- private void checkNull(int columnIndex) {
- if (!schema.hasNullableColumns()) {
- return;
- }
- if (isNull(columnIndex)) {
- ColumnSchema columnSchema = schema.getColumnByIndex(columnIndex);
- throw new IllegalArgumentException("The requested column (name: " + columnSchema.getName() +
- ", index: " + columnIndex + ") is null");
- }
- }
-
- private void checkType(int columnIndex, Type expectedType) {
- ColumnSchema columnSchema = schema.getColumnByIndex(columnIndex);
- Type columnType = columnSchema.getType();
- if (!columnType.equals(expectedType)) {
- throw new IllegalArgumentException("Column (name: " + columnSchema.getName() +
- ", index: " + columnIndex +") is of type " +
- columnType.getName() + " but was requested as a type " + expectedType.getName());
- }
- }
-
- @Override
- public String toString() {
- return "RowResult index: " + this.index + ", size: " + this.rowSize + ", " +
- "schema: " + this.schema;
- }
-
- /**
- * Transforms a timestamp into a string, whose formatting and timezone is consistent
- * across kudu.
- * @param timestamp the timestamp, in microseconds
- * @return a string, in the format: YYYY-MM-DD HH:MM:SS.ssssss GMT
- */
- static String timestampToString(long timestamp) {
- long tsMillis = timestamp / MS_IN_S;
- long tsMicros = timestamp % US_IN_S;
- StringBuffer formattedTs = new StringBuffer();
- formattedTs.append(DATE_FORMAT.format(new Date(tsMillis)));
- formattedTs.append(String.format(".%06d GMT", tsMicros));
- return formattedTs.toString();
- }
-
- /**
- * Return the actual data from this row in a stringified key=value
- * form.
- */
- public String rowToString() {
- StringBuffer buf = new StringBuffer();
- for (int i = 0; i < schema.getColumnCount(); i++) {
- ColumnSchema col = schema.getColumnByIndex(i);
- if (i != 0) {
- buf.append(", ");
- }
- buf.append(col.getType().name());
- buf.append(" ").append(col.getName()).append("=");
- if (isNull(i)) {
- buf.append("NULL");
- } else {
- switch (col.getType()) {
- case INT8: buf.append(getByte(i)); break;
- case INT16: buf.append(getShort(i));
- break;
- case INT32: buf.append(getInt(i)); break;
- case INT64: buf.append(getLong(i)); break;
- case TIMESTAMP: {
- buf.append(timestampToString(getLong(i)));
- } break;
- case STRING: buf.append(getString(i)); break;
- case BINARY: buf.append(Bytes.pretty(getBinaryCopy(i))); break;
- case FLOAT: buf.append(getFloat(i)); break;
- case DOUBLE: buf.append(getDouble(i)); break;
- case BOOL: buf.append(getBoolean(i)); break;
- default: buf.append("<unknown type!>"); break;
- }
- }
- }
- return buf.toString();
- }
-
- /**
- * @return a string describing the location of this row result within
- * the iterator as well as its data.
- */
- public String toStringLongFormat() {
- StringBuffer buf = new StringBuffer(this.rowSize); // super rough estimation
- buf.append(this.toString());
- buf.append("{");
- buf.append(rowToString());
- buf.append("}");
- return buf.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/RowResultIterator.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/RowResultIterator.java b/java/kudu-client/src/main/java/org/kududb/client/RowResultIterator.java
deleted file mode 100644
index 5705ea3..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/RowResultIterator.java
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import java.util.Iterator;
-import org.kududb.Schema;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.Slice;
-
-/**
- * Class that contains the rows sent by a tablet server, exhausting this iterator only means
- * that all the rows from the last server response were read.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class RowResultIterator extends KuduRpcResponse implements Iterator<RowResult>,
- Iterable<RowResult> {
-
- private static final RowResultIterator EMPTY =
- new RowResultIterator(0, null, null, 0, null, null);
-
- private final Schema schema;
- private final Slice bs;
- private final Slice indirectBs;
- private final int numRows;
- private final RowResult rowResult;
- private int currentRow = 0;
-
- /**
- * Package private constructor, only meant to be instantiated from AsyncKuduScanner.
- * @param ellapsedMillis ime in milliseconds since RPC creation to now
- * @param tsUUID UUID of the tablet server that handled our request
- * @param schema schema used to parse the rows
- * @param numRows how many rows are contained in the bs slice
- * @param bs normal row data
- * @param indirectBs indirect row data
- */
- private RowResultIterator(long ellapsedMillis, String tsUUID, Schema schema,
- int numRows, Slice bs, Slice indirectBs) {
- super(ellapsedMillis, tsUUID);
- this.schema = schema;
- this.bs = bs;
- this.indirectBs = indirectBs;
- this.numRows = numRows;
-
- this.rowResult = numRows == 0 ? null : new RowResult(this.schema, this.bs, this.indirectBs);
- }
-
- static RowResultIterator makeRowResultIterator(long ellapsedMillis, String tsUUID,
- Schema schema,
- WireProtocol.RowwiseRowBlockPB data,
- final CallResponse callResponse)
- throws KuduException {
- if (data == null || data.getNumRows() == 0) {
- return new RowResultIterator(ellapsedMillis, tsUUID, schema, 0, null, null);
- }
-
- Slice bs = callResponse.getSidecar(data.getRowsSidecar());
- Slice indirectBs = callResponse.getSidecar(data.getIndirectDataSidecar());
- int numRows = data.getNumRows();
-
- // Integrity check
- int rowSize = schema.getRowSize();
- int expectedSize = numRows * rowSize;
- if (expectedSize != bs.length()) {
- Status statusIllegalState = Status.IllegalState("RowResult block has " + bs.length() +
- " bytes of data but expected " + expectedSize + " for " + numRows + " rows");
- throw new NonRecoverableException(statusIllegalState);
- }
- return new RowResultIterator(ellapsedMillis, tsUUID, schema, numRows, bs, indirectBs);
- }
-
- /**
- * @return an empty row result iterator
- */
- static RowResultIterator empty() {
- return EMPTY;
- }
-
- @Override
- public boolean hasNext() {
- return this.currentRow < numRows;
- }
-
- @Override
- public RowResult next() {
- // The rowResult keeps track of where it is internally
- this.rowResult.advancePointer();
- this.currentRow++;
- return rowResult;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Get the number of rows in this iterator. If all you want is to count
- * rows, call this and skip the rest.
- * @return number of rows in this iterator
- */
- public int getNumRows() {
- return this.numRows;
- }
-
- @Override
- public String toString() {
- return "RowResultIterator for " + this.numRows + " rows";
- }
-
- @Override
- public Iterator<RowResult> iterator() {
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/SecureRpcHelper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/SecureRpcHelper.java b/java/kudu-client/src/main/java/org/kududb/client/SecureRpcHelper.java
deleted file mode 100644
index 53e108a..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/SecureRpcHelper.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the aabove copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.Channels;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.rpc.RpcHeader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import javax.security.auth.callback.Callback;
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.callback.NameCallback;
-import javax.security.auth.callback.PasswordCallback;
-import javax.security.auth.callback.UnsupportedCallbackException;
-import javax.security.sasl.RealmCallback;
-import javax.security.sasl.RealmChoiceCallback;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslClient;
-import javax.security.sasl.SaslException;
-
-@InterfaceAudience.Private
-public class SecureRpcHelper {
-
- public static final Logger LOG = LoggerFactory.getLogger(TabletClient.class);
-
- private final TabletClient client;
- private SaslClient saslClient;
- public static final String SASL_DEFAULT_REALM = "default";
- public static final Map<String, String> SASL_PROPS = new TreeMap<>();
- private static final int SASL_CALL_ID = -33;
- private static final Set<RpcHeader.RpcFeatureFlag> SUPPORTED_RPC_FEATURES =
- ImmutableSet.of(RpcHeader.RpcFeatureFlag.APPLICATION_FEATURE_FLAGS);
- private volatile boolean negoUnderway = true;
- private boolean useWrap = false; // no QOP at the moment
- private Set<RpcHeader.RpcFeatureFlag> serverFeatures;
-
- public static final String USER_AND_PASSWORD = "java_client";
-
- public SecureRpcHelper(TabletClient client) {
- this.client = client;
- try {
- saslClient = Sasl.createSaslClient(new String[]{"PLAIN"},
- null,
- null,
- SASL_DEFAULT_REALM,
- SASL_PROPS,
- new SaslClientCallbackHandler(USER_AND_PASSWORD,
- USER_AND_PASSWORD));
- } catch (SaslException e) {
- throw new RuntimeException("Could not create the SASL client", e);
- }
- }
-
- public Set<RpcHeader.RpcFeatureFlag> getServerFeatures() {
- Preconditions.checkState(!negoUnderway);
- Preconditions.checkNotNull(serverFeatures);
- return serverFeatures;
- }
-
- public void sendHello(Channel channel) {
- sendNegotiateMessage(channel);
- }
-
- private void sendNegotiateMessage(Channel channel) {
- RpcHeader.SaslMessagePB.Builder builder = RpcHeader.SaslMessagePB.newBuilder();
-
- // Advertise our supported features
- for (RpcHeader.RpcFeatureFlag flag : SUPPORTED_RPC_FEATURES) {
- builder.addSupportedFeatures(flag);
- }
-
- builder.setState(RpcHeader.SaslMessagePB.SaslState.NEGOTIATE);
- sendSaslMessage(channel, builder.build());
- }
-
- private void sendSaslMessage(Channel channel, RpcHeader.SaslMessagePB msg) {
- RpcHeader.RequestHeader.Builder builder = RpcHeader.RequestHeader.newBuilder();
- builder.setCallId(SASL_CALL_ID);
- RpcHeader.RequestHeader header = builder.build();
-
- ChannelBuffer buffer = KuduRpc.toChannelBuffer(header, msg);
- Channels.write(channel, buffer);
- }
-
- public ChannelBuffer handleResponse(ChannelBuffer buf, Channel chan) throws SaslException {
- if (!saslClient.isComplete() || negoUnderway) {
- RpcHeader.SaslMessagePB response = parseSaslMsgResponse(buf);
- switch (response.getState()) {
- case NEGOTIATE:
- handleNegotiateResponse(chan, response);
- break;
- case CHALLENGE:
- handleChallengeResponse(chan, response);
- break;
- case SUCCESS:
- handleSuccessResponse(chan, response);
- break;
- default:
- System.out.println("Wrong sasl state");
- }
- return null;
- }
- return unwrap(buf);
- }
-
- /**
- * When QOP of auth-int or auth-conf is selected
- * This is used to unwrap the contents from the passed
- * buffer payload.
- */
- public ChannelBuffer unwrap(ChannelBuffer payload) {
- if(!useWrap) {
- return payload;
- }
- int len = payload.readInt();
- try {
- payload =
- ChannelBuffers.wrappedBuffer(saslClient.unwrap(payload.readBytes(len).array(), 0, len));
- return payload;
- } catch (SaslException e) {
- throw new IllegalStateException("Failed to unwrap payload", e);
- }
- }
-
- /**
- * When QOP of auth-int or auth-conf is selected
- * This is used to wrap the contents
- * into the proper payload (ie encryption, signature, etc)
- */
- public ChannelBuffer wrap(ChannelBuffer content) {
- if(!useWrap) {
- return content;
- }
- try {
- byte[] payload = new byte[content.writerIndex()];
- content.readBytes(payload);
- byte[] wrapped = saslClient.wrap(payload, 0, payload.length);
- ChannelBuffer ret = ChannelBuffers.wrappedBuffer(new byte[4 + wrapped.length]);
- ret.clear();
- ret.writeInt(wrapped.length);
- ret.writeBytes(wrapped);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Wrapped payload: "+Bytes.pretty(ret));
- }
- return ret;
- } catch (SaslException e) {
- throw new IllegalStateException("Failed to wrap payload", e);
- }
- }
-
- private RpcHeader.SaslMessagePB parseSaslMsgResponse(ChannelBuffer buf) {
- CallResponse response = new CallResponse(buf);
- RpcHeader.ResponseHeader responseHeader = response.getHeader();
- int id = responseHeader.getCallId();
- if (id != SASL_CALL_ID) {
- throw new IllegalStateException("Received a call that wasn't for SASL");
- }
-
- RpcHeader.SaslMessagePB.Builder saslBuilder = RpcHeader.SaslMessagePB.newBuilder();
- KuduRpc.readProtobuf(response.getPBMessage(), saslBuilder);
- return saslBuilder.build();
- }
-
-
- private void handleNegotiateResponse(Channel chan, RpcHeader.SaslMessagePB response) throws
- SaslException {
- RpcHeader.SaslMessagePB.SaslAuth negotiatedAuth = null;
- for (RpcHeader.SaslMessagePB.SaslAuth auth : response.getAuthsList()) {
- negotiatedAuth = auth;
- }
-
- ImmutableSet.Builder<RpcHeader.RpcFeatureFlag> features = ImmutableSet.builder();
- for (RpcHeader.RpcFeatureFlag feature : response.getSupportedFeaturesList()) {
- if (SUPPORTED_RPC_FEATURES.contains(feature)) {
- features.add(feature);
- }
- }
- serverFeatures = features.build();
-
- byte[] saslToken = new byte[0];
- if (saslClient.hasInitialResponse())
- saslToken = saslClient.evaluateChallenge(saslToken);
-
- RpcHeader.SaslMessagePB.Builder builder = RpcHeader.SaslMessagePB.newBuilder();
- if (saslToken != null) {
- builder.setToken(ZeroCopyLiteralByteString.wrap(saslToken));
- }
- builder.setState(RpcHeader.SaslMessagePB.SaslState.INITIATE);
- builder.addAuths(negotiatedAuth);
- sendSaslMessage(chan, builder.build());
- }
-
- private void handleChallengeResponse(Channel chan, RpcHeader.SaslMessagePB response) throws
- SaslException {
- ByteString bs = response.getToken();
- byte[] saslToken = saslClient.evaluateChallenge(bs.toByteArray());
- if (saslToken == null) {
- throw new IllegalStateException("Not expecting an empty token");
- }
- RpcHeader.SaslMessagePB.Builder builder = RpcHeader.SaslMessagePB.newBuilder();
- builder.setToken(ZeroCopyLiteralByteString.wrap(saslToken));
- builder.setState(RpcHeader.SaslMessagePB.SaslState.RESPONSE);
- sendSaslMessage(chan, builder.build());
- }
-
- private void handleSuccessResponse(Channel chan, RpcHeader.SaslMessagePB response) {
- LOG.debug("nego finished");
- negoUnderway = false;
- client.sendContext(chan);
- }
-
- private static class SaslClientCallbackHandler implements CallbackHandler {
- private final String userName;
- private final char[] userPassword;
-
- public SaslClientCallbackHandler(String user, String password) {
- this.userName = user;
- this.userPassword = password.toCharArray();
- }
-
- public void handle(Callback[] callbacks)
- throws UnsupportedCallbackException {
- NameCallback nc = null;
- PasswordCallback pc = null;
- RealmCallback rc = null;
- for (Callback callback : callbacks) {
- if (callback instanceof RealmChoiceCallback) {
- continue;
- } else if (callback instanceof NameCallback) {
- nc = (NameCallback) callback;
- } else if (callback instanceof PasswordCallback) {
- pc = (PasswordCallback) callback;
- } else if (callback instanceof RealmCallback) {
- rc = (RealmCallback) callback;
- } else {
- throw new UnsupportedCallbackException(callback,
- "Unrecognized SASL client callback");
- }
- }
- if (nc != null) {
- nc.setName(userName);
- }
- if (pc != null) {
- pc.setPassword(userPassword);
- }
- if (rc != null) {
- rc.setText(rc.getDefaultText());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java b/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java
deleted file mode 100644
index 94e0a66..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/SessionConfiguration.java
+++ /dev/null
@@ -1,158 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Interface that defines the methods used to configure a session. It also exposes ways to
- * query its state.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public interface SessionConfiguration {
-
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- enum FlushMode {
- // Every write will be sent to the server in-band with the Apply()
- // call. No batching will occur. This is the default flush mode. In this
- // mode, the Flush() call never has any effect, since each Apply() call
- // has already flushed the buffer.
- AUTO_FLUSH_SYNC,
-
- // Apply() calls will return immediately, but the writes will be sent in
- // the background, potentially batched together with other writes from
- // the same session. If there is not sufficient buffer space, then Apply()
- // may block for buffer space to be available.
- //
- // Because writes are applied in the background, any errors will be stored
- // in a session-local buffer. Call CountPendingErrors() or GetPendingErrors()
- // to retrieve them.
- //
- // The Flush() call can be used to block until the buffer is empty.
- AUTO_FLUSH_BACKGROUND,
-
- // Apply() calls will return immediately, and the writes will not be
- // sent until the user calls Flush(). If the buffer runs past the
- // configured space limit, then Apply() will return an error.
- MANUAL_FLUSH
- }
-
- /**
- * Get the current flush mode.
- * @return flush mode, AUTO_FLUSH_SYNC by default
- */
- FlushMode getFlushMode();
-
- /**
- * Set the new flush mode for this session.
- * @param flushMode new flush mode, can be the same as the previous one.
- * @throws IllegalArgumentException if the buffer isn't empty.
- */
- void setFlushMode(FlushMode flushMode);
-
- /**
- * Set the number of operations that can be buffered.
- * @param size number of ops.
- * @throws IllegalArgumentException if the buffer isn't empty.
- */
- void setMutationBufferSpace(int size);
-
- /**
- * Set the low watermark for this session. The default is set to half the mutation buffer space.
- * For example, a buffer space of 1000 with a low watermark set to 50% (0.5) will start randomly
- * sending PleaseRetryExceptions once there's an outstanding flush and the buffer is over 500.
- * As the buffer gets fuller, it becomes likelier to hit the exception.
- * @param mutationBufferLowWatermarkPercentage a new low watermark as a percentage,
- * has to be between 0 and 1 (inclusive). A value of 1 disables
- * the low watermark since it's the same as the high one
- * @throws IllegalArgumentException if the buffer isn't empty or if the watermark isn't between
- * 0 and 1
- */
- void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage);
-
- /**
- * Set the flush interval, which will be used for the next scheduling decision.
- * @param interval interval in milliseconds.
- */
- void setFlushInterval(int interval);
-
- /**
- * Get the current timeout.
- * @return operation timeout in milliseconds, 0 if none was configured.
- */
- long getTimeoutMillis();
-
- /**
- * Sets the timeout for the next applied operations.
- * The default timeout is 0, which disables the timeout functionality.
- * @param timeout Timeout in milliseconds.
- */
- void setTimeoutMillis(long timeout);
-
- /**
- * Returns true if this session has already been closed.
- */
- boolean isClosed();
-
- /**
- * Check if there are operations that haven't been completely applied.
- * @return true if operations are pending, else false.
- */
- boolean hasPendingOperations();
-
- /**
- * Set the new external consistency mode for this session.
- * @param consistencyMode new external consistency mode, can the same as the previous one.
- * @throws IllegalArgumentException if the buffer isn't empty.
- */
- void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode);
-
- /**
- * Tells if the session is currently ignoring row errors when the whole list returned by a tablet
- * server is of the AlreadyPresent type.
- * @return true if the session is enforcing this, else false
- */
- boolean isIgnoreAllDuplicateRows();
-
- /**
- * Configures the option to ignore all the row errors if they are all of the AlreadyPresent type.
- * This can be needed when facing KUDU-568. The effect of enabling this is that operation
- * responses that match this pattern will be cleared of their row errors, meaning that we consider
- * them successful.
- * This is disabled by default.
- * @param ignoreAllDuplicateRows true if this session should enforce this, else false
- */
- void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows);
-
- /**
- * Return the number of errors which are pending. Errors may accumulate when
- * using the AUTO_FLUSH_BACKGROUND mode.
- * @return a count of errors
- */
- int countPendingErrors();
-
- /**
- * Return any errors from previous calls. If there were more errors
- * than could be held in the session's error storage, the overflow state is set to true.
- * Resets the pending errors.
- * @return an object that contains the errors and the overflow status
- */
- RowErrorsAndOverflowStatus getPendingErrors();
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Statistics.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Statistics.java b/java/kudu-client/src/main/java/org/kududb/client/Statistics.java
deleted file mode 100644
index ef45f9e..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Statistics.java
+++ /dev/null
@@ -1,258 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.Sets;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.Slice;
-import org.kududb.util.Slices;
-
-import java.nio.charset.Charset;
-import java.util.Collections;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLongArray;
-
-
-/**
- * A Statistics belongs to a specific AsyncKuduClient. It stores client-level
- * statistics including number of operations, number of bytes written, number of
- * rpcs. It is created along with the client's creation, and can be obtained through
- * AsyncKuduClient or KuduClient's getStatistics method. Once obtained, an instance
- * of this class can be used directly.
- * <p>
- * This class is thread-safe. The user can use it anywhere to get statistics of this
- * client.
- * <p>
- * The method {@link #toString} can be useful to get a dump of all the metrics aggregated
- * for all the tablets.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Statistics {
- private final ConcurrentHashMap<Slice, Statistics.TabletStatistics> stsMap = new ConcurrentHashMap<>();
-
- /**
- * The statistic enum to pass when querying.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public enum Statistic {
- /**
- * How many bytes have been written by this client. If one rpc fails, this
- * statistic won't be updated.
- */
- BYTES_WRITTEN(0),
- /**
- * How many operations have been sent to server and succeeded.
- */
- WRITE_OPS(1),
- /**
- * How many rpcs have been sent to server and succeeded. One rpc may contain
- * multiple operations.
- */
- WRITE_RPCS(2),
- /**
- * How many operations have been sent to server but failed.
- */
- OPS_ERRORS(3),
- /**
- * How many rpcs have been sent to server but failed.
- */
- RPC_ERRORS(4);
-
- Statistic(int idx) {
- this.idx = idx;
- }
-
- /**
- * Get index of this statistic.
- * @return index
- */
- int getIndex() {
- return this.idx;
- }
-
- private final int idx;
- };
-
- /**
- * Get the statistic count of this tablet.
- * If the specified tablet doesn't have statistics, 0 will be returned.
- * @param tabletId the tablet's id
- * @param statistic the statistic type to get
- * @return the value of the statistic
- */
- public long getTabletStatistic(String tabletId, Statistic statistic) {
- Slice tabletIdAsSlice = Slices.copiedBuffer(tabletId, Charset.defaultCharset());
- TabletStatistics tabletStatistics = stsMap.get(tabletIdAsSlice);
- if (tabletStatistics == null) {
- return 0;
- } else {
- return tabletStatistics.getStatistic(statistic);
- }
- }
-
- /**
- * Get the statistic count of this table.
- * @param tableName the table's name
- * @param statistic the statistic type to get
- * @return the value of the statistic
- */
- public long getTableStatistic(String tableName, Statistic statistic) {
- long stsResult = 0;
- for (TabletStatistics tabletStatistics : stsMap.values()) {
- if (!tabletStatistics.tableName.equals(tableName)) {
- continue;
- }
- stsResult += tabletStatistics.getStatistic(statistic);
- }
- return stsResult;
- }
-
- /**
- * Get the statistic count of the whole client.
- * @param statistic the statistic type to get
- * @return the value of the statistic
- */
- public long getClientStatistic(Statistic statistic) {
- long stsResult = 0;
- for (TabletStatistics tabletStatistics : stsMap.values()) {
- stsResult += tabletStatistics.getStatistic(statistic);
- }
- return stsResult;
- }
-
- /**
- * Get the set of tablets which have been written into by this client,
- * which have statistics information.
- * @return set of tablet ids
- */
- public Set<String> getTabletSet() {
- Set<String> tablets = Sets.newHashSet();
- for (Slice tablet : stsMap.keySet()) {
- tablets.add(tablet.toString(Charset.defaultCharset()));
- }
- return tablets;
- }
-
- /**
- * Get the set of tables which have been written into by this client,
- * which have statistics information.
- * @return set of table names
- */
- public Set<String> getTableSet() {
- Set<String> tables = Sets.newHashSet();
- for (TabletStatistics tabletStat : stsMap.values()) {
- tables.add(tabletStat.tableName);
- }
- return tables;
- }
-
- /**
- * Get table name of the given tablet id.
- * If the tablet has no statistics, null will be returned.
- * @param tabletId the tablet's id
- * @return table name
- */
- public String getTableName(String tabletId) {
- Slice tabletIdAsSlice = Slices.copiedBuffer(tabletId, Charset.defaultCharset());
- TabletStatistics tabletStatistics = stsMap.get(tabletIdAsSlice);
- if (tabletStatistics == null) {
- return null;
- } else {
- return tabletStatistics.tableName;
- }
- }
-
- /**
- * Get the TabletStatistics object for this specified tablet.
- * @param tableName the table's name
- * @param tabletId the tablet's id
- * @return a TabletStatistics object
- */
- Statistics.TabletStatistics getTabletStatistics(String tableName, Slice tabletId) {
- Statistics.TabletStatistics tabletStats = stsMap.get(tabletId);
- if (tabletStats == null) {
- Statistics.TabletStatistics newTabletStats = new Statistics.TabletStatistics(tableName,
- tabletId.toString(Charset.defaultCharset()));
- tabletStats = stsMap.putIfAbsent(tabletId, newTabletStats);
- if (tabletStats == null) {
- tabletStats = newTabletStats;
- }
- }
- return tabletStats;
- }
-
- @Override
- public String toString() {
- final StringBuilder buf = new StringBuilder();
- buf.append("Current client statistics: ");
- buf.append("bytes written:");
- buf.append(getClientStatistic(Statistic.BYTES_WRITTEN));
- buf.append(", write rpcs:");
- buf.append(getClientStatistic(Statistic.WRITE_RPCS));
- buf.append(", rpc errors:");
- buf.append(getClientStatistic(Statistic.RPC_ERRORS));
- buf.append(", write operations:");
- buf.append(getClientStatistic(Statistic.WRITE_OPS));
- buf.append(", operation errors:");
- buf.append(getClientStatistic(Statistic.OPS_ERRORS));
- return buf.toString();
- }
-
- static class TabletStatistics {
- final private AtomicLongArray statistics;
- final private String tableName;
- final private String tabletId;
-
- TabletStatistics(String tableName, String tabletId) {
- this.tableName = tableName;
- this.tabletId = tabletId;
- this.statistics = new AtomicLongArray(Statistic.values().length);
- }
-
- void incrementStatistic(Statistic statistic, long count) {
- this.statistics.addAndGet(statistic.getIndex(), count);
- }
-
- long getStatistic(Statistic statistic) {
- return this.statistics.get(statistic.getIndex());
- }
-
- public String toString() {
- final StringBuilder buf = new StringBuilder();
- buf.append("Table: ");
- buf.append(tableName);
- buf.append(", tablet:");
- buf.append(tabletId);
- buf.append(", bytes written:");
- buf.append(getStatistic(Statistic.BYTES_WRITTEN));
- buf.append(", write rpcs:");
- buf.append(getStatistic(Statistic.WRITE_RPCS));
- buf.append(", rpc errors:");
- buf.append(getStatistic(Statistic.RPC_ERRORS));
- buf.append(", write operations:");
- buf.append(getStatistic(Statistic.WRITE_OPS));
- buf.append(", operation errors:");
- buf.append(getStatistic(Statistic.OPS_ERRORS));
- return buf.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Status.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Status.java b/java/kudu-client/src/main/java/org/kududb/client/Status.java
deleted file mode 100644
index cd0a17d..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Status.java
+++ /dev/null
@@ -1,373 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.master.Master;
-import org.kududb.tserver.Tserver;
-
-/**
- * Representation of an error code and message.
- * See also {@code src/kudu/util/status.h} in the C++ codebase.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Status {
-
- // Keep a single OK status object else we'll end up instantiating tons of them.
- private static final Status STATIC_OK = new Status(WireProtocol.AppStatusPB.ErrorCode.OK);
-
- private final WireProtocol.AppStatusPB appStatusPB;
-
- private Status(WireProtocol.AppStatusPB appStatusPB) {
- this.appStatusPB = appStatusPB;
- }
-
- private Status(WireProtocol.AppStatusPB.ErrorCode code, String msg, int posixCode) {
- this.appStatusPB =
- WireProtocol.AppStatusPB.newBuilder()
- .setCode(code)
- .setMessage(msg)
- .setPosixCode(posixCode)
- .build();
- }
-
- private Status(WireProtocol.AppStatusPB.ErrorCode code, String msg) {
- this(code, msg, -1);
- }
-
- private Status(WireProtocol.AppStatusPB.ErrorCode code) {
- this(code, "", -1);
- }
-
- // Factory methods.
-
- /**
- * Create a status object from a master error.
- * @param masterErrorPB pb object received via RPC from the master
- * @return status object equivalent to the pb
- */
- static Status fromMasterErrorPB(Master.MasterErrorPB masterErrorPB) {
- if (masterErrorPB == Master.MasterErrorPB.getDefaultInstance()) {
- return Status.OK();
- } else {
- return new Status(masterErrorPB.getStatus());
- }
- }
-
- /**
- * Create a status object from a tablet server error.
- * @param tserverErrorPB pb object received via RPC from the TS
- * @return status object equivalent to the pb
- */
- static Status fromTabletServerErrorPB(Tserver.TabletServerErrorPB tserverErrorPB) {
- if (tserverErrorPB == Tserver.TabletServerErrorPB.getDefaultInstance()) {
- return Status.OK();
- } else {
- return new Status(tserverErrorPB.getStatus());
- }
- }
-
- /**
- * Create a Status object from a {@link WireProtocol.AppStatusPB} protobuf object.
- * Package-private because we shade Protobuf and this is not usable outside this package.
- */
- static Status fromPB(WireProtocol.AppStatusPB pb) {
- return new Status(pb);
- }
-
- public static Status OK() {
- return STATIC_OK;
- }
-
- public static Status NotFound(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND, msg);
- }
- public static Status NotFound(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND, msg, posixCode);
- }
-
- public static Status Corruption(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.CORRUPTION, msg);
- }
- public static Status Corruption(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.CORRUPTION, msg, posixCode);
- }
-
- public static Status NotSupported(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED, msg);
- }
- public static Status NotSupported(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED, msg, posixCode);
- }
-
- public static Status InvalidArgument(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT, msg);
- }
- public static Status InvalidArgument(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT, msg, posixCode);
- }
-
- public static Status IOError(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.IO_ERROR, msg);
- }
- public static Status IOError(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.IO_ERROR, msg, posixCode);
- }
-
- public static Status AlreadyPresent(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT, msg);
- }
- public static Status AlreadyPresent(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT, msg, posixCode);
- }
-
- public static Status RuntimeError(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR, msg);
- }
- public static Status RuntimeError(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR, msg, posixCode);
- }
-
- public static Status NetworkError(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR, msg);
- }
- public static Status NetworkError(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR, msg, posixCode);
- }
-
- public static Status IllegalState(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE, msg);
- }
- public static Status IllegalState(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE, msg, posixCode);
- }
-
- public static Status NotAuthorized(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED, msg);
- }
- public static Status NotAuthorized(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED, msg, posixCode);
- }
-
- public static Status Aborted(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.ABORTED, msg);
- }
- public static Status Aborted(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.ABORTED, msg, posixCode);
- }
-
- public static Status RemoteError(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR, msg);
- }
- public static Status RemoteError(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR, msg, posixCode);
- }
-
- public static Status ServiceUnavailable(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE, msg);
- }
- public static Status ServiceUnavailable(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE, msg, posixCode);
- }
-
- public static Status TimedOut(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT, msg);
- }
- public static Status TimedOut(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT, msg, posixCode);
- }
-
- public static Status Uninitialized(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED, msg);
- }
- public static Status Uninitialized(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED, msg, posixCode);
- }
-
- public static Status ConfigurationError(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR, msg);
- }
- public static Status ConfigurationError(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR, msg, posixCode);
- }
-
- public static Status Incomplete(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE, msg);
- }
- public static Status Incomplete(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE, msg, posixCode);
- }
-
- public static Status EndOfFile(String msg) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE, msg);
- }
- public static Status EndOfFile(String msg, int posixCode) {
- return new Status(WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE, msg, posixCode);
- }
-
- // Boolean status checks.
-
- public boolean ok() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.OK;
- }
- public boolean isCorruption() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.CORRUPTION;
- }
- public boolean isNotFound() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND;
- }
- public boolean isNotSupported() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED;
- }
- public boolean isInvalidArgument() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT;
- }
- public boolean isIOError() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.IO_ERROR;
- }
- public boolean isAlreadyPresent() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT;
- }
- public boolean isRuntimeError() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR;
- }
- public boolean isNetworkError() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR;
- }
- public boolean isIllegalState() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE;
- }
- public boolean isNotAuthorized() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED;
- }
- public boolean isAborted() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.ABORTED;
- }
- public boolean isRemoteError() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR;
- }
- public boolean isServiceUnavailable() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE;
- }
- public boolean isTimedOut() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT;
- }
- public boolean isUninitialized() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED;
- }
- public boolean isConfigurationError() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR;
- }
- public boolean isIncomplete() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE;
- }
- public boolean isEndOfFile() {
- return appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE;
- }
-
- /**
- * Return a human-readable version of the status code.
- * See also status.cc in the C++ codebase.
- */
- private String getCodeAsString() {
- switch (appStatusPB.getCode().getNumber()) {
- case WireProtocol.AppStatusPB.ErrorCode.OK_VALUE:
- return "OK";
- case WireProtocol.AppStatusPB.ErrorCode.NOT_FOUND_VALUE:
- return "Not found";
- case WireProtocol.AppStatusPB.ErrorCode.CORRUPTION_VALUE:
- return "Corruption";
- case WireProtocol.AppStatusPB.ErrorCode.NOT_SUPPORTED_VALUE:
- return "Not implemented";
- case WireProtocol.AppStatusPB.ErrorCode.INVALID_ARGUMENT_VALUE:
- return "Invalid argument";
- case WireProtocol.AppStatusPB.ErrorCode.IO_ERROR_VALUE:
- return "IO error";
- case WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT_VALUE:
- return "Already present";
- case WireProtocol.AppStatusPB.ErrorCode.RUNTIME_ERROR_VALUE:
- return "Runtime error";
- case WireProtocol.AppStatusPB.ErrorCode.NETWORK_ERROR_VALUE:
- return "Network error";
- case WireProtocol.AppStatusPB.ErrorCode.ILLEGAL_STATE_VALUE:
- return "Illegal state";
- case WireProtocol.AppStatusPB.ErrorCode.NOT_AUTHORIZED_VALUE:
- return "Not authorized";
- case WireProtocol.AppStatusPB.ErrorCode.ABORTED_VALUE:
- return "Aborted";
- case WireProtocol.AppStatusPB.ErrorCode.REMOTE_ERROR_VALUE:
- return "Remote error";
- case WireProtocol.AppStatusPB.ErrorCode.SERVICE_UNAVAILABLE_VALUE:
- return "Service unavailable";
- case WireProtocol.AppStatusPB.ErrorCode.TIMED_OUT_VALUE:
- return "Timed out";
- case WireProtocol.AppStatusPB.ErrorCode.UNINITIALIZED_VALUE:
- return "Uninitialized";
- case WireProtocol.AppStatusPB.ErrorCode.CONFIGURATION_ERROR_VALUE:
- return "Configuration error";
- case WireProtocol.AppStatusPB.ErrorCode.INCOMPLETE_VALUE:
- return "Incomplete";
- case WireProtocol.AppStatusPB.ErrorCode.END_OF_FILE_VALUE:
- return "End of file";
- default:
- return "Unknown error (" + appStatusPB.getCode().getNumber() + ")";
- }
- }
-
- /**
- * Get the posix code associated with the error.
- * @return {@code -1} if no posix code is set. Otherwise, returns the posix code.
- */
- public int getPosixCode() {
- return appStatusPB.getPosixCode();
- }
-
- /**
- * Get enum code name.
- * Intended for internal use only.
- */
- String getCodeName() {
- return appStatusPB.getCode().name();
- }
-
- /**
- * Returns string error message.
- * Intended for internal use only.
- */
- String getMessage() {
- return appStatusPB.getMessage();
- }
-
- /**
- * Get a human-readable version of the Status message fit for logging or display.
- */
- @Override
- public String toString() {
- String str = getCodeAsString();
- if (appStatusPB.getCode() == WireProtocol.AppStatusPB.ErrorCode.OK) {
- return str;
- }
- str = String.format("%s: %s", str, appStatusPB.getMessage());
- if (appStatusPB.getPosixCode() != -1) {
- str = String.format("%s (error %d)", str, appStatusPB.getPosixCode());
- }
- return str;
- }
-}
[08/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/ITClient.java b/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
deleted file mode 100644
index cb8e968..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/ITClient.java
+++ /dev/null
@@ -1,396 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Integration test for the client. RPCs are sent to Kudu from multiple threads while processes
- * are restarted and failures are injected.
- *
- * By default this test runs for 60 seconds, but this can be changed by passing a different value
- * in "itclient.runtime.seconds". For example:
- * "mvn test -Dtest=ITClient -Ditclient.runtime.seconds=120".
- */
-public class ITClient extends BaseKuduTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(ITClient.class);
-
- private static final String RUNTIME_PROPERTY_NAME = "itclient.runtime.seconds";
- private static final long DEFAULT_RUNTIME_SECONDS = 60;
- // Time we'll spend waiting at the end of the test for things to settle. Also the minimum this
- // test can run for.
- private static final long TEST_MIN_RUNTIME_SECONDS = 2;
- private static final long TEST_TIMEOUT_SECONDS = 600000;
-
- private static final String TABLE_NAME =
- ITClient.class.getName() + "-" + System.currentTimeMillis();
- // One error and we stop the test.
- private static final CountDownLatch KEEP_RUNNING_LATCH = new CountDownLatch(1);
- // Latch used to track if an error occurred and we need to stop the test early.
- private static final CountDownLatch ERROR_LATCH = new CountDownLatch(1);
-
- private static KuduClient localClient;
- private static AsyncKuduClient localAsyncClient;
- private static KuduTable table;
- private static long runtimeInSeconds;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
-
- String runtimeProp = System.getProperty(RUNTIME_PROPERTY_NAME);
- runtimeInSeconds = runtimeProp == null ? DEFAULT_RUNTIME_SECONDS : Long.parseLong(runtimeProp);
-
- if (runtimeInSeconds < TEST_MIN_RUNTIME_SECONDS || runtimeInSeconds > TEST_TIMEOUT_SECONDS) {
- Assert.fail("This test needs to run more more than " + TEST_MIN_RUNTIME_SECONDS + " seconds" +
- " and less than " + TEST_TIMEOUT_SECONDS + " seconds");
- }
-
- LOG.info ("Test running for {} seconds", runtimeInSeconds);
-
- BaseKuduTest.setUpBeforeClass();
-
- // Client we're using has low tolerance for read timeouts but a
- // higher overall operation timeout.
- localAsyncClient = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
- .defaultSocketReadTimeoutMs(500)
- .build();
- localClient = new KuduClient(localAsyncClient);
-
- CreateTableOptions builder = new CreateTableOptions().setNumReplicas(3);
- builder.setRangePartitionColumns(ImmutableList.of("key"));
- table = localClient.createTable(TABLE_NAME, basicSchema, builder);
- }
-
- @Test(timeout = TEST_TIMEOUT_SECONDS)
- public void test() throws Exception {
- ArrayList<Thread> threads = new ArrayList<>();
- Thread chaosThread = new Thread(new ChaosThread());
- Thread writerThread = new Thread(new WriterThread());
- Thread scannerThread = new Thread(new ScannerThread());
-
- threads.add(chaosThread);
- threads.add(writerThread);
- threads.add(scannerThread);
-
- for (Thread thread : threads) {
- thread.start();
- }
-
- // await() returns yes if the latch reaches 0, we don't want that.
- Assert.assertFalse("Look for the last ERROR line in the log that comes from ITCLient",
- ERROR_LATCH.await(runtimeInSeconds, TimeUnit.SECONDS));
-
- // Indicate we want to stop, then wait a little bit for it to happen.
- KEEP_RUNNING_LATCH.countDown();
-
- for (Thread thread : threads) {
- thread.interrupt();
- thread.join();
- }
-
- AsyncKuduScanner scannerBuilder = localAsyncClient.newScannerBuilder(table).build();
- int rowCount = countRowsInScan(scannerBuilder);
- Assert.assertTrue(rowCount + " should be higher than 0", rowCount > 0);
- }
-
- /**
- * Logs an error message and triggers the error count down latch, stopping this test.
- * @param message error message to print
- * @param exception optional exception to print
- */
- private void reportError(String message, Exception exception) {
- LOG.error(message, exception);
- ERROR_LATCH.countDown();
- }
-
- /**
- * Thread that introduces chaos in the cluster, one at a time.
- */
- class ChaosThread implements Runnable {
-
- private final Random random = new Random();
-
- @Override
- public void run() {
- try {
- KEEP_RUNNING_LATCH.await(2, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- return;
- }
- while (KEEP_RUNNING_LATCH.getCount() > 0) {
- try {
- boolean shouldContinue;
- if (System.currentTimeMillis() % 2 == 0) {
- shouldContinue = restartTS();
- } else {
-
- shouldContinue = disconnectNode();
- }
- // TODO restarting the master currently finds more bugs. Also, adding it to the list makes
- // it necessary to find a new weighing mechanism betweent he different chaos options.
- // shouldContinue = restartMaster();
-
- if (!shouldContinue) {
- return;
- }
- KEEP_RUNNING_LATCH.await(5, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- return;
- }
-
- }
- }
-
- /**
- * Failure injection. Picks a random tablet server from the client's cache and force
- * disconects it.
- * @return true if successfully completed or didn't find a server to disconnect, false it it
- * encountered a failure
- */
- private boolean disconnectNode() {
- try {
- if (localAsyncClient.getTabletClients().size() == 0) {
- return true;
- }
-
- int tsToDisconnect = random.nextInt(localAsyncClient.getTabletClients().size());
- localAsyncClient.getTabletClients().get(tsToDisconnect).disconnect();
-
- } catch (Exception e) {
- if (KEEP_RUNNING_LATCH.getCount() == 0) {
- // Likely shutdown() related.
- return false;
- }
- reportError("Couldn't disconnect a TS", e);
- return false;
- }
- return true;
- }
-
- /**
- * Forces the restart of a random tablet server.
- * @return true if it successfully completed, false if it failed
- */
- private boolean restartTS() {
- try {
- BaseKuduTest.restartTabletServer(table);
- } catch (Exception e) {
- reportError("Couldn't restart a TS", e);
- return false;
- }
- return true;
- }
-
- /**
- * Forces the restart of the master.
- * @return true if it successfully completed, false if it failed
- */
- private boolean restartMaster() {
- try {
- BaseKuduTest.restartLeaderMaster();
- } catch (Exception e) {
- reportError("Couldn't restart a master", e);
- return false;
- }
- return true;
- }
-
- }
-
- /**
- * Thread that writes sequentially to the table. Every 10 rows it considers setting the flush mode
- * to MANUAL_FLUSH or AUTO_FLUSH_SYNC.
- */
- class WriterThread implements Runnable {
-
- private final KuduSession session = localClient.newSession();
- private final Random random = new Random();
- private int currentRowKey = 0;
-
- @Override
- public void run() {
- while (KEEP_RUNNING_LATCH.getCount() > 0) {
- try {
- OperationResponse resp = session.apply(createBasicSchemaInsert(table, currentRowKey));
- if (hasRowErrorAndReport(resp)) {
- return;
- }
- currentRowKey++;
-
- // Every 10 rows we flush and change the flush mode randomly.
- if (currentRowKey % 10 == 0) {
-
- // First flush any accumulated rows before switching.
- List<OperationResponse> responses = session.flush();
- if (responses != null) {
- for (OperationResponse batchedResp : responses) {
- if (hasRowErrorAndReport(batchedResp)) {
- return;
- }
- }
- }
-
- if (random.nextBoolean()) {
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- } else {
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
- }
- }
- } catch (Exception e) {
- if (KEEP_RUNNING_LATCH.getCount() == 0) {
- // Likely shutdown() related.
- return;
- }
- reportError("Got error while inserting row " + currentRowKey, e);
- return;
- }
- }
- }
-
- private boolean hasRowErrorAndReport(OperationResponse resp) {
- if (resp != null && resp.hasRowError()) {
- reportError("The following RPC " + resp.getOperation().getRow() +
- " returned this error: " + resp.getRowError(), null);
- return true;
- }
- return false;
- }
- }
-
- /**
- * Thread that scans the table. Alternates randomly between random gets and full table scans.
- */
- class ScannerThread implements Runnable {
-
- private final Random random = new Random();
-
- // Updated by calling a full scan.
- private int lastRowCount = 0;
-
- @Override
- public void run() {
- while (KEEP_RUNNING_LATCH.getCount() > 0) {
-
- boolean shouldContinue;
-
- // Always scan until we find rows.
- if (lastRowCount == 0 || random.nextBoolean()) {
- shouldContinue = fullScan();
- } else {
- shouldContinue = randomGet();
- }
-
- if (!shouldContinue) {
- return;
- }
-
- if (lastRowCount == 0) {
- try {
- KEEP_RUNNING_LATCH.await(50, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- // Test is stopping.
- return;
- }
- }
- }
- }
-
- /**
- * Reads a row at random that it knows to exist (smaller than lastRowCount).
- * @return
- */
- private boolean randomGet() {
- int key = random.nextInt(lastRowCount);
- KuduPredicate predicate = KuduPredicate.newComparisonPredicate(
- table.getSchema().getColumnByIndex(0), KuduPredicate.ComparisonOp.EQUAL, key);
- KuduScanner scanner = localClient.newScannerBuilder(table).addPredicate(predicate).build();
-
- List<RowResult> results = new ArrayList<>();
- while (scanner.hasMoreRows()) {
- try {
- RowResultIterator ite = scanner.nextRows();
- for (RowResult row : ite) {
- results.add(row);
- }
- } catch (Exception e) {
- return checkAndReportError("Got error while getting row " + key, e);
- }
- }
-
- if (results.isEmpty() || results.size() > 1) {
- reportError("Random get got 0 or many rows " + results.size() + " for key " + key, null);
- return false;
- }
-
- int receivedKey = results.get(0).getInt(0);
- if (receivedKey != key) {
- reportError("Tried to get key " + key + " and received " + receivedKey, null);
- return false;
- }
- return true;
- }
-
- /**
- * Rusn a full table scan and updates the lastRowCount.
- * @return
- */
- private boolean fullScan() {
- AsyncKuduScanner scannerBuilder = localAsyncClient.newScannerBuilder(table).build();
- try {
- int rowCount = countRowsInScan(scannerBuilder);
- if (rowCount < lastRowCount) {
- reportError("Row count regressed: " + rowCount + " < " + lastRowCount, null);
- return false;
- }
- lastRowCount = rowCount;
- LOG.info("New row count {}", lastRowCount);
- } catch (Exception e) {
- checkAndReportError("Got error while row counting", e);
- }
- return true;
- }
-
- /**
- * Checks the passed exception contains "Scanner not found". If it does then it returns true,
- * else it reports the error and returns false.
- * We need to do this because the scans in this client aren't fault tolerant.
- * @param message message to print if the exception contains a real error
- * @param e the exception to check
- * @return true if the scanner failed because it wasn't false, otherwise false
- */
- private boolean checkAndReportError(String message, Exception e) {
- if (!e.getCause().getMessage().contains("Scanner not found")) {
- reportError(message, e);
- return false;
- }
- return true;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/ITScannerMultiTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/ITScannerMultiTablet.java b/java/kudu-client/src/test/java/org/kududb/client/ITScannerMultiTablet.java
deleted file mode 100644
index 13465f9..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/ITScannerMultiTablet.java
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.Lists;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.kududb.Schema;
-
-import java.util.Random;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Integration test that inserts enough data to trigger flushes and getting multiple data
- * blocks.
- */
-public class ITScannerMultiTablet extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- ITScannerMultiTablet.class.getName()+"-"+System.currentTimeMillis();
- private static final int ROW_COUNT = 20000;
- private static final int TABLET_COUNT = 3;
-
- private static Schema schema = getBasicSchema();
- private static KuduTable table;
-
- private static Random random = new Random(1234);
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
-
- CreateTableOptions builder = new CreateTableOptions();
-
- builder.addHashPartitions(
- Lists.newArrayList(schema.getColumnByIndex(0).getName()),
- TABLET_COUNT);
-
- table = createTable(TABLE_NAME, schema, builder);
-
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
-
- // Getting meaty rows.
- char[] chars = new char[1024];
- for (int i = 0; i < ROW_COUNT; i++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addInt(0, random.nextInt());
- row.addInt(1, i);
- row.addInt(2, i);
- row.addString(3, new String(chars));
- row.addBoolean(4, true);
- session.apply(insert);
- }
- session.flush();
- assertEquals(0, session.countPendingErrors());
- }
-
- /**
- * Test for KUDU-1343 with a multi-batch multi-tablet scan.
- */
- @Test(timeout = 100000)
- public void testKudu1343() throws Exception {
- KuduScanner scanner = syncClient.newScannerBuilder(table)
- .batchSizeBytes(1) // Just a hint, won't actually be that small
- .build();
-
- int rowCount = 0;
- int loopCount = 0;
- while(scanner.hasMoreRows()) {
- loopCount++;
- RowResultIterator rri = scanner.nextRows();
- while (rri.hasNext()) {
- rri.next();
- rowCount++;
- }
- }
-
- assertTrue(loopCount > TABLET_COUNT);
- assertEquals(ROW_COUNT, rowCount);
- }
-
- /**
- * Makes sure we pass all the correct information down to the server by verifying we get rows in
- * order from 4 tablets. We detect those tablet boundaries when keys suddenly become smaller than
- * what was previously seen.
- */
- @Test(timeout = 100000)
- public void testSortResultsByPrimaryKey() throws Exception {
- KuduScanner scanner = syncClient.newScannerBuilder(table)
- .sortResultsByPrimaryKey()
- .setProjectedColumnIndexes(Lists.newArrayList(0))
- .build();
-
- int rowCount = 0;
- int previousRow = -1;
- int tableBoundariesCount = 0;
- while(scanner.hasMoreRows()) {
- RowResultIterator rri = scanner.nextRows();
- while (rri.hasNext()) {
- int key = rri.next().getInt(0);
- if (key < previousRow) {
- tableBoundariesCount++;
- }
- previousRow = key;
- rowCount++;
- }
- }
- assertEquals(ROW_COUNT, rowCount);
- assertEquals(TABLET_COUNT, tableBoundariesCount);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java b/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
deleted file mode 100644
index 955e6ab..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/MiniKuduCluster.java
+++ /dev/null
@@ -1,474 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.net.HostAndPort;
-import org.apache.commons.io.FileUtils;
-import org.kududb.util.NetUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Utility class to start and manipulate Kudu clusters. Relies on being IN the Kudu source code with
- * both the kudu-master and kudu-tserver binaries already compiled. {@link BaseKuduTest} should be
- * extended instead of directly using this class in almost all cases.
- */
-public class MiniKuduCluster implements AutoCloseable {
-
- private static final Logger LOG = LoggerFactory.getLogger(MiniKuduCluster.class);
-
- // TS and Master ports will be assigned starting with this one.
- private static final int PORT_START = 64030;
-
- // List of threads that print
- private final List<Thread> PROCESS_INPUT_PRINTERS = new ArrayList<>();
-
- // Map of ports to master servers.
- private final Map<Integer, Process> masterProcesses = new ConcurrentHashMap<>();
-
- // Map of ports to tablet servers.
- private final Map<Integer, Process> tserverProcesses = new ConcurrentHashMap<>();
-
- // Map of ports to process command lines. Never removed from. Used to restart processes.
- private final Map<Integer, String[]> commandLines = new ConcurrentHashMap<>();
-
- private final List<String> pathsToDelete = new ArrayList<>();
- private final List<HostAndPort> masterHostPorts = new ArrayList<>();
- private List<Integer> tserverPorts = new ArrayList<>();
-
- // Client we can use for common operations.
- private final KuduClient syncClient;
- private final int defaultTimeoutMs;
-
- private String masterAddresses;
-
- private MiniKuduCluster(int numMasters, int numTservers, int defaultTimeoutMs) throws Exception {
- this.defaultTimeoutMs = defaultTimeoutMs;
-
- startCluster(numMasters, numTservers);
-
- syncClient = new KuduClient.KuduClientBuilder(getMasterAddresses())
- .defaultAdminOperationTimeoutMs(defaultTimeoutMs)
- .defaultOperationTimeoutMs(defaultTimeoutMs)
- .build();
- }
-
- /**
- * Wait up to this instance's "default timeout" for an expected count of TS to
- * connect to the master.
- * @param expected How many TS are expected
- * @return true if there are at least as many TS as expected, otherwise false
- */
- public boolean waitForTabletServers(int expected) throws Exception {
- int count = 0;
- Stopwatch stopwatch = Stopwatch.createStarted();
- while (count < expected && stopwatch.elapsed(TimeUnit.MILLISECONDS) < defaultTimeoutMs) {
- Thread.sleep(200);
- count = syncClient.listTabletServers().getTabletServersCount();
- }
- return count >= expected;
- }
-
- /**
- * Starts a Kudu cluster composed of the provided masters and tablet servers.
- * @param numMasters how many masters to start
- * @param numTservers how many tablet servers to start
- * @throws Exception
- */
- private void startCluster(int numMasters, int numTservers) throws Exception {
- Preconditions.checkArgument(numMasters > 0, "Need at least one master");
- Preconditions.checkArgument(numTservers > 0, "Need at least one tablet server");
- // The following props are set via kudu-client's pom.
- String baseDirPath = TestUtils.getBaseDir();
- String localhost = TestUtils.getUniqueLocalhost();
-
- long now = System.currentTimeMillis();
- LOG.info("Starting {} masters...", numMasters);
- int startPort = startMasters(PORT_START, numMasters, baseDirPath);
- LOG.info("Starting {} tablet servers...", numTservers);
- List<Integer> ports = TestUtils.findFreePorts(startPort, numTservers * 2);
- for (int i = 0; i < numTservers; i++) {
- int rpcPort = ports.get(i * 2);
- tserverPorts.add(rpcPort);
- String dataDirPath = baseDirPath + "/ts-" + i + "-" + now;
- String flagsPath = TestUtils.getFlagsPath();
- String[] tsCmdLine = {
- TestUtils.findBinary("kudu-tserver"),
- "--flagfile=" + flagsPath,
- "--fs_wal_dir=" + dataDirPath,
- "--fs_data_dirs=" + dataDirPath,
- "--flush_threshold_mb=1",
- "--enable_exactly_once",
- "--tserver_master_addrs=" + masterAddresses,
- "--webserver_interface=" + localhost,
- "--local_ip_for_outbound_sockets=" + localhost,
- "--webserver_port=" + (rpcPort + 1),
- "--rpc_bind_addresses=" + localhost + ":" + rpcPort};
- tserverProcesses.put(rpcPort, configureAndStartProcess(rpcPort, tsCmdLine));
- commandLines.put(rpcPort, tsCmdLine);
-
- if (flagsPath.startsWith(baseDirPath)) {
- // We made a temporary copy of the flags; delete them later.
- pathsToDelete.add(flagsPath);
- }
- pathsToDelete.add(dataDirPath);
- }
- }
-
- /**
- * Start the specified number of master servers with ports starting from a specified
- * number. Finds free web and RPC ports up front for all of the masters first, then
- * starts them on those ports, populating 'masters' map.
- * @param masterStartPort the starting point of the port range for the masters
- * @param numMasters number of masters to start
- * @param baseDirPath the base directory where the mini cluster stores its data
- * @return the next free port
- * @throws Exception if we are unable to start the masters
- */
- private int startMasters(int masterStartPort, int numMasters,
- String baseDirPath) throws Exception {
- LOG.info("Starting {} masters...", numMasters);
- // Get the list of web and RPC ports to use for the master consensus configuration:
- // request NUM_MASTERS * 2 free ports as we want to also reserve the web
- // ports for the consensus configuration.
- String localhost = TestUtils.getUniqueLocalhost();
- List<Integer> ports = TestUtils.findFreePorts(masterStartPort, numMasters * 2);
- int lastFreePort = ports.get(ports.size() - 1);
- List<Integer> masterRpcPorts = Lists.newArrayListWithCapacity(numMasters);
- List<Integer> masterWebPorts = Lists.newArrayListWithCapacity(numMasters);
- for (int i = 0; i < numMasters * 2; i++) {
- if (i % 2 == 0) {
- masterRpcPorts.add(ports.get(i));
- masterHostPorts.add(HostAndPort.fromParts(localhost, ports.get(i)));
- } else {
- masterWebPorts.add(ports.get(i));
- }
- }
- masterAddresses = NetUtil.hostsAndPortsToString(masterHostPorts);
- long now = System.currentTimeMillis();
- for (int i = 0; i < numMasters; i++) {
- int port = masterRpcPorts.get(i);
- String dataDirPath = baseDirPath + "/master-" + i + "-" + now;
- String flagsPath = TestUtils.getFlagsPath();
- // The web port must be reserved in the call to findFreePorts above and specified
- // to avoid the scenario where:
- // 1) findFreePorts finds RPC ports a, b, c for the 3 masters.
- // 2) start master 1 with RPC port and let it bind to any (specified as 0) web port.
- // 3) master 1 happens to bind to port b for the web port, as master 2 hasn't been
- // started yet and findFreePort(s) is "check-time-of-use" (it does not reserve the
- // ports, only checks that when it was last called, these ports could be used).
- List<String> masterCmdLine = Lists.newArrayList(
- TestUtils.findBinary("kudu-master"),
- "--flagfile=" + flagsPath,
- "--fs_wal_dir=" + dataDirPath,
- "--fs_data_dirs=" + dataDirPath,
- "--webserver_interface=" + localhost,
- "--local_ip_for_outbound_sockets=" + localhost,
- "--rpc_bind_addresses=" + localhost + ":" + port,
- "--webserver_port=" + masterWebPorts.get(i));
- if (numMasters > 1) {
- masterCmdLine.add("--master_addresses=" + masterAddresses);
- }
- String[] commandLine = masterCmdLine.toArray(new String[masterCmdLine.size()]);
- masterProcesses.put(port, configureAndStartProcess(port, commandLine));
- commandLines.put(port, commandLine);
-
- if (flagsPath.startsWith(baseDirPath)) {
- // We made a temporary copy of the flags; delete them later.
- pathsToDelete.add(flagsPath);
- }
- pathsToDelete.add(dataDirPath);
- }
- return lastFreePort + 1;
- }
-
- /**
- * Starts a process using the provided command and configures it to be daemon,
- * redirects the stderr to stdout, and starts a thread that will read from the process' input
- * stream and redirect that to LOG.
- * @param port rpc port used to identify the process
- * @param command process and options
- * @return The started process
- * @throws Exception Exception if an error prevents us from starting the process,
- * or if we were able to start the process but noticed that it was then killed (in which case
- * we'll log the exit value).
- */
- private Process configureAndStartProcess(int port, String[] command) throws Exception {
- LOG.info("Starting process: {}", Joiner.on(" ").join(command));
- ProcessBuilder processBuilder = new ProcessBuilder(command);
- processBuilder.redirectErrorStream(true);
- Process proc = processBuilder.start();
- ProcessInputStreamLogPrinterRunnable printer =
- new ProcessInputStreamLogPrinterRunnable(proc.getInputStream());
- Thread thread = new Thread(printer);
- thread.setDaemon(true);
- thread.setName(Iterables.getLast(Splitter.on(File.separatorChar).split(command[0])) + ":" + port);
- PROCESS_INPUT_PRINTERS.add(thread);
- thread.start();
-
- Thread.sleep(300);
- try {
- int ev = proc.exitValue();
- throw new Exception("We tried starting a process (" + command[0] + ") but it exited with " +
- "value=" + ev);
- } catch (IllegalThreadStateException ex) {
- // This means the process is still alive, it's like reverse psychology.
- }
- return proc;
- }
-
- /**
- * Starts a previously killed master process on the specified port.
- * @param port which port the master was listening on for RPCs
- * @throws Exception
- */
- public void restartDeadMasterOnPort(int port) throws Exception {
- restartDeadProcessOnPort(port, masterProcesses);
- }
-
- /**
- * Starts a previously killed tablet server process on the specified port.
- * @param port which port the TS was listening on for RPCs
- * @throws Exception
- */
- public void restartDeadTabletServerOnPort(int port) throws Exception {
- restartDeadProcessOnPort(port, tserverProcesses);
- }
-
- private void restartDeadProcessOnPort(int port, Map<Integer, Process> map) throws Exception {
- if (!commandLines.containsKey(port)) {
- String message = "Cannot start process on unknown port " + port;
- LOG.warn(message);
- throw new RuntimeException(message);
- }
-
- if (map.containsKey(port)) {
- String message = "Process already exists on port " + port;
- LOG.warn(message);
- throw new RuntimeException(message);
- }
-
- String[] commandLine = commandLines.get(port);
- map.put(port, configureAndStartProcess(port, commandLine));
- }
-
- /**
- * Kills the TS listening on the provided port. Doesn't do anything if the TS was already killed.
- * @param port port on which the tablet server is listening on
- * @throws InterruptedException
- */
- public void killTabletServerOnPort(int port) throws InterruptedException {
- Process ts = tserverProcesses.remove(port);
- if (ts == null) {
- // The TS is already dead, good.
- return;
- }
- LOG.info("Killing server at port " + port);
- ts.destroy();
- ts.waitFor();
- }
-
- /**
- * Kills all tablet servers.
- * @throws InterruptedException
- */
- public void killTabletServers() throws InterruptedException {
- for (Process tserver : tserverProcesses.values()) {
- tserver.destroy();
- tserver.waitFor();
- }
- tserverProcesses.clear();
- }
-
- /**
- * Restarts the dead tablet servers on the port.
- * @throws Exception
- */
- public void restartDeadTabletServers() throws Exception {
- for (int port : tserverPorts) {
- restartDeadTabletServerOnPort(port);
- }
- }
-
- /**
- * Kills the master listening on the provided port. Doesn't do anything if the master was
- * already killed.
- * @param port port on which the master is listening on
- * @throws InterruptedException
- */
- public void killMasterOnPort(int port) throws InterruptedException {
- Process master = masterProcesses.remove(port);
- if (master == null) {
- // The master is already dead, good.
- return;
- }
- LOG.info("Killing master at port " + port);
- master.destroy();
- master.waitFor();
- }
-
- /**
- * See {@link #shutdown()}.
- * @throws Exception never thrown, exceptions are logged
- */
- @Override
- public void close() throws Exception {
- shutdown();
- }
-
- /**
- * Stops all the processes and deletes the folders used to store data and the flagfile.
- */
- public void shutdown() {
- for (Iterator<Process> masterIter = masterProcesses.values().iterator(); masterIter.hasNext(); ) {
- masterIter.next().destroy();
- masterIter.remove();
- }
- for (Iterator<Process> tsIter = tserverProcesses.values().iterator(); tsIter.hasNext(); ) {
- tsIter.next().destroy();
- tsIter.remove();
- }
- for (Thread thread : PROCESS_INPUT_PRINTERS) {
- thread.interrupt();
- }
-
- for (String path : pathsToDelete) {
- try {
- File f = new File(path);
- if (f.isDirectory()) {
- FileUtils.deleteDirectory(f);
- } else {
- f.delete();
- }
- } catch (Exception e) {
- LOG.warn("Could not delete path {}", path, e);
- }
- }
- }
-
- /**
- * Returns the comma-separated list of master addresses.
- * @return master addresses
- */
- public String getMasterAddresses() {
- return masterAddresses;
- }
-
- /**
- * Returns a list of master addresses.
- * @return master addresses
- */
- public List<HostAndPort> getMasterHostPorts() {
- return masterHostPorts;
- }
-
- /**
- * Returns an unmodifiable map of all tablet servers in pairs of RPC port - > Process.
- * @return an unmodifiable map of all tablet servers
- */
- @VisibleForTesting
- Map<Integer, Process> getTabletServerProcesses() {
- return Collections.unmodifiableMap(tserverProcesses);
- }
-
- /**
- * Returns an unmodifiable map of all masters in pairs of RPC port - > Process.
- * @return an unmodifiable map of all masters
- */
- @VisibleForTesting
- Map<Integer, Process> getMasterProcesses() {
- return Collections.unmodifiableMap(masterProcesses);
- }
-
- /**
- * Helper runnable that receives stdout and logs it along with the process' identifier.
- */
- private static class ProcessInputStreamLogPrinterRunnable implements Runnable {
-
- private final InputStream is;
-
- public ProcessInputStreamLogPrinterRunnable(InputStream is) {
- this.is = is;
- }
-
- @Override
- public void run() {
- try {
- String line;
- BufferedReader in = new BufferedReader(new InputStreamReader(is));
- while ((line = in.readLine()) != null) {
- LOG.info(line);
- }
- in.close();
- }
- catch (Exception e) {
- if (!e.getMessage().contains("Stream closed")) {
- LOG.error("Caught error while reading a process' output", e);
- }
- }
- }
- }
-
- public static class MiniKuduClusterBuilder {
-
- private int numMasters = 1;
- private int numTservers = 3;
- private int defaultTimeoutMs = 50000;
-
- public MiniKuduClusterBuilder numMasters(int numMasters) {
- this.numMasters = numMasters;
- return this;
- }
-
- public MiniKuduClusterBuilder numTservers(int numTservers) {
- this.numTservers = numTservers;
- return this;
- }
-
- /**
- * Configures the internal client to use the given timeout for all operations. Also uses the
- * timeout for tasks like waiting for tablet servers to check in with the master.
- * @param defaultTimeoutMs timeout in milliseconds
- * @return this instance
- */
- public MiniKuduClusterBuilder defaultTimeoutMs(int defaultTimeoutMs) {
- this.defaultTimeoutMs = defaultTimeoutMs;
- return this;
- }
-
- public MiniKuduCluster build() throws Exception {
- return new MiniKuduCluster(numMasters, numTservers, defaultTimeoutMs);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java b/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
deleted file mode 100644
index abec53f..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
+++ /dev/null
@@ -1,157 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Stopwatch;
-import com.google.protobuf.ByteString;
-import com.stumbleupon.async.Deferred;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.kududb.Common;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.*;
-
-public class TestAsyncKuduClient extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(TestAsyncKuduClient.class);
-
- private static final String TABLE_NAME =
- TestAsyncKuduClient.class.getName() + "-" + System.currentTimeMillis();
- private static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- // Set to 1 for testDisconnect to always test disconnecting the right server.
- CreateTableOptions options = getBasicCreateTableOptions().setNumReplicas(1);
- table = createTable(TABLE_NAME, basicSchema, options);
- }
-
- @Test(timeout = 100000)
- public void testDisconnect() throws Exception {
- // Test that we can reconnect to a TS after a disconnection.
- // 1. Warm up the cache.
- assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
-
- // 2. Disconnect the client.
- disconnectAndWait();
-
- // 3. Count again, it will trigger a re-connection and we should not hang or fail to scan.
- assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
-
- // Test that we can reconnect to a TS while scanning.
- // 1. Insert enough rows to have to call next() multiple times.
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
- int rowCount = 200;
- for (int i = 0; i < rowCount; i++) {
- session.apply(createBasicSchemaInsert(table, i));
- }
- session.flush();
-
- // 2. Start a scanner with a small max num bytes.
- AsyncKuduScanner scanner = client.newScannerBuilder(table)
- .batchSizeBytes(1)
- .build();
- Deferred<RowResultIterator> rri = scanner.nextRows();
- // 3. Register the number of rows we get back. We have no control over how many rows are
- // returned. When this test was written we were getting 100 rows back.
- int numRows = rri.join(DEFAULT_SLEEP).getNumRows();
- assertNotEquals("The TS sent all the rows back, we can't properly test disconnection",
- rowCount, numRows);
-
- // 4. Disconnect the client.
- disconnectAndWait();
-
- // 5. Make sure that we can continue scanning and that we get the remaining rows back.
- assertEquals(rowCount - numRows, countRowsInScan(scanner));
- }
-
- private void disconnectAndWait() throws InterruptedException {
- for (TabletClient tabletClient : client.getTabletClients()) {
- tabletClient.disconnect();
- }
- Stopwatch sw = Stopwatch.createStarted();
- while (sw.elapsed(TimeUnit.MILLISECONDS) < DEFAULT_SLEEP) {
- if (!client.getTabletClients().isEmpty()) {
- Thread.sleep(50);
- } else {
- break;
- }
- }
- assertTrue(client.getTabletClients().isEmpty());
- }
-
- @Test
- public void testBadHostnames() throws Exception {
- String badHostname = "some-unknown-host-hopefully";
-
- // Test that a bad hostname for the master makes us error out quickly.
- AsyncKuduClient invalidClient = new AsyncKuduClient.AsyncKuduClientBuilder(badHostname).build();
- try {
- invalidClient.listTabletServers().join(1000);
- fail("This should have failed quickly");
- } catch (Exception ex) {
- assertTrue(ex instanceof NonRecoverableException);
- assertTrue(ex.getMessage().contains(badHostname));
- }
-
- List<Master.TabletLocationsPB> tabletLocations = new ArrayList<>();
-
- // Builder three bad locations.
- Master.TabletLocationsPB.Builder tabletPb = Master.TabletLocationsPB.newBuilder();
- for (int i = 0; i < 3; i++) {
- Common.PartitionPB.Builder partition = Common.PartitionPB.newBuilder();
- partition.setPartitionKeyStart(ByteString.copyFrom("a" + i, Charsets.UTF_8.name()));
- partition.setPartitionKeyEnd(ByteString.copyFrom("b" + i, Charsets.UTF_8.name()));
- tabletPb.setPartition(partition);
- tabletPb.setTabletId(ByteString.copyFromUtf8("some id " + i));
- Master.TSInfoPB.Builder tsInfoBuilder = Master.TSInfoPB.newBuilder();
- Common.HostPortPB.Builder hostBuilder = Common.HostPortPB.newBuilder();
- hostBuilder.setHost(badHostname + i);
- hostBuilder.setPort(i);
- tsInfoBuilder.addRpcAddresses(hostBuilder);
- tsInfoBuilder.setPermanentUuid(ByteString.copyFromUtf8("some uuid"));
- Master.TabletLocationsPB.ReplicaPB.Builder replicaBuilder =
- Master.TabletLocationsPB.ReplicaPB.newBuilder();
- replicaBuilder.setTsInfo(tsInfoBuilder);
- replicaBuilder.setRole(Metadata.RaftPeerPB.Role.FOLLOWER);
- tabletPb.addReplicas(replicaBuilder);
- tabletLocations.add(tabletPb.build());
- }
-
- // Test that a tablet full of unreachable replicas won't make us retry.
- try {
- KuduTable badTable = new KuduTable(client, "Invalid table name",
- "Invalid table ID", null, null);
- client.discoverTablets(badTable, tabletLocations);
- fail("This should have failed quickly");
- } catch (Exception ex) {
- assertTrue(ex instanceof NonRecoverableException);
- assertTrue(ex.getMessage().contains(badHostname));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduSession.java b/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduSession.java
deleted file mode 100644
index ba69305..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduSession.java
+++ /dev/null
@@ -1,514 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.Schema;
-import org.kududb.WireProtocol.AppStatusPB;
-import org.kududb.client.AsyncKuduClient.RemoteTablet;
-import org.kududb.tserver.Tserver.TabletServerErrorPB;
-
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-import com.stumbleupon.async.DeferredGroupException;
-import com.stumbleupon.async.TimeoutException;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- * This class can either start its own cluster or rely on an existing one.
- * By default it assumes that the master is at localhost:64000.
- * The cluster's configuration flags is found at flagsPath as defined in the pom file.
- * Set startCluster to true in order have the test start the cluster for you.
- * All those properties are set via surefire's systemPropertyVariables, meaning this:
- * $ mvn test -DstartCluster=false
- * will use an existing cluster at default address found above.
- *
- * The test creates a table with a unique(ish) name which it deletes at the end.
- */
-public class TestAsyncKuduSession extends BaseKuduTest {
- // Generate a unique table name
- private static final String TABLE_NAME =
- TestAsyncKuduSession.class.getName()+"-"+System.currentTimeMillis();
-
- private static Schema schema = getBasicSchema();
- private static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- table = createTable(TABLE_NAME, schema, getBasicCreateTableOptions());
- }
-
- /**
- * Regression test for case where an error in the previous batch could cause the next
- * batch to hang in flush()
- */
- @Test(timeout = 100000)
- public void testBatchErrorCauseSessionStuck() throws Exception {
- try {
- AsyncKuduSession session = client.newSession();
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
- session.setFlushInterval(100);
- TabletServerErrorPB error = TabletServerErrorPB.newBuilder()
- .setCode(TabletServerErrorPB.Code.UNKNOWN_ERROR)
- .setStatus(AppStatusPB.newBuilder()
- .setCode(AppStatusPB.ErrorCode.UNKNOWN_ERROR)
- .setMessage("injected error for test")
- .build())
- .build();
- Batch.injectTabletServerErrorAndLatency(error, 200);
- // 0ms: insert first row, which will be the first batch.
- Deferred<OperationResponse> resp1 = session.apply(createInsert(1));
- Thread.sleep(120);
- // 100ms: start to send first batch.
- // 100ms+: first batch got response from ts,
- // will wait 200s and throw erorr.
- // 120ms: insert another row, which will be the second batch.
- Deferred<OperationResponse> resp2 = session.apply(createInsert(2));
- // 220ms: start to send the second batch, but first batch is inflight,
- // so add callback to retry after first batch finishes.
- // 300ms: first batch's callback handles error, retry second batch.
- try {
- resp1.join(2000);
- } catch (TimeoutException e) {
- fail("First batch should not timeout in case of tablet server error");
- } catch (KuduException e) {
- // Expected.
- assertTrue(e.getMessage().contains("injected error for test"));
- }
- try {
- resp2.join(2000);
- } catch (TimeoutException e) {
- fail("Second batch should not timeout in case of tablet server error");
- } catch (KuduException e) {
- // expected
- assertTrue(e.getMessage().contains("injected error for test"));
- }
- } finally {
- Batch.injectTabletServerErrorAndLatency(null, 0);
- }
- }
-
- /**
- * Regression test for case when tablet lookup error causes original RPC to get stuck.
- * @throws Exception
- */
- @Test(timeout = 100000)
- public void testGetTableLocationsErrorCauseSessionStuck() throws Exception {
- AsyncKuduSession session = client.newSession();
- // Make sure tablet locations is cached.
- Insert insert = createInsert(1);
- session.apply(insert).join(DEFAULT_SLEEP);
- RemoteTablet rt = client.getTablet(table.getTableId(), insert.partitionKey());
- String tabletId = rt.getTabletIdAsString();
- TabletClient tc = client.clientFor(rt);
- try {
- // Delete table so we get table not found error.
- client.deleteTable(TABLE_NAME).join();
- // Wait until tablet is deleted on TS.
- while (true) {
- ListTabletsRequest req = new ListTabletsRequest();
- tc.sendRpc(req);
- ListTabletsResponse resp = req.getDeferred().join();
- if (!resp.getTabletsList().contains(tabletId)) {
- break;
- }
- Thread.sleep(100);
- }
- try {
- session.apply(createInsert(1)).join(DEFAULT_SLEEP);
- fail("Insert should not succeed");
- } catch (KuduException e) {
- assertTrue(e.getStatus().isNotFound());
- } catch (Throwable e) {
- fail("Should not throw other error: " + e);
- }
- } finally {
- table = createTable(TABLE_NAME, schema, getBasicCreateTableOptions());
- }
- }
-
- /** Regression test for a failure to correctly handle a timeout when flushing a batch. */
- @Test
- public void testInsertIntoUnavailableTablet() throws Exception {
- killTabletServers();
- try {
- AsyncKuduSession session = client.newSession();
- session.setTimeoutMillis(1);
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- Insert insert = createInsert(1);
- session.apply(insert);
- try {
- session.flush().join();
- fail("expected exception");
- } catch (DeferredGroupException e) {
- assertEquals(1, e.results().size());
- assertTrue(e.results().get(0).toString().contains("timeout"));
- }
- } finally {
- restartTabletServers();
- }
- }
-
- @Test(timeout = 100000)
- public void test() throws Exception {
-
- AsyncKuduSession session = client.newSession();
- // disable the low watermark until we need it
- session.setMutationBufferLowWatermark(1f);
-
- // First testing KUDU-232, the cache is empty and we want to force flush. We force the flush
- // interval to be higher than the sleep time so that we don't background flush while waiting.
- // If our subsequent manual flush throws, it means the logic to block on in-flight tablet
- // lookups in flush isn't working properly.
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
- session.setFlushInterval(DEFAULT_SLEEP + 1000);
- Deferred<OperationResponse> d = session.apply(createInsert(0));
- session.flush().join(DEFAULT_SLEEP);
- assertTrue(exists(0));
- // set back to default
- session.setFlushInterval(1000);
-
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
- for (int i = 1; i < 10; i++) {
- session.apply(createInsert(i)).join(DEFAULT_SLEEP);
- }
-
- assertEquals(10, countInRange(0, 10));
-
- session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
- session.setMutationBufferSpace(10);
-
- session.apply(createInsert(10));
-
- try {
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
- } catch (IllegalArgumentException ex) {
- /* expected, flush mode remains manual */
- }
-
- assertFalse(exists(10));
-
- for (int i = 11; i < 20; i++) {
- session.apply(createInsert(i));
- }
-
- assertEquals(0, countInRange(10, 20));
- try {
- session.apply(createInsert(20));
- } catch (KuduException ex) {
- /* expected, buffer would be too big */
- }
- assertEquals(0, countInRange(10, 20)); // the buffer should still be full
-
- session.flush().join(DEFAULT_SLEEP);
- assertEquals(10, countInRange(10, 20)); // now everything should be there
-
- session.flush().join(DEFAULT_SLEEP); // flushing empty buffer should be a no-op.
-
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
-
- d = session.apply(createInsert(20));
- Thread.sleep(50); // waiting a minimal amount of time to make sure the interval is in effect
- assertFalse(exists(20));
- // Add 10 items, the last one will stay in the buffer
- for (int i = 21; i < 30; i++) {
- d = session.apply(createInsert(i));
- }
- Deferred<OperationResponse> buffered = session.apply(createInsert(30));
- long now = System.currentTimeMillis();
- d.join(DEFAULT_SLEEP); // Ok to use the last d, everything is going to the buffer
- // auto flush will force flush if the buffer is full as it should be now
- // so we check that we didn't wait the full interval
- long elapsed = System.currentTimeMillis() - now;
- assertTrue(elapsed < 950);
- assertEquals(10, countInRange(20, 31));
- buffered.join();
- assertEquals(11, countInRange(20, 31));
-
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
- Update update = createUpdate(30);
- PartialRow row = update.getRow();
- row.addInt(2, 999);
- row.addString(3, "updated data");
- d = session.apply(update);
- d.addErrback(defaultErrorCB);
- d.join(DEFAULT_SLEEP);
- assertEquals(31, countInRange(0, 31));
-
- Delete del = createDelete(30);
- d = session.apply(del);
- d.addErrback(defaultErrorCB);
- d.join(DEFAULT_SLEEP);
- assertEquals(30, countInRange(0, 31));
-
- session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
- session.setMutationBufferSpace(35);
- for (int i = 0; i < 20; i++) {
- buffered = session.apply(createDelete(i));
- }
- assertEquals(30, countInRange(0, 31));
- session.flush();
- buffered.join(DEFAULT_SLEEP);
- assertEquals(10, countInRange(0, 31));
-
- for (int i = 30; i < 40; i++) {
- session.apply(createInsert(i));
- }
-
- for (int i = 20; i < 30; i++) {
- buffered = session.apply(createDelete(i));
- }
-
- assertEquals(10, countInRange(0, 40));
- session.flush();
- buffered.join(DEFAULT_SLEEP);
- assertEquals(10, countInRange(0, 40));
-
- // Test nulls
- // add 10 rows with the nullable column set to null
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
- for (int i = 40; i < 50; i++) {
- session.apply(createInsertWithNull(i)).join(DEFAULT_SLEEP);
- }
-
- // now scan those rows and make sure the column is null
- assertEquals(10, countNullColumns(40, 50));
-
- // Test sending edits too fast
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
- session.setMutationBufferSpace(10);
-
- // The buffer has a capacity of 10, we insert 21 rows, meaning we fill the first one,
- // force flush, fill a second one before the first one could come back,
- // and the 21st row will be sent back.
- boolean gotException = false;
- for (int i = 50; i < 71; i++) {
- try {
- session.apply(createInsert(i));
- } catch (PleaseThrottleException ex) {
- gotException = true;
- assertEquals(70, i);
- // Wait for the buffer to clear
- ex.getDeferred().join(DEFAULT_SLEEP);
- session.apply(ex.getFailedRpc());
- session.flush().join(DEFAULT_SLEEP);
- }
- }
- assertTrue("Expected PleaseThrottleException", gotException);
- assertEquals(21, countInRange(50, 71));
-
- // Now test a more subtle issue, basically the race where we call flush from the client when
- // there's a batch already in flight. We need to finish joining only when all the data is
- // flushed.
- for (int i = 71; i < 91; i++) {
- session.apply(createInsert(i));
- }
- session.flush().join(DEFAULT_SLEEP);
- // If we only waited after the in flight batch, there would be 10 rows here.
- assertEquals(20, countInRange(71, 91));
-
- // Test empty scanner projection
- AsyncKuduScanner scanner = getScanner(71, 91, Collections.<String>emptyList());
- assertEquals(20, countRowsInScan(scanner));
-
- // Test removing the connection and then do a rapid set of inserts
- client.getTabletClients().get(0).shutdown().join(DEFAULT_SLEEP);
- session.setMutationBufferSpace(1);
- for (int i = 91; i < 101; i++) {
- try {
- session.apply(createInsert(i));
- } catch (PleaseThrottleException ex) {
- // Wait for the buffer to clear
- ex.getDeferred().join(DEFAULT_SLEEP);
- session.apply(ex.getFailedRpc());
- }
- }
- session.flush().join(DEFAULT_SLEEP);
- assertEquals(10, countInRange(91, 101));
-
- // Test a tablet going missing or encountering a new tablet while inserting a lot
- // of data. This code used to fail in many different ways.
- client.emptyTabletsCacheForTable(table.getTableId());
- for (int i = 101; i < 151; i++) {
- Insert insert = createInsert(i);
- while (true) {
- try {
- session.apply(insert);
- break;
- } catch (PleaseThrottleException ex) {
- // Wait for the buffer to clear
- ex.getDeferred().join(DEFAULT_SLEEP);
- }
- }
- }
- session.flush().join(DEFAULT_SLEEP);
- assertEquals(50, countInRange(101, 151));
-
- // Test the low watermark.
- // Before the fix for KUDU-804, a change to the buffer space did not result in a change to the
- // low watermark causing this test to fail.
- session.setMutationBufferLowWatermark(0.1f);
- session.setMutationBufferSpace(10);
- session.setRandomSeed(12345); // Will make us hit the exception after 6 tries
- gotException = false;
- for (int i = 151; i < 171; i++) {
- try {
- session.apply(createInsert(i));
- } catch (PleaseThrottleException ex) {
- // We're going to hit the exception after filling up the buffer a first time then trying
- // to insert 6 more rows.
- assertEquals(167, i);
- gotException = true;
- assertTrue(ex.getMessage().contains("watermark"));
- // Once we hit the exception we wait on the batch to finish flushing and then insert the
- // rest of the data.
- ex.getDeferred().join(DEFAULT_SLEEP);
- session.apply(ex.getFailedRpc());
- }
- }
- session.flush().join(DEFAULT_SLEEP);
- assertEquals(20, countInRange(151, 171));
- assertTrue(gotException);
- }
-
- private Insert createInsert(int key) {
- return createBasicSchemaInsert(table, key);
- }
-
- private Insert createInsertWithNull(int key) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addInt(0, key);
- row.addInt(1, 2);
- row.addInt(2, 3);
- row.setNull(3);
- row.addBoolean(4, false);
- return insert;
- }
-
- private Update createUpdate(int key) {
- Update update = table.newUpdate();
- PartialRow row = update.getRow();
- row.addInt(0, key);
- return update;
- }
-
- private Delete createDelete(int key) {
- Delete delete = table.newDelete();
- PartialRow row = delete.getRow();
- row.addInt(0, key);
- return delete;
- }
-
- public static boolean exists(final int key) throws Exception {
-
- AsyncKuduScanner scanner = getScanner(key, key + 1);
- final AtomicBoolean exists = new AtomicBoolean(false);
-
- Callback<Object, RowResultIterator> cb =
- new Callback<Object, RowResultIterator>() {
- @Override
- public Object call(RowResultIterator arg) throws Exception {
- if (arg == null) return null;
- for (RowResult row : arg) {
- if (row.getInt(0) == key) {
- exists.set(true);
- break;
- }
- }
- return null;
- }
- };
-
- while (scanner.hasMoreRows()) {
- Deferred<RowResultIterator> data = scanner.nextRows();
- data.addCallbacks(cb, defaultErrorCB);
- data.join(DEFAULT_SLEEP);
- if (exists.get()) {
- break;
- }
- }
-
- Deferred<RowResultIterator> closer = scanner.close();
- closer.join(DEFAULT_SLEEP);
- return exists.get();
- }
-
- public static int countNullColumns(final int startKey, final int endKey) throws Exception {
-
- AsyncKuduScanner scanner = getScanner(startKey, endKey);
- final AtomicInteger ai = new AtomicInteger();
-
- Callback<Object, RowResultIterator> cb = new Callback<Object, RowResultIterator>() {
- @Override
- public Object call(RowResultIterator arg) throws Exception {
- if (arg == null) return null;
- for (RowResult row : arg) {
- if (row.isNull(3)) {
- ai.incrementAndGet();
- }
- }
- return null;
- }
- };
-
- while (scanner.hasMoreRows()) {
- Deferred<RowResultIterator> data = scanner.nextRows();
- data.addCallbacks(cb, defaultErrorCB);
- data.join(DEFAULT_SLEEP);
- }
-
- Deferred<RowResultIterator> closer = scanner.close();
- closer.join(DEFAULT_SLEEP);
- return ai.get();
- }
-
- public static int countInRange(final int start, final int exclusiveEnd) throws Exception {
-
- AsyncKuduScanner scanner = getScanner(start, exclusiveEnd);
- return countRowsInScan(scanner);
- }
-
- private static AsyncKuduScanner getScanner(int start, int exclusiveEnd) {
- return getScanner(start, exclusiveEnd, null);
- }
-
- private static AsyncKuduScanner getScanner(int start, int exclusiveEnd,
- List<String> columnNames) {
-
- PartialRow lowerBound = schema.newPartialRow();
- lowerBound.addInt(schema.getColumnByIndex(0).getName(), start);
-
- PartialRow upperBound = schema.newPartialRow();
- upperBound.addInt(schema.getColumnByIndex(0).getName(), exclusiveEnd);
-
- AsyncKuduScanner scanner = client.newScannerBuilder(table)
- .lowerBound(lowerBound)
- .exclusiveUpperBound(upperBound)
- .setProjectedColumnNames(columnNames)
- .build();
- return scanner;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestBitSet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestBitSet.java b/java/kudu-client/src/test/java/org/kududb/client/TestBitSet.java
deleted file mode 100644
index ab27e63..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestBitSet.java
+++ /dev/null
@@ -1,99 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
-import java.util.BitSet;
-
-public class TestBitSet {
-
- /**
- * Test out BitSet-related operations
- */
- @Test
- public void test() {
- int colCount = 1;
- BitSet bs = new BitSet(colCount);
- bs.set(0);
- int size = Bytes.getBitSetSize(colCount);
- byte[] result = Bytes.fromBitSet(bs, colCount);
- assertEquals(size, result.length);
- BitSet newBs = Bytes.toBitSet(result, 0, colCount);
- assertTrue(newBs.get(0));
-
- colCount = 7;
- bs = new BitSet(colCount);
- bs.set(0);
- bs.set(5);
- size = Bytes.getBitSetSize(colCount);
- result = Bytes.fromBitSet(bs, colCount);
- assertEquals(size, result.length);
- newBs = Bytes.toBitSet(result, 0, colCount);
- assertTrue(newBs.get(0));
- assertFalse(newBs.get(1));
- assertFalse(newBs.get(2));
- assertFalse(newBs.get(3));
- assertFalse(newBs.get(4));
- assertTrue(newBs.get(5));
- assertFalse(newBs.get(6));
-
- colCount = 8;
- bs = new BitSet(colCount);
- bs.set(0);
- bs.set(5);
- bs.set(7);
- size = Bytes.getBitSetSize(colCount);
- result = Bytes.fromBitSet(bs, colCount);
- assertEquals(size, result.length);
- newBs = Bytes.toBitSet(result, 0, colCount);
- assertTrue(newBs.get(0));
- assertFalse(newBs.get(1));
- assertFalse(newBs.get(2));
- assertFalse(newBs.get(3));
- assertFalse(newBs.get(4));
- assertTrue(newBs.get(5));
- assertFalse(newBs.get(6));
- assertTrue(newBs.get(7));
-
- colCount = 11;
- bs = new BitSet(colCount);
- bs.set(0);
- bs.set(5);
- bs.set(7);
- bs.set(9);
- size = Bytes.getBitSetSize(colCount);
- result = Bytes.fromBitSet(bs, colCount);
- assertEquals(size, result.length);
- newBs = Bytes.toBitSet(result, 0, colCount);
- assertTrue(newBs.get(0));
- assertFalse(newBs.get(1));
- assertFalse(newBs.get(2));
- assertFalse(newBs.get(3));
- assertFalse(newBs.get(4));
- assertTrue(newBs.get(5));
- assertFalse(newBs.get(6));
- assertTrue(newBs.get(7));
- assertFalse(newBs.get(8));
- assertTrue(newBs.get(9));
- assertFalse(newBs.get(10));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestBytes.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestBytes.java b/java/kudu-client/src/test/java/org/kududb/client/TestBytes.java
deleted file mode 100644
index 11c2035..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestBytes.java
+++ /dev/null
@@ -1,105 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.math.BigInteger;
-
-public class TestBytes {
-
- @Test
- public void test() {
- byte[] bytes = new byte[8];
-
- // Boolean
- Bytes.setUnsignedByte(bytes, (short) 1);
- assert(Bytes.getBoolean(bytes));
- Bytes.setUnsignedByte(bytes, (short) 0);
- assert(!Bytes.getBoolean(bytes));
-
- // BYTES
- short smallUbyte = 120;
- Bytes.setUnsignedByte(bytes, smallUbyte);
- assertEquals(smallUbyte, Bytes.getUnsignedByte(bytes));
- short largeUbyte = 250;
- Bytes.setUnsignedByte(bytes, largeUbyte);
- assertEquals(largeUbyte, Bytes.getUnsignedByte(bytes));
-
- // SHORTS
- short nshort = -300;
- Bytes.setShort(bytes, nshort);
- assertEquals(nshort, Bytes.getShort(bytes));
- short pshort = 300;
- Bytes.setShort(bytes, pshort);
- assertEquals(pshort, Bytes.getShort(bytes));
- int smallUshort = 300;
- Bytes.setUnsignedShort(bytes, smallUshort);
- assertEquals(smallUshort, Bytes.getUnsignedShort(bytes));
- int largeUshort = 60000;
- Bytes.setUnsignedShort(bytes, largeUshort);
- assertEquals(largeUshort, Bytes.getUnsignedShort(bytes));
-
- // INTS
- int nint = -60000;
- Bytes.setInt(bytes, nint);
- assertEquals(nint, Bytes.getInt(bytes));
- int pint = 60000;
- Bytes.setInt(bytes, pint);
- assertEquals(pint, Bytes.getInt(bytes));
- long smallUint = 60000;
- Bytes.setUnsignedInt(bytes, smallUint);
- assertEquals(smallUint, Bytes.getUnsignedInt(bytes));
- long largeUint = 4000000000L;
- Bytes.setUnsignedInt(bytes, largeUint);
- assertEquals(largeUint, Bytes.getUnsignedInt(bytes));
-
- // LONGS
- long nlong = -4000000000L;
- Bytes.setLong(bytes, nlong);
- assertEquals(nlong, Bytes.getLong(bytes));
- long plong = 4000000000L;
- Bytes.setLong(bytes, plong);
- assertEquals(plong, Bytes.getLong(bytes));
- BigInteger smallUlong = new BigInteger("4000000000");
- Bytes.setUnsignedLong(bytes, smallUlong);
- assertEquals(smallUlong, Bytes.getUnsignedLong(bytes));
- BigInteger largeUlong = new BigInteger("10000000000000000000");
- Bytes.setUnsignedLong(bytes, largeUlong);
- assertEquals(largeUlong, Bytes.getUnsignedLong(bytes));
-
- // FLOAT
- float aFloat = 123.456f;
- Bytes.setFloat(bytes, aFloat);
- assertEquals(aFloat, Bytes.getFloat(bytes), 0.001);
-
- // DOUBLE
- double aDouble = 123.456;
- Bytes.setDouble(bytes, aDouble);
- assertEquals(aDouble, Bytes.getDouble(bytes), 0.001);
- }
-
- @Test
- public void testHex() {
- byte[] bytes = new byte[] { (byte) 0x01, (byte) 0x23, (byte) 0x45, (byte) 0x67,
- (byte) 0x89, (byte) 0xAB, (byte) 0xCD, (byte) 0xEF };
- Assert.assertEquals("0x0123456789ABCDEF", Bytes.hex(bytes));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestColumnRangePredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestColumnRangePredicate.java b/java/kudu-client/src/test/java/org/kududb/client/TestColumnRangePredicate.java
deleted file mode 100644
index bf13f2d..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestColumnRangePredicate.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.Lists;
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.tserver.Tserver;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-public class TestColumnRangePredicate {
-
- @Test
- public void testRawLists() {
- ColumnSchema col1 = new ColumnSchema.ColumnSchemaBuilder("col1", Type.INT32).build();
- ColumnSchema col2 = new ColumnSchema.ColumnSchemaBuilder("col2", Type.STRING).build();
-
- ColumnRangePredicate pred1 = new ColumnRangePredicate(col1);
- pred1.setLowerBound(1);
-
- ColumnRangePredicate pred2 = new ColumnRangePredicate(col1);
- pred2.setUpperBound(2);
-
- ColumnRangePredicate pred3 = new ColumnRangePredicate(col2);
- pred3.setLowerBound("aaa");
- pred3.setUpperBound("bbb");
-
- List<ColumnRangePredicate> preds = Lists.newArrayList(pred1, pred2, pred3);
-
- byte[] rawPreds = ColumnRangePredicate.toByteArray(preds);
-
- List<Tserver.ColumnRangePredicatePB> decodedPreds = null;
- try {
- decodedPreds = ColumnRangePredicate.fromByteArray(rawPreds);
- } catch (IllegalArgumentException e) {
- fail("Couldn't decode: " + e.getMessage());
- }
-
- assertEquals(3, decodedPreds.size());
-
- assertEquals(col1.getName(), decodedPreds.get(0).getColumn().getName());
- assertEquals(1, Bytes.getInt(Bytes.get(decodedPreds.get(0).getLowerBound())));
- assertFalse(decodedPreds.get(0).hasInclusiveUpperBound());
-
- assertEquals(col1.getName(), decodedPreds.get(1).getColumn().getName());
- assertEquals(2, Bytes.getInt(Bytes.get(decodedPreds.get(1).getInclusiveUpperBound())));
- assertFalse(decodedPreds.get(1).hasLowerBound());
-
- assertEquals(col2.getName(), decodedPreds.get(2).getColumn().getName());
- assertEquals("aaa", Bytes.getString(Bytes.get(decodedPreds.get(2).getLowerBound())));
- assertEquals("bbb", Bytes.getString(Bytes.get(decodedPreds.get(2).getInclusiveUpperBound())));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestDeadlineTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestDeadlineTracker.java b/java/kudu-client/src/test/java/org/kududb/client/TestDeadlineTracker.java
deleted file mode 100644
index 20ee06a..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestDeadlineTracker.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import static org.junit.Assert.*;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.base.Ticker;
-import org.junit.Test;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class TestDeadlineTracker {
-
- @Test
- public void testTimeout() {
- final AtomicLong timeToReturn = new AtomicLong();
- Ticker ticker = new Ticker() {
- @Override
- public long read() {
- return timeToReturn.get();
- }
- };
- Stopwatch stopwatch = Stopwatch.createUnstarted(ticker);
-
- // no timeout set
- DeadlineTracker tracker = new DeadlineTracker(stopwatch);
- tracker.setDeadline(0);
- assertFalse(tracker.hasDeadline());
- assertFalse(tracker.timedOut());
-
- // 500ms timeout set
- tracker.reset();
- tracker.setDeadline(500);
- assertTrue(tracker.hasDeadline());
- assertFalse(tracker.timedOut());
- assertFalse(tracker.wouldSleepingTimeout(499));
- assertTrue(tracker.wouldSleepingTimeout(500));
- assertTrue(tracker.wouldSleepingTimeout(501));
- assertEquals(500, tracker.getMillisBeforeDeadline());
-
- // fast forward 200ms
- timeToReturn.set(200 * 1000000);
- assertTrue(tracker.hasDeadline());
- assertFalse(tracker.timedOut());
- assertFalse(tracker.wouldSleepingTimeout(299));
- assertTrue(tracker.wouldSleepingTimeout(300));
- assertTrue(tracker.wouldSleepingTimeout(301));
- assertEquals(300, tracker.getMillisBeforeDeadline());
-
- // fast forward another 400ms, so the RPC timed out
- timeToReturn.set(600 * 1000000);
- assertTrue(tracker.hasDeadline());
- assertTrue(tracker.timedOut());
- assertTrue(tracker.wouldSleepingTimeout(299));
- assertTrue(tracker.wouldSleepingTimeout(300));
- assertTrue(tracker.wouldSleepingTimeout(301));
- assertEquals(1, tracker.getMillisBeforeDeadline());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestErrorCollector.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestErrorCollector.java b/java/kudu-client/src/test/java/org/kududb/client/TestErrorCollector.java
deleted file mode 100644
index 883b2e1..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestErrorCollector.java
+++ /dev/null
@@ -1,90 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestErrorCollector {
-
- @Test
- public void testErrorCollector() {
- int maxErrors = 10;
- ErrorCollector collector = new ErrorCollector(maxErrors);
-
- // Test with no errors.
- int countToTest = 0;
- Assert.assertEquals(countToTest, collector.countErrors());
- RowErrorsAndOverflowStatus reos = collector.getErrors();
- Assert.assertEquals(0, collector.countErrors());
- Assert.assertFalse(reos.isOverflowed());
- Assert.assertEquals(countToTest, reos.getRowErrors().length);
-
- // Test a single row error.
- countToTest = 1;
- collector.addError(createRowError(countToTest));
- Assert.assertEquals(countToTest, collector.countErrors());
- reos = collector.getErrors();
- Assert.assertEquals(0, collector.countErrors());
- Assert.assertFalse(reos.isOverflowed());
- Assert.assertEquals(countToTest, reos.getRowErrors().length);
- Assert.assertEquals(countToTest, reos.getRowErrors()[0].getErrorStatus().getPosixCode());
-
- // Test filling the collector to the max.
- countToTest = maxErrors;
- fillCollectorWith(collector, countToTest);
- Assert.assertEquals(countToTest, collector.countErrors());
- reos = collector.getErrors();
- Assert.assertEquals(0, collector.countErrors());
- Assert.assertFalse(reos.isOverflowed());
- Assert.assertEquals(countToTest, reos.getRowErrors().length);
- Assert.assertEquals(countToTest - 1, reos.getRowErrors()[9].getErrorStatus().getPosixCode());
-
- // Test overflowing.
- countToTest = 95;
- fillCollectorWith(collector, countToTest);
- Assert.assertEquals(maxErrors, collector.countErrors());
- reos = collector.getErrors();
- Assert.assertEquals(0, collector.countErrors());
- Assert.assertTrue(reos.isOverflowed());
- Assert.assertEquals(maxErrors, reos.getRowErrors().length);
- Assert.assertEquals(countToTest - 1, reos.getRowErrors()[9].getErrorStatus().getPosixCode());
-
- // Test overflowing on a newly created collector.
- countToTest = 95;
- collector = new ErrorCollector(maxErrors);
- fillCollectorWith(collector, countToTest);
- Assert.assertEquals(maxErrors, collector.countErrors());
- reos = collector.getErrors();
- Assert.assertEquals(0, collector.countErrors());
- Assert.assertTrue(reos.isOverflowed());
- Assert.assertEquals(maxErrors, reos.getRowErrors().length);
- Assert.assertEquals(countToTest - 1, reos.getRowErrors()[9].getErrorStatus().getPosixCode());
- }
-
- private void fillCollectorWith(ErrorCollector collector, int errorsToAdd) {
- for (int i = 0; i < errorsToAdd; i++) {
- collector.addError(createRowError(i));
- }
- }
-
- private RowError createRowError(int id) {
- // Use the error status as a way to message pass and so that we can test we're getting the right
- // messages on the other end.
- return new RowError(Status.NotAuthorized("test", id), null, "test");
- }
-}
[12/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
new file mode 100644
index 0000000..1464fa4
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
@@ -0,0 +1,438 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.net.HostAndPort;
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.master.Master;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.fail;
+
+public class BaseKuduTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseKuduTest.class);
+
+ private static final String NUM_MASTERS_PROP = "NUM_MASTERS";
+ private static final int NUM_TABLET_SERVERS = 3;
+ private static final int DEFAULT_NUM_MASTERS = 1;
+
+ // Number of masters that will be started for this test if we're starting
+ // a cluster.
+ private static final int NUM_MASTERS =
+ Integer.getInteger(NUM_MASTERS_PROP, DEFAULT_NUM_MASTERS);
+
+ protected static final int DEFAULT_SLEEP = 50000;
+
+ // Currently not specifying a seed since we want a random behavior when running tests that
+ // restart tablet servers. Would be nice to have the same kind of facility that C++ has that dumps
+ // the seed it picks so that you can re-run tests with it.
+ private static final Random randomForTSRestart = new Random();
+
+ private static MiniKuduCluster miniCluster;
+
+ // Comma separate describing the master addresses and ports.
+ protected static String masterAddresses;
+ protected static List<HostAndPort> masterHostPorts;
+
+ // We create both versions of the client for ease of use.
+ protected static AsyncKuduClient client;
+ protected static KuduClient syncClient;
+ protected static final Schema basicSchema = getBasicSchema();
+ protected static final Schema allTypesSchema = getSchemaWithAllTypes();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ LOG.info("Setting up before class...");
+
+ miniCluster = new MiniKuduCluster.MiniKuduClusterBuilder()
+ .numMasters(NUM_MASTERS)
+ .numTservers(NUM_TABLET_SERVERS)
+ .defaultTimeoutMs(DEFAULT_SLEEP)
+ .build();
+ masterAddresses = miniCluster.getMasterAddresses();
+ masterHostPorts = miniCluster.getMasterHostPorts();
+
+ LOG.info("Creating new Kudu client...");
+ client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
+ .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
+ .build();
+ syncClient = new KuduClient(client);
+ LOG.info("Waiting for tablet servers...");
+ if (!miniCluster.waitForTabletServers(NUM_TABLET_SERVERS)) {
+ fail("Couldn't get " + NUM_TABLET_SERVERS + " tablet servers running, aborting");
+ }
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ try {
+ if (client != null) {
+ Deferred<ArrayList<Void>> d = client.shutdown();
+ d.addErrback(defaultErrorCB);
+ d.join(DEFAULT_SLEEP);
+ // No need to explicitly shutdown the sync client,
+ // shutting down the async client effectively does that.
+ }
+ } finally {
+ if (miniCluster != null) {
+ miniCluster.shutdown();
+ }
+ }
+ }
+
+ protected static KuduTable createTable(String tableName, Schema schema,
+ CreateTableOptions builder) throws Exception {
+ LOG.info("Creating table: {}", tableName);
+ return client.syncClient().createTable(tableName, schema, builder);
+ }
+
+ /**
+ * Counts the rows from the {@code scanner} until exhaustion. It doesn't require the scanner to
+ * be new, so it can be used to finish scanning a previously-started scan.
+ */
+ protected static int countRowsInScan(AsyncKuduScanner scanner)
+ throws Exception {
+ final AtomicInteger counter = new AtomicInteger();
+
+ Callback<Object, RowResultIterator> cb = new Callback<Object, RowResultIterator>() {
+ @Override
+ public Object call(RowResultIterator arg) throws Exception {
+ if (arg == null) return null;
+ counter.addAndGet(arg.getNumRows());
+ return null;
+ }
+ };
+
+ while (scanner.hasMoreRows()) {
+ Deferred<RowResultIterator> data = scanner.nextRows();
+ data.addCallbacks(cb, defaultErrorCB);
+ data.join(DEFAULT_SLEEP);
+ }
+
+ Deferred<RowResultIterator> closer = scanner.close();
+ closer.addCallbacks(cb, defaultErrorCB);
+ closer.join(DEFAULT_SLEEP);
+ return counter.get();
+ }
+
+ protected List<String> scanTableToStrings(KuduTable table,
+ KuduPredicate... predicates) throws Exception {
+ List<String> rowStrings = Lists.newArrayList();
+ KuduScanner.KuduScannerBuilder scanBuilder = syncClient.newScannerBuilder(table);
+ for (KuduPredicate predicate : predicates) {
+ scanBuilder.addPredicate(predicate);
+ }
+ KuduScanner scanner = scanBuilder.build();
+ while (scanner.hasMoreRows()) {
+ RowResultIterator rows = scanner.nextRows();
+ for (RowResult r : rows) {
+ rowStrings.add(r.rowToString());
+ }
+ }
+ Collections.sort(rowStrings);
+ return rowStrings;
+ }
+
+ private static final int[] KEYS = new int[] {10, 20, 30};
+ protected static KuduTable createFourTabletsTableWithNineRows(String tableName) throws
+ Exception {
+ CreateTableOptions builder = getBasicCreateTableOptions();
+ for (int i : KEYS) {
+ PartialRow splitRow = basicSchema.newPartialRow();
+ splitRow.addInt(0, i);
+ builder.addSplitRow(splitRow);
+ }
+ KuduTable table = createTable(tableName, basicSchema, builder);
+ AsyncKuduSession session = client.newSession();
+
+ // create a table with on empty tablet and 3 tablets of 3 rows each
+ for (int key1 : KEYS) {
+ for (int key2 = 1; key2 <= 3; key2++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addInt(0, key1 + key2);
+ row.addInt(1, key1);
+ row.addInt(2, key2);
+ row.addString(3, "a string");
+ row.addBoolean(4, true);
+ session.apply(insert).join(DEFAULT_SLEEP);
+ }
+ }
+ session.close().join(DEFAULT_SLEEP);
+ return table;
+ }
+
+ public static Schema getSchemaWithAllTypes() {
+ List<ColumnSchema> columns =
+ ImmutableList.of(
+ new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).key(true).build(),
+ new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build(),
+ new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build(),
+ new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build(),
+ new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build(),
+ new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build(),
+ new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build(),
+ new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).build(),
+ new ColumnSchema.ColumnSchemaBuilder("binary-array", Type.BINARY).build(),
+ new ColumnSchema.ColumnSchemaBuilder("binary-bytebuffer", Type.BINARY).build(),
+ new ColumnSchema.ColumnSchemaBuilder("null", Type.STRING).nullable(true).build(),
+ new ColumnSchema.ColumnSchemaBuilder("timestamp", Type.TIMESTAMP).build());
+
+ return new Schema(columns);
+ }
+
+ public static CreateTableOptions getAllTypesCreateTableOptions() {
+ return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("int8"));
+ }
+
+ public static Schema getBasicSchema() {
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(5);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column2_i", Type.INT32).build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column3_s", Type.STRING)
+ .nullable(true)
+ .desiredBlockSize(4096)
+ .encoding(ColumnSchema.Encoding.DICT_ENCODING)
+ .compressionAlgorithm(ColumnSchema.CompressionAlgorithm.LZ4)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column4_b", Type.BOOL).build());
+ return new Schema(columns);
+ }
+
+ public static CreateTableOptions getBasicCreateTableOptions() {
+ return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
+ }
+
+ /**
+ * Creates table options with non-covering range partitioning for a table with
+ * the basic schema. Range partition key ranges fall between the following values:
+ *
+ * [ 0, 50)
+ * [ 50, 100)
+ * [200, 300)
+ */
+ public static CreateTableOptions getBasicTableOptionsWithNonCoveredRange() {
+ Schema schema = basicSchema;
+ CreateTableOptions option = new CreateTableOptions();
+ option.setRangePartitionColumns(ImmutableList.of("key"));
+
+ PartialRow aLowerBound = schema.newPartialRow();
+ aLowerBound.addInt("key", 0);
+ PartialRow aUpperBound = schema.newPartialRow();
+ aUpperBound.addInt("key", 100);
+ option.addRangeBound(aLowerBound, aUpperBound);
+
+ PartialRow bLowerBound = schema.newPartialRow();
+ bLowerBound.addInt("key", 200);
+ PartialRow bUpperBound = schema.newPartialRow();
+ bUpperBound.addInt("key", 300);
+ option.addRangeBound(bLowerBound, bUpperBound);
+
+ PartialRow split = schema.newPartialRow();
+ split.addInt("key", 50);
+ option.addSplitRow(split);
+ return option;
+ }
+
+ protected Insert createBasicSchemaInsert(KuduTable table, int key) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addInt(0, key);
+ row.addInt(1, 2);
+ row.addInt(2, 3);
+ row.addString(3, "a string");
+ row.addBoolean(4, true);
+ return insert;
+ }
+
+ static Callback<Object, Object> defaultErrorCB = new Callback<Object, Object>() {
+ @Override
+ public Object call(Object arg) throws Exception {
+ if (arg == null) return null;
+ if (arg instanceof Exception) {
+ LOG.warn("Got exception", (Exception) arg);
+ } else {
+ LOG.warn("Got an error response back " + arg);
+ }
+ return new Exception("Can't recover from error, see previous WARN");
+ }
+ };
+
+ /**
+ * Helper method to open a table. It sets the default sleep time when joining on the Deferred.
+ * @param name Name of the table
+ * @return A KuduTable
+ * @throws Exception MasterErrorException if the table doesn't exist
+ */
+ protected static KuduTable openTable(String name) throws Exception {
+ Deferred<KuduTable> d = client.openTable(name);
+ return d.join(DEFAULT_SLEEP);
+ }
+
+ /**
+ * Helper method to easily kill a tablet server that serves the given table's only tablet's
+ * leader. The currently running test case will be failed if there's more than one tablet,
+ * if the tablet has no leader after some retries, or if the tablet server was already killed.
+ *
+ * This method is thread-safe.
+ * @param table a KuduTable which will get its single tablet's leader killed.
+ * @throws Exception
+ */
+ protected static void killTabletLeader(KuduTable table) throws Exception {
+ LocatedTablet.Replica leader = null;
+ DeadlineTracker deadlineTracker = new DeadlineTracker();
+ deadlineTracker.setDeadline(DEFAULT_SLEEP);
+ while (leader == null) {
+ if (deadlineTracker.timedOut()) {
+ fail("Timed out while trying to find a leader for this table: " + table.getName());
+ }
+ List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
+ if (tablets.isEmpty() || tablets.size() > 1) {
+ fail("Currently only support killing leaders for tables containing 1 tablet, table " +
+ table.getName() + " has " + tablets.size());
+ }
+ LocatedTablet tablet = tablets.get(0);
+ if (tablet.getReplicas().size() == 1) {
+ fail("Table " + table.getName() + " only has 1 tablet, please enable replication");
+ }
+ leader = tablet.getLeaderReplica();
+ if (leader == null) {
+ LOG.info("Sleeping while waiting for a tablet LEADER to arise, currently slept " +
+ deadlineTracker.getElapsedMillis() + "ms");
+ Thread.sleep(50);
+ }
+ }
+
+ Integer port = leader.getRpcPort();
+ miniCluster.killTabletServerOnPort(port);
+ }
+
+ /**
+ * Helper method to easily kill the leader master.
+ *
+ * This method is thread-safe.
+ * @throws Exception If there is an error finding or killing the leader master.
+ */
+ protected static void killMasterLeader() throws Exception {
+ int leaderPort = findLeaderMasterPort();
+ miniCluster.killMasterOnPort(leaderPort);
+ }
+
+ /**
+ * Find the port of the leader master in order to retrieve it from the port to process map.
+ * @return The port of the leader master.
+ * @throws Exception If we are unable to find the leader master.
+ */
+ protected static int findLeaderMasterPort() throws Exception {
+ Stopwatch sw = Stopwatch.createStarted();
+ int leaderPort = -1;
+ while (leaderPort == -1 && sw.elapsed(TimeUnit.MILLISECONDS) < DEFAULT_SLEEP) {
+ Deferred<Master.GetTableLocationsResponsePB> masterLocD = client.getMasterTableLocationsPB();
+ Master.GetTableLocationsResponsePB r = masterLocD.join(DEFAULT_SLEEP);
+ leaderPort = r.getTabletLocations(0)
+ .getReplicas(0)
+ .getTsInfo()
+ .getRpcAddresses(0)
+ .getPort();
+ }
+ if (leaderPort == -1) {
+ fail("No leader master found after " + DEFAULT_SLEEP + " ms.");
+ }
+ return leaderPort;
+ }
+
+ /**
+ * Picks at random a tablet server that serves tablets from the passed table and restarts it.
+ * Waits between killing and restarting the process.
+ * @param table table to query for a TS to restart
+ * @throws Exception
+ */
+ protected static void restartTabletServer(KuduTable table) throws Exception {
+ List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
+ if (tablets.isEmpty()) {
+ fail("Table " + table.getName() + " doesn't have any tablets");
+ }
+
+ LocatedTablet tablet = tablets.get(0);
+
+ int port = tablet.getReplicas().get(
+ randomForTSRestart.nextInt(tablet.getReplicas().size())).getRpcPort();
+
+ miniCluster.killTabletServerOnPort(port);
+
+ Thread.sleep(1000);
+
+ miniCluster.restartDeadTabletServerOnPort(port);
+ }
+
+ /**
+ * Kills, sleeps, then restarts the leader master.
+ * @throws Exception
+ */
+ protected static void restartLeaderMaster() throws Exception {
+ int master = findLeaderMasterPort();
+ miniCluster.killMasterOnPort(master);
+
+ Thread.sleep(1000);
+
+ miniCluster.restartDeadMasterOnPort(master);
+ }
+
+ /**
+ * Return the comma-separated list of "host:port" pairs that describes the master
+ * config for this cluster.
+ * @return The master config string.
+ */
+ protected static String getMasterAddresses() {
+ return masterAddresses;
+ }
+
+ /**
+ * Kills all tablet servers in the cluster.
+ * @throws InterruptedException
+ */
+ protected void killTabletServers() throws InterruptedException {
+ miniCluster.killTabletServers();
+ }
+
+ /**
+ * Restarts killed tablet servers in the cluster.
+ * @throws Exception
+ */
+ protected void restartTabletServers() throws Exception {
+ miniCluster.restartDeadTabletServers();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
new file mode 100644
index 0000000..cb8e968
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java
@@ -0,0 +1,396 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Integration test for the client. RPCs are sent to Kudu from multiple threads while processes
+ * are restarted and failures are injected.
+ *
+ * By default this test runs for 60 seconds, but this can be changed by passing a different value
+ * in "itclient.runtime.seconds". For example:
+ * "mvn test -Dtest=ITClient -Ditclient.runtime.seconds=120".
+ */
+public class ITClient extends BaseKuduTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ITClient.class);
+
+ private static final String RUNTIME_PROPERTY_NAME = "itclient.runtime.seconds";
+ private static final long DEFAULT_RUNTIME_SECONDS = 60;
+ // Time we'll spend waiting at the end of the test for things to settle. Also the minimum this
+ // test can run for.
+ private static final long TEST_MIN_RUNTIME_SECONDS = 2;
+ private static final long TEST_TIMEOUT_SECONDS = 600000;
+
+ private static final String TABLE_NAME =
+ ITClient.class.getName() + "-" + System.currentTimeMillis();
+ // One error and we stop the test.
+ private static final CountDownLatch KEEP_RUNNING_LATCH = new CountDownLatch(1);
+ // Latch used to track if an error occurred and we need to stop the test early.
+ private static final CountDownLatch ERROR_LATCH = new CountDownLatch(1);
+
+ private static KuduClient localClient;
+ private static AsyncKuduClient localAsyncClient;
+ private static KuduTable table;
+ private static long runtimeInSeconds;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+
+ String runtimeProp = System.getProperty(RUNTIME_PROPERTY_NAME);
+ runtimeInSeconds = runtimeProp == null ? DEFAULT_RUNTIME_SECONDS : Long.parseLong(runtimeProp);
+
+ if (runtimeInSeconds < TEST_MIN_RUNTIME_SECONDS || runtimeInSeconds > TEST_TIMEOUT_SECONDS) {
+ Assert.fail("This test needs to run more more than " + TEST_MIN_RUNTIME_SECONDS + " seconds" +
+ " and less than " + TEST_TIMEOUT_SECONDS + " seconds");
+ }
+
+ LOG.info ("Test running for {} seconds", runtimeInSeconds);
+
+ BaseKuduTest.setUpBeforeClass();
+
+ // Client we're using has low tolerance for read timeouts but a
+ // higher overall operation timeout.
+ localAsyncClient = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
+ .defaultSocketReadTimeoutMs(500)
+ .build();
+ localClient = new KuduClient(localAsyncClient);
+
+ CreateTableOptions builder = new CreateTableOptions().setNumReplicas(3);
+ builder.setRangePartitionColumns(ImmutableList.of("key"));
+ table = localClient.createTable(TABLE_NAME, basicSchema, builder);
+ }
+
+ @Test(timeout = TEST_TIMEOUT_SECONDS)
+ public void test() throws Exception {
+ ArrayList<Thread> threads = new ArrayList<>();
+ Thread chaosThread = new Thread(new ChaosThread());
+ Thread writerThread = new Thread(new WriterThread());
+ Thread scannerThread = new Thread(new ScannerThread());
+
+ threads.add(chaosThread);
+ threads.add(writerThread);
+ threads.add(scannerThread);
+
+ for (Thread thread : threads) {
+ thread.start();
+ }
+
+ // await() returns yes if the latch reaches 0, we don't want that.
+ Assert.assertFalse("Look for the last ERROR line in the log that comes from ITCLient",
+ ERROR_LATCH.await(runtimeInSeconds, TimeUnit.SECONDS));
+
+ // Indicate we want to stop, then wait a little bit for it to happen.
+ KEEP_RUNNING_LATCH.countDown();
+
+ for (Thread thread : threads) {
+ thread.interrupt();
+ thread.join();
+ }
+
+ AsyncKuduScanner scannerBuilder = localAsyncClient.newScannerBuilder(table).build();
+ int rowCount = countRowsInScan(scannerBuilder);
+ Assert.assertTrue(rowCount + " should be higher than 0", rowCount > 0);
+ }
+
+ /**
+ * Logs an error message and triggers the error count down latch, stopping this test.
+ * @param message error message to print
+ * @param exception optional exception to print
+ */
+ private void reportError(String message, Exception exception) {
+ LOG.error(message, exception);
+ ERROR_LATCH.countDown();
+ }
+
+ /**
+ * Thread that introduces chaos in the cluster, one at a time.
+ */
+ class ChaosThread implements Runnable {
+
+ private final Random random = new Random();
+
+ @Override
+ public void run() {
+ try {
+ KEEP_RUNNING_LATCH.await(2, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ return;
+ }
+ while (KEEP_RUNNING_LATCH.getCount() > 0) {
+ try {
+ boolean shouldContinue;
+ if (System.currentTimeMillis() % 2 == 0) {
+ shouldContinue = restartTS();
+ } else {
+
+ shouldContinue = disconnectNode();
+ }
+ // TODO restarting the master currently finds more bugs. Also, adding it to the list makes
+ // it necessary to find a new weighing mechanism betweent he different chaos options.
+ // shouldContinue = restartMaster();
+
+ if (!shouldContinue) {
+ return;
+ }
+ KEEP_RUNNING_LATCH.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ return;
+ }
+
+ }
+ }
+
+ /**
+ * Failure injection. Picks a random tablet server from the client's cache and force
+ * disconects it.
+ * @return true if successfully completed or didn't find a server to disconnect, false it it
+ * encountered a failure
+ */
+ private boolean disconnectNode() {
+ try {
+ if (localAsyncClient.getTabletClients().size() == 0) {
+ return true;
+ }
+
+ int tsToDisconnect = random.nextInt(localAsyncClient.getTabletClients().size());
+ localAsyncClient.getTabletClients().get(tsToDisconnect).disconnect();
+
+ } catch (Exception e) {
+ if (KEEP_RUNNING_LATCH.getCount() == 0) {
+ // Likely shutdown() related.
+ return false;
+ }
+ reportError("Couldn't disconnect a TS", e);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Forces the restart of a random tablet server.
+ * @return true if it successfully completed, false if it failed
+ */
+ private boolean restartTS() {
+ try {
+ BaseKuduTest.restartTabletServer(table);
+ } catch (Exception e) {
+ reportError("Couldn't restart a TS", e);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Forces the restart of the master.
+ * @return true if it successfully completed, false if it failed
+ */
+ private boolean restartMaster() {
+ try {
+ BaseKuduTest.restartLeaderMaster();
+ } catch (Exception e) {
+ reportError("Couldn't restart a master", e);
+ return false;
+ }
+ return true;
+ }
+
+ }
+
+ /**
+ * Thread that writes sequentially to the table. Every 10 rows it considers setting the flush mode
+ * to MANUAL_FLUSH or AUTO_FLUSH_SYNC.
+ */
+ class WriterThread implements Runnable {
+
+ private final KuduSession session = localClient.newSession();
+ private final Random random = new Random();
+ private int currentRowKey = 0;
+
+ @Override
+ public void run() {
+ while (KEEP_RUNNING_LATCH.getCount() > 0) {
+ try {
+ OperationResponse resp = session.apply(createBasicSchemaInsert(table, currentRowKey));
+ if (hasRowErrorAndReport(resp)) {
+ return;
+ }
+ currentRowKey++;
+
+ // Every 10 rows we flush and change the flush mode randomly.
+ if (currentRowKey % 10 == 0) {
+
+ // First flush any accumulated rows before switching.
+ List<OperationResponse> responses = session.flush();
+ if (responses != null) {
+ for (OperationResponse batchedResp : responses) {
+ if (hasRowErrorAndReport(batchedResp)) {
+ return;
+ }
+ }
+ }
+
+ if (random.nextBoolean()) {
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ } else {
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
+ }
+ }
+ } catch (Exception e) {
+ if (KEEP_RUNNING_LATCH.getCount() == 0) {
+ // Likely shutdown() related.
+ return;
+ }
+ reportError("Got error while inserting row " + currentRowKey, e);
+ return;
+ }
+ }
+ }
+
+ private boolean hasRowErrorAndReport(OperationResponse resp) {
+ if (resp != null && resp.hasRowError()) {
+ reportError("The following RPC " + resp.getOperation().getRow() +
+ " returned this error: " + resp.getRowError(), null);
+ return true;
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Thread that scans the table. Alternates randomly between random gets and full table scans.
+ */
+ class ScannerThread implements Runnable {
+
+ private final Random random = new Random();
+
+ // Updated by calling a full scan.
+ private int lastRowCount = 0;
+
+ @Override
+ public void run() {
+ while (KEEP_RUNNING_LATCH.getCount() > 0) {
+
+ boolean shouldContinue;
+
+ // Always scan until we find rows.
+ if (lastRowCount == 0 || random.nextBoolean()) {
+ shouldContinue = fullScan();
+ } else {
+ shouldContinue = randomGet();
+ }
+
+ if (!shouldContinue) {
+ return;
+ }
+
+ if (lastRowCount == 0) {
+ try {
+ KEEP_RUNNING_LATCH.await(50, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ // Test is stopping.
+ return;
+ }
+ }
+ }
+ }
+
+ /**
+ * Reads a row at random that it knows to exist (smaller than lastRowCount).
+ * @return
+ */
+ private boolean randomGet() {
+ int key = random.nextInt(lastRowCount);
+ KuduPredicate predicate = KuduPredicate.newComparisonPredicate(
+ table.getSchema().getColumnByIndex(0), KuduPredicate.ComparisonOp.EQUAL, key);
+ KuduScanner scanner = localClient.newScannerBuilder(table).addPredicate(predicate).build();
+
+ List<RowResult> results = new ArrayList<>();
+ while (scanner.hasMoreRows()) {
+ try {
+ RowResultIterator ite = scanner.nextRows();
+ for (RowResult row : ite) {
+ results.add(row);
+ }
+ } catch (Exception e) {
+ return checkAndReportError("Got error while getting row " + key, e);
+ }
+ }
+
+ if (results.isEmpty() || results.size() > 1) {
+ reportError("Random get got 0 or many rows " + results.size() + " for key " + key, null);
+ return false;
+ }
+
+ int receivedKey = results.get(0).getInt(0);
+ if (receivedKey != key) {
+ reportError("Tried to get key " + key + " and received " + receivedKey, null);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Rusn a full table scan and updates the lastRowCount.
+ * @return
+ */
+ private boolean fullScan() {
+ AsyncKuduScanner scannerBuilder = localAsyncClient.newScannerBuilder(table).build();
+ try {
+ int rowCount = countRowsInScan(scannerBuilder);
+ if (rowCount < lastRowCount) {
+ reportError("Row count regressed: " + rowCount + " < " + lastRowCount, null);
+ return false;
+ }
+ lastRowCount = rowCount;
+ LOG.info("New row count {}", lastRowCount);
+ } catch (Exception e) {
+ checkAndReportError("Got error while row counting", e);
+ }
+ return true;
+ }
+
+ /**
+ * Checks the passed exception contains "Scanner not found". If it does then it returns true,
+ * else it reports the error and returns false.
+ * We need to do this because the scans in this client aren't fault tolerant.
+ * @param message message to print if the exception contains a real error
+ * @param e the exception to check
+ * @return true if the scanner failed because it wasn't false, otherwise false
+ */
+ private boolean checkAndReportError(String message, Exception e) {
+ if (!e.getCause().getMessage().contains("Scanner not found")) {
+ reportError(message, e);
+ return false;
+ }
+ return true;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java
new file mode 100644
index 0000000..13465f9
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/ITScannerMultiTablet.java
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.Lists;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kududb.Schema;
+
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Integration test that inserts enough data to trigger flushes and getting multiple data
+ * blocks.
+ */
+public class ITScannerMultiTablet extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ ITScannerMultiTablet.class.getName()+"-"+System.currentTimeMillis();
+ private static final int ROW_COUNT = 20000;
+ private static final int TABLET_COUNT = 3;
+
+ private static Schema schema = getBasicSchema();
+ private static KuduTable table;
+
+ private static Random random = new Random(1234);
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+
+ CreateTableOptions builder = new CreateTableOptions();
+
+ builder.addHashPartitions(
+ Lists.newArrayList(schema.getColumnByIndex(0).getName()),
+ TABLET_COUNT);
+
+ table = createTable(TABLE_NAME, schema, builder);
+
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+
+ // Getting meaty rows.
+ char[] chars = new char[1024];
+ for (int i = 0; i < ROW_COUNT; i++) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addInt(0, random.nextInt());
+ row.addInt(1, i);
+ row.addInt(2, i);
+ row.addString(3, new String(chars));
+ row.addBoolean(4, true);
+ session.apply(insert);
+ }
+ session.flush();
+ assertEquals(0, session.countPendingErrors());
+ }
+
+ /**
+ * Test for KUDU-1343 with a multi-batch multi-tablet scan.
+ */
+ @Test(timeout = 100000)
+ public void testKudu1343() throws Exception {
+ KuduScanner scanner = syncClient.newScannerBuilder(table)
+ .batchSizeBytes(1) // Just a hint, won't actually be that small
+ .build();
+
+ int rowCount = 0;
+ int loopCount = 0;
+ while(scanner.hasMoreRows()) {
+ loopCount++;
+ RowResultIterator rri = scanner.nextRows();
+ while (rri.hasNext()) {
+ rri.next();
+ rowCount++;
+ }
+ }
+
+ assertTrue(loopCount > TABLET_COUNT);
+ assertEquals(ROW_COUNT, rowCount);
+ }
+
+ /**
+ * Makes sure we pass all the correct information down to the server by verifying we get rows in
+ * order from 4 tablets. We detect those tablet boundaries when keys suddenly become smaller than
+ * what was previously seen.
+ */
+ @Test(timeout = 100000)
+ public void testSortResultsByPrimaryKey() throws Exception {
+ KuduScanner scanner = syncClient.newScannerBuilder(table)
+ .sortResultsByPrimaryKey()
+ .setProjectedColumnIndexes(Lists.newArrayList(0))
+ .build();
+
+ int rowCount = 0;
+ int previousRow = -1;
+ int tableBoundariesCount = 0;
+ while(scanner.hasMoreRows()) {
+ RowResultIterator rri = scanner.nextRows();
+ while (rri.hasNext()) {
+ int key = rri.next().getInt(0);
+ if (key < previousRow) {
+ tableBoundariesCount++;
+ }
+ previousRow = key;
+ rowCount++;
+ }
+ }
+ assertEquals(ROW_COUNT, rowCount);
+ assertEquals(TABLET_COUNT, tableBoundariesCount);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
new file mode 100644
index 0000000..955e6ab
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/MiniKuduCluster.java
@@ -0,0 +1,474 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.kududb.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.net.HostAndPort;
+import org.apache.commons.io.FileUtils;
+import org.kududb.util.NetUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utility class to start and manipulate Kudu clusters. Relies on being IN the Kudu source code with
+ * both the kudu-master and kudu-tserver binaries already compiled. {@link BaseKuduTest} should be
+ * extended instead of directly using this class in almost all cases.
+ */
+public class MiniKuduCluster implements AutoCloseable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MiniKuduCluster.class);
+
+ // TS and Master ports will be assigned starting with this one.
+ private static final int PORT_START = 64030;
+
+ // List of threads that print
+ private final List<Thread> PROCESS_INPUT_PRINTERS = new ArrayList<>();
+
+ // Map of ports to master servers.
+ private final Map<Integer, Process> masterProcesses = new ConcurrentHashMap<>();
+
+ // Map of ports to tablet servers.
+ private final Map<Integer, Process> tserverProcesses = new ConcurrentHashMap<>();
+
+ // Map of ports to process command lines. Never removed from. Used to restart processes.
+ private final Map<Integer, String[]> commandLines = new ConcurrentHashMap<>();
+
+ private final List<String> pathsToDelete = new ArrayList<>();
+ private final List<HostAndPort> masterHostPorts = new ArrayList<>();
+ private List<Integer> tserverPorts = new ArrayList<>();
+
+ // Client we can use for common operations.
+ private final KuduClient syncClient;
+ private final int defaultTimeoutMs;
+
+ private String masterAddresses;
+
+ private MiniKuduCluster(int numMasters, int numTservers, int defaultTimeoutMs) throws Exception {
+ this.defaultTimeoutMs = defaultTimeoutMs;
+
+ startCluster(numMasters, numTservers);
+
+ syncClient = new KuduClient.KuduClientBuilder(getMasterAddresses())
+ .defaultAdminOperationTimeoutMs(defaultTimeoutMs)
+ .defaultOperationTimeoutMs(defaultTimeoutMs)
+ .build();
+ }
+
+ /**
+ * Wait up to this instance's "default timeout" for an expected count of TS to
+ * connect to the master.
+ * @param expected How many TS are expected
+ * @return true if there are at least as many TS as expected, otherwise false
+ */
+ public boolean waitForTabletServers(int expected) throws Exception {
+ int count = 0;
+ Stopwatch stopwatch = Stopwatch.createStarted();
+ while (count < expected && stopwatch.elapsed(TimeUnit.MILLISECONDS) < defaultTimeoutMs) {
+ Thread.sleep(200);
+ count = syncClient.listTabletServers().getTabletServersCount();
+ }
+ return count >= expected;
+ }
+
+ /**
+ * Starts a Kudu cluster composed of the provided masters and tablet servers.
+ * @param numMasters how many masters to start
+ * @param numTservers how many tablet servers to start
+ * @throws Exception
+ */
+ private void startCluster(int numMasters, int numTservers) throws Exception {
+ Preconditions.checkArgument(numMasters > 0, "Need at least one master");
+ Preconditions.checkArgument(numTservers > 0, "Need at least one tablet server");
+ // The following props are set via kudu-client's pom.
+ String baseDirPath = TestUtils.getBaseDir();
+ String localhost = TestUtils.getUniqueLocalhost();
+
+ long now = System.currentTimeMillis();
+ LOG.info("Starting {} masters...", numMasters);
+ int startPort = startMasters(PORT_START, numMasters, baseDirPath);
+ LOG.info("Starting {} tablet servers...", numTservers);
+ List<Integer> ports = TestUtils.findFreePorts(startPort, numTservers * 2);
+ for (int i = 0; i < numTservers; i++) {
+ int rpcPort = ports.get(i * 2);
+ tserverPorts.add(rpcPort);
+ String dataDirPath = baseDirPath + "/ts-" + i + "-" + now;
+ String flagsPath = TestUtils.getFlagsPath();
+ String[] tsCmdLine = {
+ TestUtils.findBinary("kudu-tserver"),
+ "--flagfile=" + flagsPath,
+ "--fs_wal_dir=" + dataDirPath,
+ "--fs_data_dirs=" + dataDirPath,
+ "--flush_threshold_mb=1",
+ "--enable_exactly_once",
+ "--tserver_master_addrs=" + masterAddresses,
+ "--webserver_interface=" + localhost,
+ "--local_ip_for_outbound_sockets=" + localhost,
+ "--webserver_port=" + (rpcPort + 1),
+ "--rpc_bind_addresses=" + localhost + ":" + rpcPort};
+ tserverProcesses.put(rpcPort, configureAndStartProcess(rpcPort, tsCmdLine));
+ commandLines.put(rpcPort, tsCmdLine);
+
+ if (flagsPath.startsWith(baseDirPath)) {
+ // We made a temporary copy of the flags; delete them later.
+ pathsToDelete.add(flagsPath);
+ }
+ pathsToDelete.add(dataDirPath);
+ }
+ }
+
+ /**
+ * Start the specified number of master servers with ports starting from a specified
+ * number. Finds free web and RPC ports up front for all of the masters first, then
+ * starts them on those ports, populating 'masters' map.
+ * @param masterStartPort the starting point of the port range for the masters
+ * @param numMasters number of masters to start
+ * @param baseDirPath the base directory where the mini cluster stores its data
+ * @return the next free port
+ * @throws Exception if we are unable to start the masters
+ */
+ private int startMasters(int masterStartPort, int numMasters,
+ String baseDirPath) throws Exception {
+ LOG.info("Starting {} masters...", numMasters);
+ // Get the list of web and RPC ports to use for the master consensus configuration:
+ // request NUM_MASTERS * 2 free ports as we want to also reserve the web
+ // ports for the consensus configuration.
+ String localhost = TestUtils.getUniqueLocalhost();
+ List<Integer> ports = TestUtils.findFreePorts(masterStartPort, numMasters * 2);
+ int lastFreePort = ports.get(ports.size() - 1);
+ List<Integer> masterRpcPorts = Lists.newArrayListWithCapacity(numMasters);
+ List<Integer> masterWebPorts = Lists.newArrayListWithCapacity(numMasters);
+ for (int i = 0; i < numMasters * 2; i++) {
+ if (i % 2 == 0) {
+ masterRpcPorts.add(ports.get(i));
+ masterHostPorts.add(HostAndPort.fromParts(localhost, ports.get(i)));
+ } else {
+ masterWebPorts.add(ports.get(i));
+ }
+ }
+ masterAddresses = NetUtil.hostsAndPortsToString(masterHostPorts);
+ long now = System.currentTimeMillis();
+ for (int i = 0; i < numMasters; i++) {
+ int port = masterRpcPorts.get(i);
+ String dataDirPath = baseDirPath + "/master-" + i + "-" + now;
+ String flagsPath = TestUtils.getFlagsPath();
+ // The web port must be reserved in the call to findFreePorts above and specified
+ // to avoid the scenario where:
+ // 1) findFreePorts finds RPC ports a, b, c for the 3 masters.
+ // 2) start master 1 with RPC port and let it bind to any (specified as 0) web port.
+ // 3) master 1 happens to bind to port b for the web port, as master 2 hasn't been
+ // started yet and findFreePort(s) is "check-time-of-use" (it does not reserve the
+ // ports, only checks that when it was last called, these ports could be used).
+ List<String> masterCmdLine = Lists.newArrayList(
+ TestUtils.findBinary("kudu-master"),
+ "--flagfile=" + flagsPath,
+ "--fs_wal_dir=" + dataDirPath,
+ "--fs_data_dirs=" + dataDirPath,
+ "--webserver_interface=" + localhost,
+ "--local_ip_for_outbound_sockets=" + localhost,
+ "--rpc_bind_addresses=" + localhost + ":" + port,
+ "--webserver_port=" + masterWebPorts.get(i));
+ if (numMasters > 1) {
+ masterCmdLine.add("--master_addresses=" + masterAddresses);
+ }
+ String[] commandLine = masterCmdLine.toArray(new String[masterCmdLine.size()]);
+ masterProcesses.put(port, configureAndStartProcess(port, commandLine));
+ commandLines.put(port, commandLine);
+
+ if (flagsPath.startsWith(baseDirPath)) {
+ // We made a temporary copy of the flags; delete them later.
+ pathsToDelete.add(flagsPath);
+ }
+ pathsToDelete.add(dataDirPath);
+ }
+ return lastFreePort + 1;
+ }
+
+ /**
+ * Starts a process using the provided command and configures it to be daemon,
+ * redirects the stderr to stdout, and starts a thread that will read from the process' input
+ * stream and redirect that to LOG.
+ * @param port rpc port used to identify the process
+ * @param command process and options
+ * @return The started process
+ * @throws Exception Exception if an error prevents us from starting the process,
+ * or if we were able to start the process but noticed that it was then killed (in which case
+ * we'll log the exit value).
+ */
+ private Process configureAndStartProcess(int port, String[] command) throws Exception {
+ LOG.info("Starting process: {}", Joiner.on(" ").join(command));
+ ProcessBuilder processBuilder = new ProcessBuilder(command);
+ processBuilder.redirectErrorStream(true);
+ Process proc = processBuilder.start();
+ ProcessInputStreamLogPrinterRunnable printer =
+ new ProcessInputStreamLogPrinterRunnable(proc.getInputStream());
+ Thread thread = new Thread(printer);
+ thread.setDaemon(true);
+ thread.setName(Iterables.getLast(Splitter.on(File.separatorChar).split(command[0])) + ":" + port);
+ PROCESS_INPUT_PRINTERS.add(thread);
+ thread.start();
+
+ Thread.sleep(300);
+ try {
+ int ev = proc.exitValue();
+ throw new Exception("We tried starting a process (" + command[0] + ") but it exited with " +
+ "value=" + ev);
+ } catch (IllegalThreadStateException ex) {
+ // This means the process is still alive, it's like reverse psychology.
+ }
+ return proc;
+ }
+
+ /**
+ * Starts a previously killed master process on the specified port.
+ * @param port which port the master was listening on for RPCs
+ * @throws Exception
+ */
+ public void restartDeadMasterOnPort(int port) throws Exception {
+ restartDeadProcessOnPort(port, masterProcesses);
+ }
+
+ /**
+ * Starts a previously killed tablet server process on the specified port.
+ * @param port which port the TS was listening on for RPCs
+ * @throws Exception
+ */
+ public void restartDeadTabletServerOnPort(int port) throws Exception {
+ restartDeadProcessOnPort(port, tserverProcesses);
+ }
+
+ private void restartDeadProcessOnPort(int port, Map<Integer, Process> map) throws Exception {
+ if (!commandLines.containsKey(port)) {
+ String message = "Cannot start process on unknown port " + port;
+ LOG.warn(message);
+ throw new RuntimeException(message);
+ }
+
+ if (map.containsKey(port)) {
+ String message = "Process already exists on port " + port;
+ LOG.warn(message);
+ throw new RuntimeException(message);
+ }
+
+ String[] commandLine = commandLines.get(port);
+ map.put(port, configureAndStartProcess(port, commandLine));
+ }
+
+ /**
+ * Kills the TS listening on the provided port. Doesn't do anything if the TS was already killed.
+ * @param port port on which the tablet server is listening on
+ * @throws InterruptedException
+ */
+ public void killTabletServerOnPort(int port) throws InterruptedException {
+ Process ts = tserverProcesses.remove(port);
+ if (ts == null) {
+ // The TS is already dead, good.
+ return;
+ }
+ LOG.info("Killing server at port " + port);
+ ts.destroy();
+ ts.waitFor();
+ }
+
+ /**
+ * Kills all tablet servers.
+ * @throws InterruptedException
+ */
+ public void killTabletServers() throws InterruptedException {
+ for (Process tserver : tserverProcesses.values()) {
+ tserver.destroy();
+ tserver.waitFor();
+ }
+ tserverProcesses.clear();
+ }
+
+ /**
+ * Restarts the dead tablet servers on the port.
+ * @throws Exception
+ */
+ public void restartDeadTabletServers() throws Exception {
+ for (int port : tserverPorts) {
+ restartDeadTabletServerOnPort(port);
+ }
+ }
+
+ /**
+ * Kills the master listening on the provided port. Doesn't do anything if the master was
+ * already killed.
+ * @param port port on which the master is listening on
+ * @throws InterruptedException
+ */
+ public void killMasterOnPort(int port) throws InterruptedException {
+ Process master = masterProcesses.remove(port);
+ if (master == null) {
+ // The master is already dead, good.
+ return;
+ }
+ LOG.info("Killing master at port " + port);
+ master.destroy();
+ master.waitFor();
+ }
+
+ /**
+ * See {@link #shutdown()}.
+ * @throws Exception never thrown, exceptions are logged
+ */
+ @Override
+ public void close() throws Exception {
+ shutdown();
+ }
+
+ /**
+ * Stops all the processes and deletes the folders used to store data and the flagfile.
+ */
+ public void shutdown() {
+ for (Iterator<Process> masterIter = masterProcesses.values().iterator(); masterIter.hasNext(); ) {
+ masterIter.next().destroy();
+ masterIter.remove();
+ }
+ for (Iterator<Process> tsIter = tserverProcesses.values().iterator(); tsIter.hasNext(); ) {
+ tsIter.next().destroy();
+ tsIter.remove();
+ }
+ for (Thread thread : PROCESS_INPUT_PRINTERS) {
+ thread.interrupt();
+ }
+
+ for (String path : pathsToDelete) {
+ try {
+ File f = new File(path);
+ if (f.isDirectory()) {
+ FileUtils.deleteDirectory(f);
+ } else {
+ f.delete();
+ }
+ } catch (Exception e) {
+ LOG.warn("Could not delete path {}", path, e);
+ }
+ }
+ }
+
+ /**
+ * Returns the comma-separated list of master addresses.
+ * @return master addresses
+ */
+ public String getMasterAddresses() {
+ return masterAddresses;
+ }
+
+ /**
+ * Returns a list of master addresses.
+ * @return master addresses
+ */
+ public List<HostAndPort> getMasterHostPorts() {
+ return masterHostPorts;
+ }
+
+ /**
+ * Returns an unmodifiable map of all tablet servers in pairs of RPC port - > Process.
+ * @return an unmodifiable map of all tablet servers
+ */
+ @VisibleForTesting
+ Map<Integer, Process> getTabletServerProcesses() {
+ return Collections.unmodifiableMap(tserverProcesses);
+ }
+
+ /**
+ * Returns an unmodifiable map of all masters in pairs of RPC port - > Process.
+ * @return an unmodifiable map of all masters
+ */
+ @VisibleForTesting
+ Map<Integer, Process> getMasterProcesses() {
+ return Collections.unmodifiableMap(masterProcesses);
+ }
+
+ /**
+ * Helper runnable that receives stdout and logs it along with the process' identifier.
+ */
+ private static class ProcessInputStreamLogPrinterRunnable implements Runnable {
+
+ private final InputStream is;
+
+ public ProcessInputStreamLogPrinterRunnable(InputStream is) {
+ this.is = is;
+ }
+
+ @Override
+ public void run() {
+ try {
+ String line;
+ BufferedReader in = new BufferedReader(new InputStreamReader(is));
+ while ((line = in.readLine()) != null) {
+ LOG.info(line);
+ }
+ in.close();
+ }
+ catch (Exception e) {
+ if (!e.getMessage().contains("Stream closed")) {
+ LOG.error("Caught error while reading a process' output", e);
+ }
+ }
+ }
+ }
+
+ public static class MiniKuduClusterBuilder {
+
+ private int numMasters = 1;
+ private int numTservers = 3;
+ private int defaultTimeoutMs = 50000;
+
+ public MiniKuduClusterBuilder numMasters(int numMasters) {
+ this.numMasters = numMasters;
+ return this;
+ }
+
+ public MiniKuduClusterBuilder numTservers(int numTservers) {
+ this.numTservers = numTservers;
+ return this;
+ }
+
+ /**
+ * Configures the internal client to use the given timeout for all operations. Also uses the
+ * timeout for tasks like waiting for tablet servers to check in with the master.
+ * @param defaultTimeoutMs timeout in milliseconds
+ * @return this instance
+ */
+ public MiniKuduClusterBuilder defaultTimeoutMs(int defaultTimeoutMs) {
+ this.defaultTimeoutMs = defaultTimeoutMs;
+ return this;
+ }
+
+ public MiniKuduCluster build() throws Exception {
+ return new MiniKuduCluster(numMasters, numTservers, defaultTimeoutMs);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
new file mode 100644
index 0000000..abec53f
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduClient.java
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
+import com.google.protobuf.ByteString;
+import com.stumbleupon.async.Deferred;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kududb.Common;
+import org.kududb.consensus.Metadata;
+import org.kududb.master.Master;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.*;
+
+public class TestAsyncKuduClient extends BaseKuduTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TestAsyncKuduClient.class);
+
+ private static final String TABLE_NAME =
+ TestAsyncKuduClient.class.getName() + "-" + System.currentTimeMillis();
+ private static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ // Set to 1 for testDisconnect to always test disconnecting the right server.
+ CreateTableOptions options = getBasicCreateTableOptions().setNumReplicas(1);
+ table = createTable(TABLE_NAME, basicSchema, options);
+ }
+
+ @Test(timeout = 100000)
+ public void testDisconnect() throws Exception {
+ // Test that we can reconnect to a TS after a disconnection.
+ // 1. Warm up the cache.
+ assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
+
+ // 2. Disconnect the client.
+ disconnectAndWait();
+
+ // 3. Count again, it will trigger a re-connection and we should not hang or fail to scan.
+ assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
+
+ // Test that we can reconnect to a TS while scanning.
+ // 1. Insert enough rows to have to call next() multiple times.
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
+ int rowCount = 200;
+ for (int i = 0; i < rowCount; i++) {
+ session.apply(createBasicSchemaInsert(table, i));
+ }
+ session.flush();
+
+ // 2. Start a scanner with a small max num bytes.
+ AsyncKuduScanner scanner = client.newScannerBuilder(table)
+ .batchSizeBytes(1)
+ .build();
+ Deferred<RowResultIterator> rri = scanner.nextRows();
+ // 3. Register the number of rows we get back. We have no control over how many rows are
+ // returned. When this test was written we were getting 100 rows back.
+ int numRows = rri.join(DEFAULT_SLEEP).getNumRows();
+ assertNotEquals("The TS sent all the rows back, we can't properly test disconnection",
+ rowCount, numRows);
+
+ // 4. Disconnect the client.
+ disconnectAndWait();
+
+ // 5. Make sure that we can continue scanning and that we get the remaining rows back.
+ assertEquals(rowCount - numRows, countRowsInScan(scanner));
+ }
+
+ private void disconnectAndWait() throws InterruptedException {
+ for (TabletClient tabletClient : client.getTabletClients()) {
+ tabletClient.disconnect();
+ }
+ Stopwatch sw = Stopwatch.createStarted();
+ while (sw.elapsed(TimeUnit.MILLISECONDS) < DEFAULT_SLEEP) {
+ if (!client.getTabletClients().isEmpty()) {
+ Thread.sleep(50);
+ } else {
+ break;
+ }
+ }
+ assertTrue(client.getTabletClients().isEmpty());
+ }
+
+ @Test
+ public void testBadHostnames() throws Exception {
+ String badHostname = "some-unknown-host-hopefully";
+
+ // Test that a bad hostname for the master makes us error out quickly.
+ AsyncKuduClient invalidClient = new AsyncKuduClient.AsyncKuduClientBuilder(badHostname).build();
+ try {
+ invalidClient.listTabletServers().join(1000);
+ fail("This should have failed quickly");
+ } catch (Exception ex) {
+ assertTrue(ex instanceof NonRecoverableException);
+ assertTrue(ex.getMessage().contains(badHostname));
+ }
+
+ List<Master.TabletLocationsPB> tabletLocations = new ArrayList<>();
+
+ // Builder three bad locations.
+ Master.TabletLocationsPB.Builder tabletPb = Master.TabletLocationsPB.newBuilder();
+ for (int i = 0; i < 3; i++) {
+ Common.PartitionPB.Builder partition = Common.PartitionPB.newBuilder();
+ partition.setPartitionKeyStart(ByteString.copyFrom("a" + i, Charsets.UTF_8.name()));
+ partition.setPartitionKeyEnd(ByteString.copyFrom("b" + i, Charsets.UTF_8.name()));
+ tabletPb.setPartition(partition);
+ tabletPb.setTabletId(ByteString.copyFromUtf8("some id " + i));
+ Master.TSInfoPB.Builder tsInfoBuilder = Master.TSInfoPB.newBuilder();
+ Common.HostPortPB.Builder hostBuilder = Common.HostPortPB.newBuilder();
+ hostBuilder.setHost(badHostname + i);
+ hostBuilder.setPort(i);
+ tsInfoBuilder.addRpcAddresses(hostBuilder);
+ tsInfoBuilder.setPermanentUuid(ByteString.copyFromUtf8("some uuid"));
+ Master.TabletLocationsPB.ReplicaPB.Builder replicaBuilder =
+ Master.TabletLocationsPB.ReplicaPB.newBuilder();
+ replicaBuilder.setTsInfo(tsInfoBuilder);
+ replicaBuilder.setRole(Metadata.RaftPeerPB.Role.FOLLOWER);
+ tabletPb.addReplicas(replicaBuilder);
+ tabletLocations.add(tabletPb.build());
+ }
+
+ // Test that a tablet full of unreachable replicas won't make us retry.
+ try {
+ KuduTable badTable = new KuduTable(client, "Invalid table name",
+ "Invalid table ID", null, null);
+ client.discoverTablets(badTable, tabletLocations);
+ fail("This should have failed quickly");
+ } catch (Exception ex) {
+ assertTrue(ex instanceof NonRecoverableException);
+ assertTrue(ex.getMessage().contains(badHostname));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
new file mode 100644
index 0000000..ba69305
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestAsyncKuduSession.java
@@ -0,0 +1,514 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.Schema;
+import org.kududb.WireProtocol.AppStatusPB;
+import org.kududb.client.AsyncKuduClient.RemoteTablet;
+import org.kududb.tserver.Tserver.TabletServerErrorPB;
+
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+import com.stumbleupon.async.DeferredGroupException;
+import com.stumbleupon.async.TimeoutException;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * This class can either start its own cluster or rely on an existing one.
+ * By default it assumes that the master is at localhost:64000.
+ * The cluster's configuration flags is found at flagsPath as defined in the pom file.
+ * Set startCluster to true in order have the test start the cluster for you.
+ * All those properties are set via surefire's systemPropertyVariables, meaning this:
+ * $ mvn test -DstartCluster=false
+ * will use an existing cluster at default address found above.
+ *
+ * The test creates a table with a unique(ish) name which it deletes at the end.
+ */
+public class TestAsyncKuduSession extends BaseKuduTest {
+ // Generate a unique table name
+ private static final String TABLE_NAME =
+ TestAsyncKuduSession.class.getName()+"-"+System.currentTimeMillis();
+
+ private static Schema schema = getBasicSchema();
+ private static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ table = createTable(TABLE_NAME, schema, getBasicCreateTableOptions());
+ }
+
+ /**
+ * Regression test for case where an error in the previous batch could cause the next
+ * batch to hang in flush()
+ */
+ @Test(timeout = 100000)
+ public void testBatchErrorCauseSessionStuck() throws Exception {
+ try {
+ AsyncKuduSession session = client.newSession();
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
+ session.setFlushInterval(100);
+ TabletServerErrorPB error = TabletServerErrorPB.newBuilder()
+ .setCode(TabletServerErrorPB.Code.UNKNOWN_ERROR)
+ .setStatus(AppStatusPB.newBuilder()
+ .setCode(AppStatusPB.ErrorCode.UNKNOWN_ERROR)
+ .setMessage("injected error for test")
+ .build())
+ .build();
+ Batch.injectTabletServerErrorAndLatency(error, 200);
+ // 0ms: insert first row, which will be the first batch.
+ Deferred<OperationResponse> resp1 = session.apply(createInsert(1));
+ Thread.sleep(120);
+ // 100ms: start to send first batch.
+ // 100ms+: first batch got response from ts,
+ // will wait 200s and throw erorr.
+ // 120ms: insert another row, which will be the second batch.
+ Deferred<OperationResponse> resp2 = session.apply(createInsert(2));
+ // 220ms: start to send the second batch, but first batch is inflight,
+ // so add callback to retry after first batch finishes.
+ // 300ms: first batch's callback handles error, retry second batch.
+ try {
+ resp1.join(2000);
+ } catch (TimeoutException e) {
+ fail("First batch should not timeout in case of tablet server error");
+ } catch (KuduException e) {
+ // Expected.
+ assertTrue(e.getMessage().contains("injected error for test"));
+ }
+ try {
+ resp2.join(2000);
+ } catch (TimeoutException e) {
+ fail("Second batch should not timeout in case of tablet server error");
+ } catch (KuduException e) {
+ // expected
+ assertTrue(e.getMessage().contains("injected error for test"));
+ }
+ } finally {
+ Batch.injectTabletServerErrorAndLatency(null, 0);
+ }
+ }
+
+ /**
+ * Regression test for case when tablet lookup error causes original RPC to get stuck.
+ * @throws Exception
+ */
+ @Test(timeout = 100000)
+ public void testGetTableLocationsErrorCauseSessionStuck() throws Exception {
+ AsyncKuduSession session = client.newSession();
+ // Make sure tablet locations is cached.
+ Insert insert = createInsert(1);
+ session.apply(insert).join(DEFAULT_SLEEP);
+ RemoteTablet rt = client.getTablet(table.getTableId(), insert.partitionKey());
+ String tabletId = rt.getTabletIdAsString();
+ TabletClient tc = client.clientFor(rt);
+ try {
+ // Delete table so we get table not found error.
+ client.deleteTable(TABLE_NAME).join();
+ // Wait until tablet is deleted on TS.
+ while (true) {
+ ListTabletsRequest req = new ListTabletsRequest();
+ tc.sendRpc(req);
+ ListTabletsResponse resp = req.getDeferred().join();
+ if (!resp.getTabletsList().contains(tabletId)) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+ try {
+ session.apply(createInsert(1)).join(DEFAULT_SLEEP);
+ fail("Insert should not succeed");
+ } catch (KuduException e) {
+ assertTrue(e.getStatus().isNotFound());
+ } catch (Throwable e) {
+ fail("Should not throw other error: " + e);
+ }
+ } finally {
+ table = createTable(TABLE_NAME, schema, getBasicCreateTableOptions());
+ }
+ }
+
+ /** Regression test for a failure to correctly handle a timeout when flushing a batch. */
+ @Test
+ public void testInsertIntoUnavailableTablet() throws Exception {
+ killTabletServers();
+ try {
+ AsyncKuduSession session = client.newSession();
+ session.setTimeoutMillis(1);
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ Insert insert = createInsert(1);
+ session.apply(insert);
+ try {
+ session.flush().join();
+ fail("expected exception");
+ } catch (DeferredGroupException e) {
+ assertEquals(1, e.results().size());
+ assertTrue(e.results().get(0).toString().contains("timeout"));
+ }
+ } finally {
+ restartTabletServers();
+ }
+ }
+
+ @Test(timeout = 100000)
+ public void test() throws Exception {
+
+ AsyncKuduSession session = client.newSession();
+ // disable the low watermark until we need it
+ session.setMutationBufferLowWatermark(1f);
+
+ // First testing KUDU-232, the cache is empty and we want to force flush. We force the flush
+ // interval to be higher than the sleep time so that we don't background flush while waiting.
+ // If our subsequent manual flush throws, it means the logic to block on in-flight tablet
+ // lookups in flush isn't working properly.
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
+ session.setFlushInterval(DEFAULT_SLEEP + 1000);
+ Deferred<OperationResponse> d = session.apply(createInsert(0));
+ session.flush().join(DEFAULT_SLEEP);
+ assertTrue(exists(0));
+ // set back to default
+ session.setFlushInterval(1000);
+
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
+ for (int i = 1; i < 10; i++) {
+ session.apply(createInsert(i)).join(DEFAULT_SLEEP);
+ }
+
+ assertEquals(10, countInRange(0, 10));
+
+ session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
+ session.setMutationBufferSpace(10);
+
+ session.apply(createInsert(10));
+
+ try {
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
+ } catch (IllegalArgumentException ex) {
+ /* expected, flush mode remains manual */
+ }
+
+ assertFalse(exists(10));
+
+ for (int i = 11; i < 20; i++) {
+ session.apply(createInsert(i));
+ }
+
+ assertEquals(0, countInRange(10, 20));
+ try {
+ session.apply(createInsert(20));
+ } catch (KuduException ex) {
+ /* expected, buffer would be too big */
+ }
+ assertEquals(0, countInRange(10, 20)); // the buffer should still be full
+
+ session.flush().join(DEFAULT_SLEEP);
+ assertEquals(10, countInRange(10, 20)); // now everything should be there
+
+ session.flush().join(DEFAULT_SLEEP); // flushing empty buffer should be a no-op.
+
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
+
+ d = session.apply(createInsert(20));
+ Thread.sleep(50); // waiting a minimal amount of time to make sure the interval is in effect
+ assertFalse(exists(20));
+ // Add 10 items, the last one will stay in the buffer
+ for (int i = 21; i < 30; i++) {
+ d = session.apply(createInsert(i));
+ }
+ Deferred<OperationResponse> buffered = session.apply(createInsert(30));
+ long now = System.currentTimeMillis();
+ d.join(DEFAULT_SLEEP); // Ok to use the last d, everything is going to the buffer
+ // auto flush will force flush if the buffer is full as it should be now
+ // so we check that we didn't wait the full interval
+ long elapsed = System.currentTimeMillis() - now;
+ assertTrue(elapsed < 950);
+ assertEquals(10, countInRange(20, 31));
+ buffered.join();
+ assertEquals(11, countInRange(20, 31));
+
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
+ Update update = createUpdate(30);
+ PartialRow row = update.getRow();
+ row.addInt(2, 999);
+ row.addString(3, "updated data");
+ d = session.apply(update);
+ d.addErrback(defaultErrorCB);
+ d.join(DEFAULT_SLEEP);
+ assertEquals(31, countInRange(0, 31));
+
+ Delete del = createDelete(30);
+ d = session.apply(del);
+ d.addErrback(defaultErrorCB);
+ d.join(DEFAULT_SLEEP);
+ assertEquals(30, countInRange(0, 31));
+
+ session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
+ session.setMutationBufferSpace(35);
+ for (int i = 0; i < 20; i++) {
+ buffered = session.apply(createDelete(i));
+ }
+ assertEquals(30, countInRange(0, 31));
+ session.flush();
+ buffered.join(DEFAULT_SLEEP);
+ assertEquals(10, countInRange(0, 31));
+
+ for (int i = 30; i < 40; i++) {
+ session.apply(createInsert(i));
+ }
+
+ for (int i = 20; i < 30; i++) {
+ buffered = session.apply(createDelete(i));
+ }
+
+ assertEquals(10, countInRange(0, 40));
+ session.flush();
+ buffered.join(DEFAULT_SLEEP);
+ assertEquals(10, countInRange(0, 40));
+
+ // Test nulls
+ // add 10 rows with the nullable column set to null
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
+ for (int i = 40; i < 50; i++) {
+ session.apply(createInsertWithNull(i)).join(DEFAULT_SLEEP);
+ }
+
+ // now scan those rows and make sure the column is null
+ assertEquals(10, countNullColumns(40, 50));
+
+ // Test sending edits too fast
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
+ session.setMutationBufferSpace(10);
+
+ // The buffer has a capacity of 10, we insert 21 rows, meaning we fill the first one,
+ // force flush, fill a second one before the first one could come back,
+ // and the 21st row will be sent back.
+ boolean gotException = false;
+ for (int i = 50; i < 71; i++) {
+ try {
+ session.apply(createInsert(i));
+ } catch (PleaseThrottleException ex) {
+ gotException = true;
+ assertEquals(70, i);
+ // Wait for the buffer to clear
+ ex.getDeferred().join(DEFAULT_SLEEP);
+ session.apply(ex.getFailedRpc());
+ session.flush().join(DEFAULT_SLEEP);
+ }
+ }
+ assertTrue("Expected PleaseThrottleException", gotException);
+ assertEquals(21, countInRange(50, 71));
+
+ // Now test a more subtle issue, basically the race where we call flush from the client when
+ // there's a batch already in flight. We need to finish joining only when all the data is
+ // flushed.
+ for (int i = 71; i < 91; i++) {
+ session.apply(createInsert(i));
+ }
+ session.flush().join(DEFAULT_SLEEP);
+ // If we only waited after the in flight batch, there would be 10 rows here.
+ assertEquals(20, countInRange(71, 91));
+
+ // Test empty scanner projection
+ AsyncKuduScanner scanner = getScanner(71, 91, Collections.<String>emptyList());
+ assertEquals(20, countRowsInScan(scanner));
+
+ // Test removing the connection and then do a rapid set of inserts
+ client.getTabletClients().get(0).shutdown().join(DEFAULT_SLEEP);
+ session.setMutationBufferSpace(1);
+ for (int i = 91; i < 101; i++) {
+ try {
+ session.apply(createInsert(i));
+ } catch (PleaseThrottleException ex) {
+ // Wait for the buffer to clear
+ ex.getDeferred().join(DEFAULT_SLEEP);
+ session.apply(ex.getFailedRpc());
+ }
+ }
+ session.flush().join(DEFAULT_SLEEP);
+ assertEquals(10, countInRange(91, 101));
+
+ // Test a tablet going missing or encountering a new tablet while inserting a lot
+ // of data. This code used to fail in many different ways.
+ client.emptyTabletsCacheForTable(table.getTableId());
+ for (int i = 101; i < 151; i++) {
+ Insert insert = createInsert(i);
+ while (true) {
+ try {
+ session.apply(insert);
+ break;
+ } catch (PleaseThrottleException ex) {
+ // Wait for the buffer to clear
+ ex.getDeferred().join(DEFAULT_SLEEP);
+ }
+ }
+ }
+ session.flush().join(DEFAULT_SLEEP);
+ assertEquals(50, countInRange(101, 151));
+
+ // Test the low watermark.
+ // Before the fix for KUDU-804, a change to the buffer space did not result in a change to the
+ // low watermark causing this test to fail.
+ session.setMutationBufferLowWatermark(0.1f);
+ session.setMutationBufferSpace(10);
+ session.setRandomSeed(12345); // Will make us hit the exception after 6 tries
+ gotException = false;
+ for (int i = 151; i < 171; i++) {
+ try {
+ session.apply(createInsert(i));
+ } catch (PleaseThrottleException ex) {
+ // We're going to hit the exception after filling up the buffer a first time then trying
+ // to insert 6 more rows.
+ assertEquals(167, i);
+ gotException = true;
+ assertTrue(ex.getMessage().contains("watermark"));
+ // Once we hit the exception we wait on the batch to finish flushing and then insert the
+ // rest of the data.
+ ex.getDeferred().join(DEFAULT_SLEEP);
+ session.apply(ex.getFailedRpc());
+ }
+ }
+ session.flush().join(DEFAULT_SLEEP);
+ assertEquals(20, countInRange(151, 171));
+ assertTrue(gotException);
+ }
+
+ private Insert createInsert(int key) {
+ return createBasicSchemaInsert(table, key);
+ }
+
+ private Insert createInsertWithNull(int key) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addInt(0, key);
+ row.addInt(1, 2);
+ row.addInt(2, 3);
+ row.setNull(3);
+ row.addBoolean(4, false);
+ return insert;
+ }
+
+ private Update createUpdate(int key) {
+ Update update = table.newUpdate();
+ PartialRow row = update.getRow();
+ row.addInt(0, key);
+ return update;
+ }
+
+ private Delete createDelete(int key) {
+ Delete delete = table.newDelete();
+ PartialRow row = delete.getRow();
+ row.addInt(0, key);
+ return delete;
+ }
+
+ public static boolean exists(final int key) throws Exception {
+
+ AsyncKuduScanner scanner = getScanner(key, key + 1);
+ final AtomicBoolean exists = new AtomicBoolean(false);
+
+ Callback<Object, RowResultIterator> cb =
+ new Callback<Object, RowResultIterator>() {
+ @Override
+ public Object call(RowResultIterator arg) throws Exception {
+ if (arg == null) return null;
+ for (RowResult row : arg) {
+ if (row.getInt(0) == key) {
+ exists.set(true);
+ break;
+ }
+ }
+ return null;
+ }
+ };
+
+ while (scanner.hasMoreRows()) {
+ Deferred<RowResultIterator> data = scanner.nextRows();
+ data.addCallbacks(cb, defaultErrorCB);
+ data.join(DEFAULT_SLEEP);
+ if (exists.get()) {
+ break;
+ }
+ }
+
+ Deferred<RowResultIterator> closer = scanner.close();
+ closer.join(DEFAULT_SLEEP);
+ return exists.get();
+ }
+
+ public static int countNullColumns(final int startKey, final int endKey) throws Exception {
+
+ AsyncKuduScanner scanner = getScanner(startKey, endKey);
+ final AtomicInteger ai = new AtomicInteger();
+
+ Callback<Object, RowResultIterator> cb = new Callback<Object, RowResultIterator>() {
+ @Override
+ public Object call(RowResultIterator arg) throws Exception {
+ if (arg == null) return null;
+ for (RowResult row : arg) {
+ if (row.isNull(3)) {
+ ai.incrementAndGet();
+ }
+ }
+ return null;
+ }
+ };
+
+ while (scanner.hasMoreRows()) {
+ Deferred<RowResultIterator> data = scanner.nextRows();
+ data.addCallbacks(cb, defaultErrorCB);
+ data.join(DEFAULT_SLEEP);
+ }
+
+ Deferred<RowResultIterator> closer = scanner.close();
+ closer.join(DEFAULT_SLEEP);
+ return ai.get();
+ }
+
+ public static int countInRange(final int start, final int exclusiveEnd) throws Exception {
+
+ AsyncKuduScanner scanner = getScanner(start, exclusiveEnd);
+ return countRowsInScan(scanner);
+ }
+
+ private static AsyncKuduScanner getScanner(int start, int exclusiveEnd) {
+ return getScanner(start, exclusiveEnd, null);
+ }
+
+ private static AsyncKuduScanner getScanner(int start, int exclusiveEnd,
+ List<String> columnNames) {
+
+ PartialRow lowerBound = schema.newPartialRow();
+ lowerBound.addInt(schema.getColumnByIndex(0).getName(), start);
+
+ PartialRow upperBound = schema.newPartialRow();
+ upperBound.addInt(schema.getColumnByIndex(0).getName(), exclusiveEnd);
+
+ AsyncKuduScanner scanner = client.newScannerBuilder(table)
+ .lowerBound(lowerBound)
+ .exclusiveUpperBound(upperBound)
+ .setProjectedColumnNames(columnNames)
+ .build();
+ return scanner;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java
new file mode 100644
index 0000000..ab27e63
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBitSet.java
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.util.BitSet;
+
+public class TestBitSet {
+
+ /**
+ * Test out BitSet-related operations
+ */
+ @Test
+ public void test() {
+ int colCount = 1;
+ BitSet bs = new BitSet(colCount);
+ bs.set(0);
+ int size = Bytes.getBitSetSize(colCount);
+ byte[] result = Bytes.fromBitSet(bs, colCount);
+ assertEquals(size, result.length);
+ BitSet newBs = Bytes.toBitSet(result, 0, colCount);
+ assertTrue(newBs.get(0));
+
+ colCount = 7;
+ bs = new BitSet(colCount);
+ bs.set(0);
+ bs.set(5);
+ size = Bytes.getBitSetSize(colCount);
+ result = Bytes.fromBitSet(bs, colCount);
+ assertEquals(size, result.length);
+ newBs = Bytes.toBitSet(result, 0, colCount);
+ assertTrue(newBs.get(0));
+ assertFalse(newBs.get(1));
+ assertFalse(newBs.get(2));
+ assertFalse(newBs.get(3));
+ assertFalse(newBs.get(4));
+ assertTrue(newBs.get(5));
+ assertFalse(newBs.get(6));
+
+ colCount = 8;
+ bs = new BitSet(colCount);
+ bs.set(0);
+ bs.set(5);
+ bs.set(7);
+ size = Bytes.getBitSetSize(colCount);
+ result = Bytes.fromBitSet(bs, colCount);
+ assertEquals(size, result.length);
+ newBs = Bytes.toBitSet(result, 0, colCount);
+ assertTrue(newBs.get(0));
+ assertFalse(newBs.get(1));
+ assertFalse(newBs.get(2));
+ assertFalse(newBs.get(3));
+ assertFalse(newBs.get(4));
+ assertTrue(newBs.get(5));
+ assertFalse(newBs.get(6));
+ assertTrue(newBs.get(7));
+
+ colCount = 11;
+ bs = new BitSet(colCount);
+ bs.set(0);
+ bs.set(5);
+ bs.set(7);
+ bs.set(9);
+ size = Bytes.getBitSetSize(colCount);
+ result = Bytes.fromBitSet(bs, colCount);
+ assertEquals(size, result.length);
+ newBs = Bytes.toBitSet(result, 0, colCount);
+ assertTrue(newBs.get(0));
+ assertFalse(newBs.get(1));
+ assertFalse(newBs.get(2));
+ assertFalse(newBs.get(3));
+ assertFalse(newBs.get(4));
+ assertTrue(newBs.get(5));
+ assertFalse(newBs.get(6));
+ assertTrue(newBs.get(7));
+ assertFalse(newBs.get(8));
+ assertTrue(newBs.get(9));
+ assertFalse(newBs.get(10));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java
new file mode 100644
index 0000000..11c2035
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestBytes.java
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.math.BigInteger;
+
+public class TestBytes {
+
+ @Test
+ public void test() {
+ byte[] bytes = new byte[8];
+
+ // Boolean
+ Bytes.setUnsignedByte(bytes, (short) 1);
+ assert(Bytes.getBoolean(bytes));
+ Bytes.setUnsignedByte(bytes, (short) 0);
+ assert(!Bytes.getBoolean(bytes));
+
+ // BYTES
+ short smallUbyte = 120;
+ Bytes.setUnsignedByte(bytes, smallUbyte);
+ assertEquals(smallUbyte, Bytes.getUnsignedByte(bytes));
+ short largeUbyte = 250;
+ Bytes.setUnsignedByte(bytes, largeUbyte);
+ assertEquals(largeUbyte, Bytes.getUnsignedByte(bytes));
+
+ // SHORTS
+ short nshort = -300;
+ Bytes.setShort(bytes, nshort);
+ assertEquals(nshort, Bytes.getShort(bytes));
+ short pshort = 300;
+ Bytes.setShort(bytes, pshort);
+ assertEquals(pshort, Bytes.getShort(bytes));
+ int smallUshort = 300;
+ Bytes.setUnsignedShort(bytes, smallUshort);
+ assertEquals(smallUshort, Bytes.getUnsignedShort(bytes));
+ int largeUshort = 60000;
+ Bytes.setUnsignedShort(bytes, largeUshort);
+ assertEquals(largeUshort, Bytes.getUnsignedShort(bytes));
+
+ // INTS
+ int nint = -60000;
+ Bytes.setInt(bytes, nint);
+ assertEquals(nint, Bytes.getInt(bytes));
+ int pint = 60000;
+ Bytes.setInt(bytes, pint);
+ assertEquals(pint, Bytes.getInt(bytes));
+ long smallUint = 60000;
+ Bytes.setUnsignedInt(bytes, smallUint);
+ assertEquals(smallUint, Bytes.getUnsignedInt(bytes));
+ long largeUint = 4000000000L;
+ Bytes.setUnsignedInt(bytes, largeUint);
+ assertEquals(largeUint, Bytes.getUnsignedInt(bytes));
+
+ // LONGS
+ long nlong = -4000000000L;
+ Bytes.setLong(bytes, nlong);
+ assertEquals(nlong, Bytes.getLong(bytes));
+ long plong = 4000000000L;
+ Bytes.setLong(bytes, plong);
+ assertEquals(plong, Bytes.getLong(bytes));
+ BigInteger smallUlong = new BigInteger("4000000000");
+ Bytes.setUnsignedLong(bytes, smallUlong);
+ assertEquals(smallUlong, Bytes.getUnsignedLong(bytes));
+ BigInteger largeUlong = new BigInteger("10000000000000000000");
+ Bytes.setUnsignedLong(bytes, largeUlong);
+ assertEquals(largeUlong, Bytes.getUnsignedLong(bytes));
+
+ // FLOAT
+ float aFloat = 123.456f;
+ Bytes.setFloat(bytes, aFloat);
+ assertEquals(aFloat, Bytes.getFloat(bytes), 0.001);
+
+ // DOUBLE
+ double aDouble = 123.456;
+ Bytes.setDouble(bytes, aDouble);
+ assertEquals(aDouble, Bytes.getDouble(bytes), 0.001);
+ }
+
+ @Test
+ public void testHex() {
+ byte[] bytes = new byte[] { (byte) 0x01, (byte) 0x23, (byte) 0x45, (byte) 0x67,
+ (byte) 0x89, (byte) 0xAB, (byte) 0xCD, (byte) 0xEF };
+ Assert.assertEquals("0x0123456789ABCDEF", Bytes.hex(bytes));
+ }
+}
[09/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java
new file mode 100644
index 0000000..d67380b
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScanPredicate.java
@@ -0,0 +1,609 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedSet;
+import org.junit.Assert;
+import org.junit.Test;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.client.KuduPredicate.ComparisonOp;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+public class TestScanPredicate extends BaseKuduTest {
+
+ private Schema createTableSchema(Type type) {
+ ColumnSchema key = new ColumnSchema.ColumnSchemaBuilder("key", Type.INT64).key(true).build();
+ ColumnSchema val = new ColumnSchema.ColumnSchemaBuilder("value", type).nullable(true).build();
+ return new Schema(ImmutableList.of(key, val));
+ }
+
+ private static CreateTableOptions createTableOptions() {
+ return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
+ }
+
+ private int countRows(KuduTable table, KuduPredicate... predicates) throws Exception {
+ KuduScanner.KuduScannerBuilder scanBuilder = new KuduScanner.KuduScannerBuilder(client, table);
+ for (KuduPredicate predicate : predicates) {
+ scanBuilder.addPredicate(predicate);
+ }
+
+ KuduScanner scanner = scanBuilder.build();
+
+ int count = 0;
+ while (scanner.hasMoreRows()) {
+ count += scanner.nextRows().getNumRows();
+ }
+ return count;
+ }
+
+ private NavigableSet<Long> createIntegerValues(Type type) {
+ NavigableSet<Long> values = new TreeSet<>();
+ for (long i = -50; i < 50; i++) {
+ values.add(i);
+ }
+ values.add(KuduPredicate.minIntValue(type));
+ values.add(KuduPredicate.minIntValue(type) + 1);
+ values.add(KuduPredicate.maxIntValue(type) - 1);
+ values.add(KuduPredicate.maxIntValue(type));
+ return values;
+ }
+
+ private List<Long> createIntegerTestValues(Type type) {
+ return ImmutableList.of(
+ KuduPredicate.minIntValue(type),
+ KuduPredicate.minIntValue(type) + 1,
+ -51L,
+ 50L,
+ 0L,
+ 49L,
+ 50L,
+ KuduPredicate.maxIntValue(type) - 1,
+ KuduPredicate.maxIntValue(type));
+ }
+
+ private NavigableSet<Float> createFloatValues() {
+ NavigableSet<Float> values = new TreeSet<>();
+ for (long i = -50; i < 50; i++) {
+ values.add((float) i + (float) i / 100.0F);
+ }
+
+ values.add(Float.NEGATIVE_INFINITY);
+ values.add(-Float.MAX_VALUE);
+ values.add(-Float.MIN_NORMAL);
+ values.add(-Float.MIN_VALUE);
+ values.add(Float.MIN_VALUE);
+ values.add(Float.MIN_NORMAL);
+ values.add(Float.MAX_VALUE);
+ values.add(Float.POSITIVE_INFINITY);
+
+ // TODO: uncomment after fixing KUDU-1386
+ // values.add(Float.NaN);
+ return values;
+ }
+
+ private List<Float> createFloatTestValues() {
+ return ImmutableList.of(
+ Float.NEGATIVE_INFINITY,
+ -Float.MAX_VALUE,
+ -100.0F,
+ -1.1F,
+ -1.0F,
+ -Float.MIN_NORMAL,
+ -Float.MIN_VALUE,
+ 0.0F,
+ Float.MIN_VALUE,
+ Float.MIN_NORMAL,
+ 1.0F,
+ 1.1F,
+ 100.0F,
+ Float.MAX_VALUE,
+ Float.POSITIVE_INFINITY
+
+ // TODO: uncomment after fixing KUDU-1386
+ // Float.NaN
+ );
+ }
+
+ private NavigableSet<Double> createDoubleValues() {
+ NavigableSet<Double> values = new TreeSet<>();
+ for (long i = -50; i < 50; i++) {
+ values.add((double) i + (double) i / 100.0);
+ }
+
+ values.add(Double.NEGATIVE_INFINITY);
+ values.add(-Double.MAX_VALUE);
+ values.add(-Double.MIN_NORMAL);
+ values.add(-Double.MIN_VALUE);
+ values.add(Double.MIN_VALUE);
+ values.add(Double.MIN_NORMAL);
+ values.add(Double.MAX_VALUE);
+ values.add(Double.POSITIVE_INFINITY);
+
+ // TODO: uncomment after fixing KUDU-1386
+ // values.add(Double.NaN);
+ return values;
+ }
+
+ private List<Double> createDoubleTestValues() {
+ return ImmutableList.of(
+ Double.NEGATIVE_INFINITY,
+ -Double.MAX_VALUE,
+ -100.0,
+ -1.1,
+ -1.0,
+ -Double.MIN_NORMAL,
+ -Double.MIN_VALUE,
+ 0.0,
+ Double.MIN_VALUE,
+ Double.MIN_NORMAL,
+ 1.0,
+ 1.1,
+ 100.0,
+ Double.MAX_VALUE,
+ Double.POSITIVE_INFINITY
+
+ // TODO: uncomment after fixing KUDU-1386
+ // Double.NaN
+ );
+ }
+
+ private NavigableSet<String> createStringValues() {
+ return ImmutableSortedSet.of("", "\0", "\0\0", "a", "a\0", "a\0a", "aa\0");
+ }
+
+ private List<String> createStringTestValues() {
+ List<String> values = new ArrayList<>(createStringValues());
+ values.add("aa");
+ values.add("\1");
+ values.add("a\1");
+ return values;
+ }
+
+ private void checkIntPredicates(KuduTable table,
+ NavigableSet<Long> values,
+ List<Long> testValues) throws Exception {
+ ColumnSchema col = table.getSchema().getColumn("value");
+ Assert.assertEquals(values.size() + 1, countRows(table));
+ for (long v : testValues) {
+ // value = v
+ KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
+ Assert.assertEquals(values.contains(v) ? 1 : 0, countRows(table, equal));
+
+ // value >= v
+ KuduPredicate greaterEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
+ Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
+
+ // value <= v
+ KuduPredicate lessEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
+ Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
+
+ // value > v
+ KuduPredicate greater =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
+ Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
+
+ // value < v
+ KuduPredicate less =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
+ Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
+ }
+ }
+
+ @Test
+ public void testBoolPredicates() throws Exception {
+ Schema schema = createTableSchema(Type.BOOL);
+ syncClient.createTable("bool-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("bool-table");
+
+ NavigableSet<Boolean> values = ImmutableSortedSet.of(false, true);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (boolean value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addBoolean("value", value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ ColumnSchema col = table.getSchema().getColumn("value");
+ Assert.assertEquals(values.size() + 1, countRows(table));
+
+ for (boolean v : values) {
+ // value = v
+ KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
+ Assert.assertEquals(values.contains(v) ? 1 : 0, countRows(table, equal));
+
+ // value >= v
+ KuduPredicate greaterEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
+ Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
+
+ // value <= v
+ KuduPredicate lessEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
+ Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
+
+ // value > v
+ KuduPredicate greater =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
+ Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
+
+ // value < v
+ KuduPredicate less =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
+ Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
+ }
+ }
+
+ @Test
+ public void testBytePredicates() throws Exception {
+ Schema schema = createTableSchema(Type.INT8);
+ syncClient.createTable("byte-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("byte-table");
+
+ NavigableSet<Long> values = createIntegerValues(Type.INT8);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (long value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addByte("value", (byte) value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ checkIntPredicates(table, values, createIntegerTestValues(Type.INT8));
+ }
+
+ @Test
+ public void testShortPredicates() throws Exception {
+ Schema schema = createTableSchema(Type.INT16);
+ syncClient.createTable("short-table", schema,
+ new CreateTableOptions().setRangePartitionColumns(
+ ImmutableList.<String>of()));
+ KuduTable table = syncClient.openTable("short-table");
+
+ NavigableSet<Long> values = createIntegerValues(Type.INT16);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (long value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addShort("value", (short) value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ checkIntPredicates(table, values, createIntegerTestValues(Type.INT16));
+ }
+
+ @Test
+ public void testIntPredicates() throws Exception {
+ Schema schema = createTableSchema(Type.INT32);
+ syncClient.createTable("int-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("int-table");
+
+ NavigableSet<Long> values = createIntegerValues(Type.INT32);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (long value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addInt("value", (int) value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ checkIntPredicates(table, values, createIntegerTestValues(Type.INT32));
+ }
+
+ @Test
+ public void testLongPredicates() throws Exception {
+ Schema schema = createTableSchema(Type.INT64);
+ syncClient.createTable("long-table", schema,
+ new CreateTableOptions().setRangePartitionColumns(
+ ImmutableList.<String>of()));
+ KuduTable table = syncClient.openTable("long-table");
+
+ NavigableSet<Long> values = createIntegerValues(Type.INT64);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (long value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addLong("value", value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ checkIntPredicates(table, values, createIntegerTestValues(Type.INT64));
+ }
+
+ @Test
+ public void testTimestampPredicate() throws Exception {
+ Schema schema = createTableSchema(Type.INT64);
+ syncClient.createTable("timestamp-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("timestamp-table");
+
+ NavigableSet<Long> values = createIntegerValues(Type.INT64);
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (long value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addLong("value", value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ checkIntPredicates(table, values, createIntegerTestValues(Type.INT64));
+ }
+
+ @Test
+ public void testFloatPredicates() throws Exception {
+ Schema schema = createTableSchema(Type.FLOAT);
+ syncClient.createTable("float-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("float-table");
+
+ NavigableSet<Float> values = createFloatValues();
+ List<Float> testValues = createFloatTestValues();
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (float value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addFloat("value", value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ ColumnSchema col = table.getSchema().getColumn("value");
+ Assert.assertEquals(values.size() + 1, countRows(table));
+
+ for (float v : testValues) {
+ // value = v
+ KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
+ Assert.assertEquals(values.subSet(v, true, v, true).size(), countRows(table, equal));
+
+ // value >= v
+ KuduPredicate greaterEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
+ Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
+
+ // value <= v
+ KuduPredicate lessEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
+ Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
+
+ // value > v
+ KuduPredicate greater =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
+ Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
+
+ // value < v
+ KuduPredicate less =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
+ Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
+ }
+ }
+
+ @Test
+ public void testDoublePredicates() throws Exception {
+ Schema schema = createTableSchema(Type.DOUBLE);
+ syncClient.createTable("double-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("double-table");
+
+ NavigableSet<Double> values = createDoubleValues();
+ List<Double> testValues = createDoubleTestValues();
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (double value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addDouble("value", value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ ColumnSchema col = table.getSchema().getColumn("value");
+ Assert.assertEquals(values.size() + 1, countRows(table));
+
+ for (double v : testValues) {
+ // value = v
+ KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
+ Assert.assertEquals(values.subSet(v, true, v, true).size(), countRows(table, equal));
+
+ // value >= v
+ KuduPredicate greaterEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
+ Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
+
+ // value <= v
+ KuduPredicate lessEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
+ Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
+
+ // value > v
+ KuduPredicate greater =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
+ Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
+
+ // value < v
+ KuduPredicate less =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
+ Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
+ }
+ }
+
+ @Test
+ public void testStringPredicates() throws Exception {
+ Schema schema = createTableSchema(Type.STRING);
+ syncClient.createTable("string-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("string-table");
+
+ NavigableSet<String> values = createStringValues();
+ List<String> testValues = createStringTestValues();
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (String value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addString("value", value);
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ ColumnSchema col = table.getSchema().getColumn("value");
+ Assert.assertEquals(values.size() + 1, countRows(table));
+
+ for (String v : testValues) {
+ // value = v
+ KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
+ Assert.assertEquals(values.subSet(v, true, v, true).size(), countRows(table, equal));
+
+ // value >= v
+ KuduPredicate greaterEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
+ Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
+
+ // value <= v
+ KuduPredicate lessEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
+ Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
+
+ // value > v
+ KuduPredicate greater =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
+ Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
+
+ // value < v
+ KuduPredicate less =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
+ Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
+ }
+ }
+
+ @Test
+ public void testBinaryPredicates() throws Exception {
+ Schema schema = createTableSchema(Type.BINARY);
+ syncClient.createTable("binary-table", schema, createTableOptions());
+ KuduTable table = syncClient.openTable("binary-table");
+
+ NavigableSet<String> values = createStringValues();
+ List<String> testValues = createStringTestValues();
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ long i = 0;
+ for (String value : values) {
+ Insert insert = table.newInsert();
+ insert.getRow().addLong("key", i++);
+ insert.getRow().addBinary("value", Bytes.fromString(value));
+ session.apply(insert);
+ }
+ Insert nullInsert = table.newInsert();
+ nullInsert.getRow().addLong("key", i++);
+ nullInsert.getRow().setNull("value");
+ session.apply(nullInsert);
+ session.flush();
+
+ ColumnSchema col = table.getSchema().getColumn("value");
+ Assert.assertEquals(values.size() + 1, countRows(table));
+
+ for (String s : testValues) {
+ byte[] v = Bytes.fromString(s);
+ // value = v
+ KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
+ Assert.assertEquals(values.subSet(s, true, s, true).size(), countRows(table, equal));
+
+ // value >= v
+ KuduPredicate greaterEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
+ Assert.assertEquals(values.tailSet(s).size(), countRows(table, greaterEqual));
+
+ // value <= v
+ KuduPredicate lessEqual =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
+ Assert.assertEquals(values.headSet(s, true).size(), countRows(table, lessEqual));
+
+ // value > v
+ KuduPredicate greater =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
+ Assert.assertEquals(values.tailSet(s, false).size(), countRows(table, greater));
+
+ // value < v
+ KuduPredicate less =
+ KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
+ Assert.assertEquals(values.headSet(s).size(), countRows(table, less));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
new file mode 100644
index 0000000..251057f
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
@@ -0,0 +1,236 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.stumbleupon.async.Deferred;
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertNull;
+import static org.kududb.Type.STRING;
+import static org.junit.Assert.assertEquals;
+
+public class TestScannerMultiTablet extends BaseKuduTest {
+ // Generate a unique table name
+ private static final String TABLE_NAME =
+ TestScannerMultiTablet.class.getName()+"-"+System.currentTimeMillis();
+
+ private static Schema schema = getSchema();
+ private static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ // create a 4-tablets table for scanning
+ CreateTableOptions builder =
+ new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key1", "key2"));
+
+ for (int i = 1; i < 4; i++){
+ PartialRow splitRow = schema.newPartialRow();
+ splitRow.addString("key1", "" + i);
+ splitRow.addString("key2", "");
+ builder.addSplitRow(splitRow);
+ }
+
+ createTable(TABLE_NAME, schema, builder);
+
+ table = openTable(TABLE_NAME);
+
+ AsyncKuduSession session = client.newSession();
+ session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
+
+ // The data layout ends up like this:
+ // tablet '', '1': no rows
+ // tablet '1', '2': '111', '122', '133'
+ // tablet '2', '3': '211', '222', '233'
+ // tablet '3', '': '311', '322', '333'
+ String[] keys = new String[] {"1", "2", "3"};
+ for (String key1 : keys) {
+ for (String key2 : keys) {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addString(0, key1);
+ row.addString(1, key2);
+ row.addString(2, key2);
+ Deferred<OperationResponse> d = session.apply(insert);
+ d.join(DEFAULT_SLEEP);
+ }
+ }
+ }
+
+ // Test various combinations of start/end row keys.
+ @Test(timeout = 100000)
+ public void testKeyStartEnd() throws Exception {
+ assertEquals(0,
+ countRowsInScan(getScanner("", "", "1", ""))); // There's nothing in the 1st tablet
+ assertEquals(1, countRowsInScan(getScanner("", "", "1", "2"))); // Grab the very first row
+ assertEquals(3, countRowsInScan(getScanner("1", "1", "1", "4"))); // Grab the whole 2nd tablet
+ assertEquals(3, countRowsInScan(getScanner("1", "1", "2", ""))); // Same, and peek at the 3rd
+ assertEquals(3, countRowsInScan(getScanner("1", "1", "2", "0"))); // Same, different peek
+ assertEquals(4,
+ countRowsInScan(getScanner("1", "2", "2", "3"))); // Middle of 2nd to middle of 3rd
+ assertEquals(3,
+ countRowsInScan(getScanner("1", "4", "2", "4"))); // Peek at the 2nd then whole 3rd
+ assertEquals(6, countRowsInScan(getScanner("1", "5", "3", "4"))); // Whole 3rd and 4th
+ assertEquals(9, countRowsInScan(getScanner("", "", "4", ""))); // Full table scan
+
+ assertEquals(9,
+ countRowsInScan(getScanner("", "", null, null))); // Full table scan with empty upper
+ assertEquals(9,
+ countRowsInScan(getScanner(null, null, "4", ""))); // Full table scan with empty lower
+ assertEquals(9,
+ countRowsInScan(getScanner(null, null, null, null))); // Full table scan with empty bounds
+
+ // Test that we can close a scanner while in between two tablets. We start on the second
+ // tablet and our first nextRows() will get 3 rows. At that moment we want to close the scanner
+ // before getting on the 3rd tablet.
+ AsyncKuduScanner scanner = getScanner("1", "", null, null);
+ Deferred<RowResultIterator> d = scanner.nextRows();
+ RowResultIterator rri = d.join(DEFAULT_SLEEP);
+ assertEquals(3, rri.getNumRows());
+ d = scanner.close();
+ rri = d.join(DEFAULT_SLEEP);
+ assertNull(rri);
+ }
+
+ // Test mixing start/end row keys with predicates.
+ @Test(timeout = 100000)
+ public void testKeysAndPredicates() throws Exception {
+ // First row from the 2nd tablet.
+ ColumnRangePredicate predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
+ predicate.setLowerBound("1");
+ predicate.setUpperBound("1");
+ assertEquals(1, countRowsInScan(getScanner("1", "", "2", "", predicate)));
+
+ // All the 2nd tablet.
+ predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
+ predicate.setLowerBound("1");
+ predicate.setUpperBound("3");
+ assertEquals(3, countRowsInScan(getScanner("1", "", "2", "", predicate)));
+
+ // Value that doesn't exist.
+ predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
+ predicate.setLowerBound("4");
+ assertEquals(0, countRowsInScan(getScanner("1", "", "2", "", predicate)));
+
+ // First row from every tablet.
+ predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
+ predicate.setLowerBound("1");
+ predicate.setUpperBound("1");
+ assertEquals(3, countRowsInScan(getScanner(null, null, null, null, predicate)));
+
+ // All the rows.
+ predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
+ predicate.setLowerBound("1");
+ assertEquals(9, countRowsInScan(getScanner(null, null, null, null, predicate)));
+ }
+
+ @Test(timeout = 100000)
+ public void testProjections() throws Exception {
+ // Test with column names.
+ AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
+ builder.setProjectedColumnNames(Lists.newArrayList(schema.getColumnByIndex(0).getName(),
+ schema.getColumnByIndex(1).getName()));
+ buildScannerAndCheckColumnsCount(builder, 2);
+
+ // Test with column indexes.
+ builder = client.newScannerBuilder(table);
+ builder.setProjectedColumnIndexes(Lists.newArrayList(0, 1));
+ buildScannerAndCheckColumnsCount(builder, 2);
+
+ // Test with column names overriding indexes.
+ builder = client.newScannerBuilder(table);
+ builder.setProjectedColumnIndexes(Lists.newArrayList(0, 1));
+ builder.setProjectedColumnNames(Lists.newArrayList(schema.getColumnByIndex(0).getName()));
+ buildScannerAndCheckColumnsCount(builder, 1);
+
+ // Test with keys last with indexes.
+ builder = client.newScannerBuilder(table);
+ builder.setProjectedColumnIndexes(Lists.newArrayList(2, 1, 0));
+ buildScannerAndCheckColumnsCount(builder, 3);
+
+ // Test with keys last with column names.
+ builder = client.newScannerBuilder(table);
+ builder.setProjectedColumnNames(Lists.newArrayList(schema.getColumnByIndex(2).getName(),
+ schema.getColumnByIndex(0).getName()));
+ buildScannerAndCheckColumnsCount(builder, 2);
+ }
+
+ private AsyncKuduScanner getScanner(String lowerBoundKeyOne,
+ String lowerBoundKeyTwo,
+ String exclusiveUpperBoundKeyOne,
+ String exclusiveUpperBoundKeyTwo) {
+ return getScanner(lowerBoundKeyOne, lowerBoundKeyTwo,
+ exclusiveUpperBoundKeyOne, exclusiveUpperBoundKeyTwo, null);
+ }
+
+ private AsyncKuduScanner getScanner(String lowerBoundKeyOne,
+ String lowerBoundKeyTwo,
+ String exclusiveUpperBoundKeyOne,
+ String exclusiveUpperBoundKeyTwo,
+ ColumnRangePredicate predicate) {
+ AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
+
+ if (lowerBoundKeyOne != null) {
+ PartialRow lowerBoundRow = schema.newPartialRow();
+ lowerBoundRow.addString(0, lowerBoundKeyOne);
+ lowerBoundRow.addString(1, lowerBoundKeyTwo);
+ builder.lowerBound(lowerBoundRow);
+ }
+
+ if (exclusiveUpperBoundKeyOne != null) {
+ PartialRow upperBoundRow = schema.newPartialRow();
+ upperBoundRow.addString(0, exclusiveUpperBoundKeyOne);
+ upperBoundRow.addString(1, exclusiveUpperBoundKeyTwo);
+ builder.exclusiveUpperBound(upperBoundRow);
+ }
+
+ if (predicate != null) {
+ builder.addColumnRangePredicate(predicate);
+ }
+
+ return builder.build();
+ }
+
+ private void buildScannerAndCheckColumnsCount(AsyncKuduScanner.AsyncKuduScannerBuilder builder,
+ int count) throws Exception {
+ AsyncKuduScanner scanner = builder.build();
+ scanner.nextRows().join(DEFAULT_SLEEP);
+ RowResultIterator rri = scanner.nextRows().join(DEFAULT_SLEEP);
+ assertEquals(count, rri.next().getSchema().getColumns().size());
+ }
+
+ private static Schema getSchema() {
+ ArrayList<ColumnSchema> columns = new ArrayList<>(3);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key1", STRING)
+ .key(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key2", STRING)
+ .key(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("val", STRING)
+ .nullable(true) // Important because we need to make sure it gets passed in projections
+ .build());
+ return new Schema(columns);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java
new file mode 100644
index 0000000..6cfbee7
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatistics.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kududb.client.Statistics.Statistic;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestStatistics extends BaseKuduTest {
+
+ private static final String TABLE_NAME = TestStatistics.class.getName() + "-"
+ + System.currentTimeMillis();
+ private static KuduTable table;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ CreateTableOptions options = getBasicCreateTableOptions().setNumReplicas(1);
+ table = createTable(TABLE_NAME, basicSchema, options);
+ }
+
+ @Test(timeout = 10000)
+ public void test() throws Exception {
+ KuduSession session = syncClient.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ int rowCount = 20;
+ for (int i = 0; i < rowCount; i++) {
+ Insert insert = createBasicSchemaInsert(table, i);
+ session.apply(insert);
+ if (i % 2 == 1) {
+ session.flush();
+ }
+ }
+ Statistics statistics = syncClient.getStatistics();
+ assertEquals(rowCount / 2, statistics.getClientStatistic(Statistic.WRITE_RPCS));
+ assertEquals(rowCount, statistics.getClientStatistic(Statistic.WRITE_OPS));
+ assertEquals(0, statistics.getClientStatistic(Statistic.RPC_ERRORS));
+ assertEquals(0, statistics.getClientStatistic(Statistic.OPS_ERRORS));
+
+ // Use default flush mode.
+ session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
+ // Insert duplicate rows, expect to get ALREADY_PRESENT error.
+ long byteSize = 0;
+ for (int i = 0; i < rowCount; i++) {
+ Insert insert = createBasicSchemaInsert(table, i);
+ session.apply(insert);
+ byteSize += insert.getRowOperationSizeBytes();
+ }
+ assertEquals(rowCount + rowCount / 2, statistics.getClientStatistic(Statistic.WRITE_RPCS));
+ assertEquals(rowCount, statistics.getClientStatistic(Statistic.WRITE_OPS));
+ assertEquals(0, statistics.getClientStatistic(Statistic.RPC_ERRORS));
+ assertEquals(rowCount, statistics.getClientStatistic(Statistic.OPS_ERRORS));
+ assertEquals(byteSize * 2, statistics.getClientStatistic(Statistic.BYTES_WRITTEN));
+
+ assertEquals(1, statistics.getTableSet().size());
+ assertEquals(1, statistics.getTabletSet().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java
new file mode 100644
index 0000000..11bd23b
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestStatus.java
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.Test;
+import org.kududb.client.Status;
+
+import static org.junit.Assert.*;
+
+public class TestStatus {
+
+ @Test
+ public void testOKStatus() {
+ Status s = Status.OK();
+ assertTrue(s.ok());
+ assertFalse(s.isNotAuthorized());
+ assertEquals(-1, s.getPosixCode());
+ assertEquals("OK", s.toString());
+ }
+
+ @Test
+ public void testStatusNonPosix() {
+ Status s = Status.Aborted("foo");
+ assertFalse(s.ok());
+ assertTrue(s.isAborted());
+ assertEquals("ABORTED", s.getCodeName());
+ assertEquals("foo", s.getMessage());
+ assertEquals(-1, s.getPosixCode());
+ assertEquals("Aborted: foo", s.toString());
+ }
+
+ @Test
+ public void testPosixCode() {
+ Status s = Status.NotFound("File not found", 2);
+ assertFalse(s.ok());
+ assertFalse(s.isAborted());
+ assertTrue(s.isNotFound());
+ assertEquals(2, s.getPosixCode());
+ assertEquals("Not found: File not found (error 2)", s.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java
new file mode 100644
index 0000000..b150f8e
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTestUtils.java
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.junit.After;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for non-trivial helper methods in TestUtils.
+ */
+public class TestTestUtils {
+
+ public static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
+
+ private Process proc;
+
+ @After
+ public void tearDown() {
+ if (proc != null) {
+ proc.destroy();
+ }
+ }
+
+ /**
+ * Starts a process that executes the "yes" command (which prints 'y' in a loop),
+ * sends a SIGSTOP to the process, and ensures that SIGSTOP does indeed pause the process.
+ * Afterwards, sends a SIGCONT to the process and ensures that the process resumes.
+ */
+ @Test(timeout = 2000)
+ public void testPauseAndResume() throws Exception {
+ ProcessBuilder processBuilder = new ProcessBuilder("yes");
+ proc = processBuilder.start();
+ LineCounterRunnable lineCounter = new LineCounterRunnable(proc.getInputStream());
+ Thread thread = new Thread(lineCounter);
+ thread.setDaemon(true);
+ thread.start();
+ TestUtils.pauseProcess(proc);
+ long prevCount;
+ do {
+ prevCount = lineCounter.getCount();
+ Thread.sleep(10);
+ } while (prevCount != lineCounter.getCount());
+ assertEquals(prevCount, lineCounter.getCount());
+ TestUtils.resumeProcess(proc);
+ do {
+ prevCount = lineCounter.getCount();
+ Thread.sleep(10);
+ } while (prevCount == lineCounter.getCount());
+ assertTrue(lineCounter.getCount() > prevCount);
+ }
+
+ /**
+ * Counts the number of lines in a specified input stream.
+ */
+ static class LineCounterRunnable implements Runnable {
+ private final AtomicLong counter;
+ private final InputStream is;
+
+ public LineCounterRunnable(InputStream is) {
+ this.is = is;
+ counter = new AtomicLong(0);
+ }
+
+ @Override
+ public void run() {
+ BufferedReader in = null;
+ try {
+ in = new BufferedReader(new InputStreamReader(is));
+ while (in.readLine() != null) {
+ counter.incrementAndGet();
+ }
+ } catch (Exception e) {
+ LOG.error("Error while reading from the process", e);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.error("Error closing the stream", e);
+ }
+ }
+ }
+ }
+
+ public long getCount() {
+ return counter.get();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java
new file mode 100644
index 0000000..3e78918
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestTimeouts.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.stumbleupon.async.TimeoutException;
+import org.junit.Test;
+
+public class TestTimeouts extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ TestTimeouts.class.getName() + "-" + System.currentTimeMillis();
+
+ /**
+ * This test case tries different methods that should all timeout, while relying on the client to
+ * pass down the timeouts to the session and scanner.
+ */
+ @Test(timeout = 100000)
+ public void testLowTimeouts() throws Exception {
+ KuduClient lowTimeoutsClient = new KuduClient.KuduClientBuilder(masterAddresses)
+ .defaultAdminOperationTimeoutMs(1)
+ .defaultOperationTimeoutMs(1)
+ .build();
+
+ try {
+ lowTimeoutsClient.listTabletServers();
+ fail("Should have timed out");
+ } catch (KuduException ex) {
+ // Expected.
+ }
+
+ createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions());
+ KuduTable table = openTable(TABLE_NAME);
+
+ KuduSession lowTimeoutSession = lowTimeoutsClient.newSession();
+
+ try {
+ lowTimeoutSession.apply(createBasicSchemaInsert(table, 1));
+ fail("Should have timed out");
+ } catch (KuduException ex) {
+ assertTrue(ex.getStatus().isTimedOut());
+ }
+
+ KuduScanner lowTimeoutScanner = lowTimeoutsClient.newScannerBuilder(table).build();
+ try {
+ lowTimeoutScanner.nextRows();
+ fail("Should have timed out");
+ } catch (KuduException ex) {
+ assertTrue(ex.getStatus().isTimedOut());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
new file mode 100644
index 0000000..392bed9
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestUtils.java
@@ -0,0 +1,289 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.sun.security.auth.module.UnixSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sun.management.VMManagement;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.SocketAddress;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A grouping of methods that help unit testing.
+ */
+public class TestUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
+
+ // Used by pidOfProcess()
+ private static String UNIX_PROCESS_CLS_NAME = "java.lang.UNIXProcess";
+ private static Set<String> VALID_SIGNALS = ImmutableSet.of("STOP", "CONT", "TERM", "KILL");
+
+ private static final String BIN_DIR_PROP = "binDir";
+
+ /**
+ * @return the path of the flags file to pass to daemon processes
+ * started by the tests
+ */
+ public static String getFlagsPath() {
+ URL u = BaseKuduTest.class.getResource("/flags");
+ if (u == null) {
+ throw new RuntimeException("Unable to find 'flags' file");
+ }
+ if (u.getProtocol() == "file") {
+ return urlToPath(u);
+ }
+ // If the flags are inside a JAR, extract them into our temporary
+ // test directory.
+ try {
+ // Somewhat unintuitively, createTempFile() actually creates the file,
+ // not just the path, so we have to use REPLACE_EXISTING below.
+ Path tmpFile = Files.createTempFile(
+ Paths.get(getBaseDir()), "kudu-flags", ".flags");
+ Files.copy(BaseKuduTest.class.getResourceAsStream("/flags"), tmpFile,
+ StandardCopyOption.REPLACE_EXISTING);
+ return tmpFile.toAbsolutePath().toString();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to extract flags file into tmp", e);
+ }
+ }
+
+ /**
+ * Return the path portion of a file URL, after decoding the escaped
+ * components. This fixes issues when trying to build within a
+ * working directory with special characters.
+ */
+ private static String urlToPath(URL u) {
+ try {
+ return URLDecoder.decode(u.getPath(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static String findBuildDir() {
+ URL myUrl = BaseKuduTest.class.getProtectionDomain().getCodeSource().getLocation();
+ File myPath = new File(urlToPath(myUrl));
+ while (myPath != null) {
+ if (new File(myPath, ".git").isDirectory()) {
+ return new File(myPath, "build/latest/bin").getAbsolutePath();
+ }
+ myPath = myPath.getParentFile();
+ }
+ LOG.warn("Unable to find build dir! myUrl={}", myUrl);
+ return null;
+ }
+
+ /**
+ * @param binName the binary to look for (eg 'kudu-tserver')
+ * @return the absolute path of that binary
+ * @throws FileNotFoundException if no such binary is found
+ */
+ public static String findBinary(String binName) throws FileNotFoundException {
+ String binDir = System.getProperty(BIN_DIR_PROP);
+ if (binDir != null) {
+ LOG.info("Using binary directory specified by property: {}",
+ binDir);
+ } else {
+ binDir = findBuildDir();
+ }
+
+ File candidate = new File(binDir, binName);
+ if (candidate.canExecute()) {
+ return candidate.getAbsolutePath();
+ }
+ throw new FileNotFoundException("Cannot find binary " + binName +
+ " in binary directory " + binDir);
+ }
+
+ /**
+ * @return the base directory within which we will store server data
+ */
+ public static String getBaseDir() {
+ String s = System.getenv("TEST_TMPDIR");
+ if (s == null) {
+ s = String.format("/tmp/kudutest-%d", new UnixSystem().getUid());
+ }
+ File f = new File(s);
+ f.mkdirs();
+ return f.getAbsolutePath();
+ }
+
+ /**
+ * Finds the next free port, starting with the one passed. Keep in mind the
+ * time-of-check-time-of-use nature of this method, the returned port might become occupied
+ * after it was checked for availability.
+ * @param startPort first port to be probed
+ * @return a currently usable port
+ * @throws IOException IOE is thrown if we can't close a socket we tried to open or if we run
+ * out of ports to try
+ */
+ public static int findFreePort(int startPort) throws IOException {
+ ServerSocket ss;
+ for(int i = startPort; i < 65536; i++) {
+ try {
+ ss = new ServerSocket();
+ SocketAddress address = new InetSocketAddress(getUniqueLocalhost(), i);
+ ss.bind(address);
+ } catch (IOException e) {
+ continue;
+ }
+ ss.close();
+ return i;
+ }
+ throw new IOException("Ran out of ports.");
+ }
+
+ /**
+ * Finds a specified number of parts, starting with one passed. Keep in mind the
+ * time-of-check-time-of-use nature of this method.
+ * @see {@link #findFreePort(int)}
+ * @param startPort First port to be probed.
+ * @param numPorts Number of ports to reserve.
+ * @return A list of currently usable ports.
+ * @throws IOException IOE Is thrown if we can't close a socket we tried to open or if run
+ * out of ports to try.
+ */
+ public static List<Integer> findFreePorts(int startPort, int numPorts) throws IOException {
+ List<Integer> ports = Lists.newArrayListWithCapacity(numPorts);
+ for (int i = 0; i < numPorts; i++) {
+ startPort = findFreePort(startPort);
+ ports.add(startPort++);
+ }
+ return ports;
+ }
+
+ /**
+ * Gets the pid of a specified process. Relies on reflection and only works on
+ * UNIX process, not guaranteed to work on JDKs other than Oracle and OpenJDK.
+ * @param proc The specified process.
+ * @return The process UNIX pid.
+ * @throws IllegalArgumentException If the process is not a UNIXProcess.
+ * @throws Exception If there are other getting the pid via reflection.
+ */
+ static int pidOfProcess(Process proc) throws Exception {
+ Class<?> procCls = proc.getClass();
+ if (!procCls.getName().equals(UNIX_PROCESS_CLS_NAME)) {
+ throw new IllegalArgumentException("stopProcess() expects objects of class " +
+ UNIX_PROCESS_CLS_NAME + ", but " + procCls.getName() + " was passed in instead!");
+ }
+ Field pidField = procCls.getDeclaredField("pid");
+ pidField.setAccessible(true);
+ return (Integer) pidField.get(proc);
+ }
+
+ /**
+ * Send a code specified by its string representation to the specified process.
+ * TODO: Use a JNR/JNR-Posix instead of forking the JVM to exec "kill".
+ * @param proc The specified process.
+ * @param sig The string representation of the process (e.g., STOP for SIGSTOP).
+ * @throws IllegalArgumentException If the signal type is not supported.
+ * @throws IllegalStateException If we are unable to send the specified signal.
+ */
+ static void signalProcess(Process proc, String sig) throws Exception {
+ if (!VALID_SIGNALS.contains(sig)) {
+ throw new IllegalArgumentException(sig + " is not a supported signal, only " +
+ Joiner.on(",").join(VALID_SIGNALS) + " are supported");
+ }
+ int pid = pidOfProcess(proc);
+ int rv = Runtime.getRuntime()
+ .exec(String.format("kill -%s %d", sig, pid))
+ .waitFor();
+ if (rv != 0) {
+ throw new IllegalStateException(String.format("unable to send SIG%s to process %s(pid=%d): " +
+ "expected return code from kill, but got %d instead", sig, proc, pid, rv));
+ }
+ }
+
+ /**
+ * Pause the specified process by sending a SIGSTOP using the kill command.
+ * @param proc The specified process.
+ * @throws Exception If error prevents us from pausing the process.
+ */
+ static void pauseProcess(Process proc) throws Exception {
+ signalProcess(proc, "STOP");
+ }
+
+ /**
+ * Resumes the specified process by sending a SIGCONT using the kill command.
+ * @param proc The specified process.
+ * @throws Exception If error prevents us from resuming the process.
+ */
+ static void resumeProcess(Process proc) throws Exception {
+ signalProcess(proc, "CONT");
+ }
+
+ /**
+ * This is used to generate unique loopback IPs for parallel test running.
+ * @return the local PID of this process
+ */
+ static int getPid() {
+ try {
+ RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
+ java.lang.reflect.Field jvm = runtime.getClass().getDeclaredField("jvm");
+ jvm.setAccessible(true);
+ VMManagement mgmt = (VMManagement)jvm.get(runtime);
+ Method pid_method = mgmt.getClass().getDeclaredMethod("getProcessId");
+ pid_method.setAccessible(true);
+
+ return (Integer)pid_method.invoke(mgmt);
+ } catch (Exception e) {
+ LOG.warn("Cannot get PID", e);
+ return 1;
+ }
+ }
+
+ /**
+ * The generated IP is based on pid, so this requires that the parallel tests
+ * run in separate VMs.
+ *
+ * On OSX, the above trick doesn't work, so we can't run parallel tests on OSX.
+ * Given that, we just return the normal localhost IP.
+ *
+ * @return a unique loopback IP address for this PID. This allows running
+ * tests in parallel, since 127.0.0.0/8 all act as loopbacks on Linux.
+ */
+ static String getUniqueLocalhost() {
+ if ("Mac OS X".equals(System.getProperty("os.name"))) {
+ return "127.0.0.1";
+ }
+
+ int pid = getPid();
+ return "127." + ((pid & 0xff00) >> 8) + "." + (pid & 0xff) + ".1";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java
new file mode 100644
index 0000000..fce7ddc
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestAsyncUtil.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.util;
+
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test for {@link AsyncUtil}.
+ */
+public class TestAsyncUtil {
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testAddCallbacksDeferring() throws Exception {
+ Deferred<String> d = new Deferred<String>();
+ TestCallback cb = new TestCallback();
+ TestErrback eb = new TestErrback();
+
+ // Test normal callbacks.
+ AsyncUtil.addCallbacksDeferring(d, cb, eb);
+ final String testStr = "hello world";
+ d.callback(testStr);
+ assertEquals(d.join(), "callback: " + testStr);
+
+ d = new Deferred<String>();
+ AsyncUtil.addCallbacksDeferring(d, cb, eb);
+ d.callback(new IllegalArgumentException());
+ assertEquals(d.join(), "illegal arg");
+
+ d = new Deferred<String>();
+ AsyncUtil.addCallbacksDeferring(d, cb, eb);
+ d.callback(new IllegalStateException());
+ exception.expect(IllegalStateException.class);
+ d.join();
+ }
+
+ final class TestCallback implements Callback<Deferred<String>, String> {
+ @Override
+ public Deferred<String> call(String arg) throws Exception {
+ return Deferred.fromResult("callback: " + arg);
+ }
+ }
+
+ final class TestErrback implements Callback<Deferred<String>, Exception> {
+ @Override
+ public Deferred<String> call(Exception arg) {
+ if (arg instanceof IllegalArgumentException) {
+ return Deferred.fromResult("illegal arg");
+ }
+ return Deferred.fromError(arg);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java
new file mode 100644
index 0000000..051107c
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestMurmurHash.java
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.util;
+
+import com.google.common.primitives.UnsignedLongs;
+import com.sangupta.murmur.Murmur2;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test Murmur2 Hash64 returns the expected values for inputs.
+ *
+ * These tests are duplicated on the C++ side to ensure that hash computations
+ * are stable across both platforms.
+ */
+public class TestMurmurHash {
+
+ @Test
+ public void testMurmur2Hash64() throws Exception {
+ long hash;
+
+ hash = Murmur2.hash64("ab".getBytes("UTF-8"), 2, 0);
+ assertEquals(UnsignedLongs.parseUnsignedLong("7115271465109541368"), hash);
+
+ hash = Murmur2.hash64("abcdefg".getBytes("UTF-8"), 7, 0);
+ assertEquals(UnsignedLongs.parseUnsignedLong("2601573339036254301"), hash);
+
+ hash = Murmur2.hash64("quick brown fox".getBytes("UTF-8"), 15, 42);
+ assertEquals(UnsignedLongs.parseUnsignedLong("3575930248840144026"), hash);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java b/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java
new file mode 100644
index 0000000..5c003ae
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/TestNetUtil.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.util;
+
+import com.google.common.net.HostAndPort;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test for {@link NetUtil}.
+ */
+public class TestNetUtil {
+
+ /**
+ * Tests parsing strings into {@link HostAndPort} objects with and without specifying
+ * the port in the string.
+ */
+ @Test
+ public void testParseString() {
+ String aStringWithPort = "1.2.3.4:1234";
+ HostAndPort hostAndPortForAStringWithPort = NetUtil.parseString(aStringWithPort, 0);
+ assertEquals(hostAndPortForAStringWithPort.getHostText(), "1.2.3.4");
+ assertEquals(hostAndPortForAStringWithPort.getPort(), 1234);
+
+ String aStringWithoutPort = "1.2.3.4";
+ HostAndPort hostAndPortForAStringWithoutPort = NetUtil.parseString(aStringWithoutPort, 12345);
+ assertEquals(hostAndPortForAStringWithoutPort.getHostText(), aStringWithoutPort);
+ assertEquals(hostAndPortForAStringWithoutPort.getPort(), 12345);
+ }
+
+ /**
+ * Tests parsing comma separated list of "host:port" pairs and hosts into a list of
+ * {@link HostAndPort} objects.
+ */
+ @Test
+ public void testParseStrings() {
+ String testAddrs = "1.2.3.4.5,10.0.0.1:5555,127.0.0.1:7777";
+ List<HostAndPort> hostsAndPorts = NetUtil.parseStrings(testAddrs, 3333);
+ assertArrayEquals(hostsAndPorts.toArray(),
+ new HostAndPort[] { HostAndPort.fromParts("1.2.3.4.5", 3333),
+ HostAndPort.fromParts("10.0.0.1", 5555),
+ HostAndPort.fromParts("127.0.0.1", 7777) }
+ );
+ }
+
+ @Test
+ public void testHostsAndPortsToString() {
+ List<HostAndPort> hostsAndPorts = Arrays.asList(
+ HostAndPort.fromParts("127.0.0.1", 1111),
+ HostAndPort.fromParts("1.2.3.4.5", 0)
+ );
+ assertEquals(NetUtil.hostsAndPortsToString(hostsAndPorts), "127.0.0.1:1111,1.2.3.4.5:0");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
deleted file mode 100644
index 1464fa4..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/BaseKuduTest.java
+++ /dev/null
@@ -1,438 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.net.HostAndPort;
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.master.Master;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.junit.Assert.fail;
-
-public class BaseKuduTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(BaseKuduTest.class);
-
- private static final String NUM_MASTERS_PROP = "NUM_MASTERS";
- private static final int NUM_TABLET_SERVERS = 3;
- private static final int DEFAULT_NUM_MASTERS = 1;
-
- // Number of masters that will be started for this test if we're starting
- // a cluster.
- private static final int NUM_MASTERS =
- Integer.getInteger(NUM_MASTERS_PROP, DEFAULT_NUM_MASTERS);
-
- protected static final int DEFAULT_SLEEP = 50000;
-
- // Currently not specifying a seed since we want a random behavior when running tests that
- // restart tablet servers. Would be nice to have the same kind of facility that C++ has that dumps
- // the seed it picks so that you can re-run tests with it.
- private static final Random randomForTSRestart = new Random();
-
- private static MiniKuduCluster miniCluster;
-
- // Comma separate describing the master addresses and ports.
- protected static String masterAddresses;
- protected static List<HostAndPort> masterHostPorts;
-
- // We create both versions of the client for ease of use.
- protected static AsyncKuduClient client;
- protected static KuduClient syncClient;
- protected static final Schema basicSchema = getBasicSchema();
- protected static final Schema allTypesSchema = getSchemaWithAllTypes();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- LOG.info("Setting up before class...");
-
- miniCluster = new MiniKuduCluster.MiniKuduClusterBuilder()
- .numMasters(NUM_MASTERS)
- .numTservers(NUM_TABLET_SERVERS)
- .defaultTimeoutMs(DEFAULT_SLEEP)
- .build();
- masterAddresses = miniCluster.getMasterAddresses();
- masterHostPorts = miniCluster.getMasterHostPorts();
-
- LOG.info("Creating new Kudu client...");
- client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses)
- .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
- .build();
- syncClient = new KuduClient(client);
- LOG.info("Waiting for tablet servers...");
- if (!miniCluster.waitForTabletServers(NUM_TABLET_SERVERS)) {
- fail("Couldn't get " + NUM_TABLET_SERVERS + " tablet servers running, aborting");
- }
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- try {
- if (client != null) {
- Deferred<ArrayList<Void>> d = client.shutdown();
- d.addErrback(defaultErrorCB);
- d.join(DEFAULT_SLEEP);
- // No need to explicitly shutdown the sync client,
- // shutting down the async client effectively does that.
- }
- } finally {
- if (miniCluster != null) {
- miniCluster.shutdown();
- }
- }
- }
-
- protected static KuduTable createTable(String tableName, Schema schema,
- CreateTableOptions builder) throws Exception {
- LOG.info("Creating table: {}", tableName);
- return client.syncClient().createTable(tableName, schema, builder);
- }
-
- /**
- * Counts the rows from the {@code scanner} until exhaustion. It doesn't require the scanner to
- * be new, so it can be used to finish scanning a previously-started scan.
- */
- protected static int countRowsInScan(AsyncKuduScanner scanner)
- throws Exception {
- final AtomicInteger counter = new AtomicInteger();
-
- Callback<Object, RowResultIterator> cb = new Callback<Object, RowResultIterator>() {
- @Override
- public Object call(RowResultIterator arg) throws Exception {
- if (arg == null) return null;
- counter.addAndGet(arg.getNumRows());
- return null;
- }
- };
-
- while (scanner.hasMoreRows()) {
- Deferred<RowResultIterator> data = scanner.nextRows();
- data.addCallbacks(cb, defaultErrorCB);
- data.join(DEFAULT_SLEEP);
- }
-
- Deferred<RowResultIterator> closer = scanner.close();
- closer.addCallbacks(cb, defaultErrorCB);
- closer.join(DEFAULT_SLEEP);
- return counter.get();
- }
-
- protected List<String> scanTableToStrings(KuduTable table,
- KuduPredicate... predicates) throws Exception {
- List<String> rowStrings = Lists.newArrayList();
- KuduScanner.KuduScannerBuilder scanBuilder = syncClient.newScannerBuilder(table);
- for (KuduPredicate predicate : predicates) {
- scanBuilder.addPredicate(predicate);
- }
- KuduScanner scanner = scanBuilder.build();
- while (scanner.hasMoreRows()) {
- RowResultIterator rows = scanner.nextRows();
- for (RowResult r : rows) {
- rowStrings.add(r.rowToString());
- }
- }
- Collections.sort(rowStrings);
- return rowStrings;
- }
-
- private static final int[] KEYS = new int[] {10, 20, 30};
- protected static KuduTable createFourTabletsTableWithNineRows(String tableName) throws
- Exception {
- CreateTableOptions builder = getBasicCreateTableOptions();
- for (int i : KEYS) {
- PartialRow splitRow = basicSchema.newPartialRow();
- splitRow.addInt(0, i);
- builder.addSplitRow(splitRow);
- }
- KuduTable table = createTable(tableName, basicSchema, builder);
- AsyncKuduSession session = client.newSession();
-
- // create a table with on empty tablet and 3 tablets of 3 rows each
- for (int key1 : KEYS) {
- for (int key2 = 1; key2 <= 3; key2++) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addInt(0, key1 + key2);
- row.addInt(1, key1);
- row.addInt(2, key2);
- row.addString(3, "a string");
- row.addBoolean(4, true);
- session.apply(insert).join(DEFAULT_SLEEP);
- }
- }
- session.close().join(DEFAULT_SLEEP);
- return table;
- }
-
- public static Schema getSchemaWithAllTypes() {
- List<ColumnSchema> columns =
- ImmutableList.of(
- new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).key(true).build(),
- new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build(),
- new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build(),
- new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build(),
- new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build(),
- new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build(),
- new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build(),
- new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).build(),
- new ColumnSchema.ColumnSchemaBuilder("binary-array", Type.BINARY).build(),
- new ColumnSchema.ColumnSchemaBuilder("binary-bytebuffer", Type.BINARY).build(),
- new ColumnSchema.ColumnSchemaBuilder("null", Type.STRING).nullable(true).build(),
- new ColumnSchema.ColumnSchemaBuilder("timestamp", Type.TIMESTAMP).build());
-
- return new Schema(columns);
- }
-
- public static CreateTableOptions getAllTypesCreateTableOptions() {
- return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("int8"));
- }
-
- public static Schema getBasicSchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(5);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column2_i", Type.INT32).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column3_s", Type.STRING)
- .nullable(true)
- .desiredBlockSize(4096)
- .encoding(ColumnSchema.Encoding.DICT_ENCODING)
- .compressionAlgorithm(ColumnSchema.CompressionAlgorithm.LZ4)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("column4_b", Type.BOOL).build());
- return new Schema(columns);
- }
-
- public static CreateTableOptions getBasicCreateTableOptions() {
- return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
- }
-
- /**
- * Creates table options with non-covering range partitioning for a table with
- * the basic schema. Range partition key ranges fall between the following values:
- *
- * [ 0, 50)
- * [ 50, 100)
- * [200, 300)
- */
- public static CreateTableOptions getBasicTableOptionsWithNonCoveredRange() {
- Schema schema = basicSchema;
- CreateTableOptions option = new CreateTableOptions();
- option.setRangePartitionColumns(ImmutableList.of("key"));
-
- PartialRow aLowerBound = schema.newPartialRow();
- aLowerBound.addInt("key", 0);
- PartialRow aUpperBound = schema.newPartialRow();
- aUpperBound.addInt("key", 100);
- option.addRangeBound(aLowerBound, aUpperBound);
-
- PartialRow bLowerBound = schema.newPartialRow();
- bLowerBound.addInt("key", 200);
- PartialRow bUpperBound = schema.newPartialRow();
- bUpperBound.addInt("key", 300);
- option.addRangeBound(bLowerBound, bUpperBound);
-
- PartialRow split = schema.newPartialRow();
- split.addInt("key", 50);
- option.addSplitRow(split);
- return option;
- }
-
- protected Insert createBasicSchemaInsert(KuduTable table, int key) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addInt(0, key);
- row.addInt(1, 2);
- row.addInt(2, 3);
- row.addString(3, "a string");
- row.addBoolean(4, true);
- return insert;
- }
-
- static Callback<Object, Object> defaultErrorCB = new Callback<Object, Object>() {
- @Override
- public Object call(Object arg) throws Exception {
- if (arg == null) return null;
- if (arg instanceof Exception) {
- LOG.warn("Got exception", (Exception) arg);
- } else {
- LOG.warn("Got an error response back " + arg);
- }
- return new Exception("Can't recover from error, see previous WARN");
- }
- };
-
- /**
- * Helper method to open a table. It sets the default sleep time when joining on the Deferred.
- * @param name Name of the table
- * @return A KuduTable
- * @throws Exception MasterErrorException if the table doesn't exist
- */
- protected static KuduTable openTable(String name) throws Exception {
- Deferred<KuduTable> d = client.openTable(name);
- return d.join(DEFAULT_SLEEP);
- }
-
- /**
- * Helper method to easily kill a tablet server that serves the given table's only tablet's
- * leader. The currently running test case will be failed if there's more than one tablet,
- * if the tablet has no leader after some retries, or if the tablet server was already killed.
- *
- * This method is thread-safe.
- * @param table a KuduTable which will get its single tablet's leader killed.
- * @throws Exception
- */
- protected static void killTabletLeader(KuduTable table) throws Exception {
- LocatedTablet.Replica leader = null;
- DeadlineTracker deadlineTracker = new DeadlineTracker();
- deadlineTracker.setDeadline(DEFAULT_SLEEP);
- while (leader == null) {
- if (deadlineTracker.timedOut()) {
- fail("Timed out while trying to find a leader for this table: " + table.getName());
- }
- List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
- if (tablets.isEmpty() || tablets.size() > 1) {
- fail("Currently only support killing leaders for tables containing 1 tablet, table " +
- table.getName() + " has " + tablets.size());
- }
- LocatedTablet tablet = tablets.get(0);
- if (tablet.getReplicas().size() == 1) {
- fail("Table " + table.getName() + " only has 1 tablet, please enable replication");
- }
- leader = tablet.getLeaderReplica();
- if (leader == null) {
- LOG.info("Sleeping while waiting for a tablet LEADER to arise, currently slept " +
- deadlineTracker.getElapsedMillis() + "ms");
- Thread.sleep(50);
- }
- }
-
- Integer port = leader.getRpcPort();
- miniCluster.killTabletServerOnPort(port);
- }
-
- /**
- * Helper method to easily kill the leader master.
- *
- * This method is thread-safe.
- * @throws Exception If there is an error finding or killing the leader master.
- */
- protected static void killMasterLeader() throws Exception {
- int leaderPort = findLeaderMasterPort();
- miniCluster.killMasterOnPort(leaderPort);
- }
-
- /**
- * Find the port of the leader master in order to retrieve it from the port to process map.
- * @return The port of the leader master.
- * @throws Exception If we are unable to find the leader master.
- */
- protected static int findLeaderMasterPort() throws Exception {
- Stopwatch sw = Stopwatch.createStarted();
- int leaderPort = -1;
- while (leaderPort == -1 && sw.elapsed(TimeUnit.MILLISECONDS) < DEFAULT_SLEEP) {
- Deferred<Master.GetTableLocationsResponsePB> masterLocD = client.getMasterTableLocationsPB();
- Master.GetTableLocationsResponsePB r = masterLocD.join(DEFAULT_SLEEP);
- leaderPort = r.getTabletLocations(0)
- .getReplicas(0)
- .getTsInfo()
- .getRpcAddresses(0)
- .getPort();
- }
- if (leaderPort == -1) {
- fail("No leader master found after " + DEFAULT_SLEEP + " ms.");
- }
- return leaderPort;
- }
-
- /**
- * Picks at random a tablet server that serves tablets from the passed table and restarts it.
- * Waits between killing and restarting the process.
- * @param table table to query for a TS to restart
- * @throws Exception
- */
- protected static void restartTabletServer(KuduTable table) throws Exception {
- List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
- if (tablets.isEmpty()) {
- fail("Table " + table.getName() + " doesn't have any tablets");
- }
-
- LocatedTablet tablet = tablets.get(0);
-
- int port = tablet.getReplicas().get(
- randomForTSRestart.nextInt(tablet.getReplicas().size())).getRpcPort();
-
- miniCluster.killTabletServerOnPort(port);
-
- Thread.sleep(1000);
-
- miniCluster.restartDeadTabletServerOnPort(port);
- }
-
- /**
- * Kills, sleeps, then restarts the leader master.
- * @throws Exception
- */
- protected static void restartLeaderMaster() throws Exception {
- int master = findLeaderMasterPort();
- miniCluster.killMasterOnPort(master);
-
- Thread.sleep(1000);
-
- miniCluster.restartDeadMasterOnPort(master);
- }
-
- /**
- * Return the comma-separated list of "host:port" pairs that describes the master
- * config for this cluster.
- * @return The master config string.
- */
- protected static String getMasterAddresses() {
- return masterAddresses;
- }
-
- /**
- * Kills all tablet servers in the cluster.
- * @throws InterruptedException
- */
- protected void killTabletServers() throws InterruptedException {
- miniCluster.killTabletServers();
- }
-
- /**
- * Restarts killed tablet servers in the cluster.
- * @throws Exception
- */
- protected void restartTabletServers() throws Exception {
- miniCluster.restartDeadTabletServers();
- }
-}
[06/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestKuduSession.java b/java/kudu-client/src/test/java/org/kududb/client/TestKuduSession.java
deleted file mode 100644
index df2367f..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestKuduSession.java
+++ /dev/null
@@ -1,337 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import java.util.List;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-import static org.junit.Assert.*;
-
-import com.google.common.collect.ImmutableList;
-
-public class TestKuduSession extends BaseKuduTest {
- @Rule
- public TestName name = new TestName();
-
- private KuduTable table;
-
- @Test(timeout = 100000)
- public void testBasicOps() throws Exception {
- String tableName = name.getMethodName();
- table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
-
- KuduSession session = syncClient.newSession();
- for (int i = 0; i < 10; i++) {
- session.apply(createInsert(i));
- }
- assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build()));
-
- OperationResponse resp = session.apply(createInsert(0));
- assertTrue(resp.hasRowError());
-
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
-
- for (int i = 10; i < 20; i++) {
- session.apply(createInsert(i));
- }
- session.flush();
- assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build()));
- }
-
- @Test(timeout = 100000)
- public void testIgnoreAllDuplicateRows() throws Exception {
- String tableName = name.getMethodName();
- table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
-
- KuduSession session = syncClient.newSession();
- session.setIgnoreAllDuplicateRows(true);
- for (int i = 0; i < 10; i++) {
- session.apply(createInsert(i));
- }
- for (SessionConfiguration.FlushMode mode : SessionConfiguration.FlushMode.values()) {
- session.setFlushMode(mode);
- for (int i = 0; i < 10; i++) {
- OperationResponse resp = session.apply(createInsert(i));
- if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC) {
- assertFalse(resp.hasRowError());
- }
- }
- if (mode == SessionConfiguration.FlushMode.MANUAL_FLUSH) {
- List<OperationResponse> responses = session.flush();
- for (OperationResponse resp : responses) {
- assertFalse(resp.hasRowError());
- }
- } else if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) {
- while (session.hasPendingOperations()) {
- Thread.sleep(100);
- }
- assertEquals(0, session.countPendingErrors());
- }
- }
- }
-
- @Test(timeout = 100000)
- public void testBatchWithSameRow() throws Exception {
- String tableName = name.getMethodName();
- table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
-
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
-
- // Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end,
- // while also clearing the cache between each batch half the time. The delete is added here
- // so that a misplaced update would fail if it happens later than its delete.
- for (int i = 0; i < 25; i++) {
- session.apply(createInsert(i));
- for (int j = 0; j < 50; j++) {
- Update update = table.newUpdate();
- PartialRow row = update.getRow();
- row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
- row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000);
- session.apply(update);
- }
- Delete del = table.newDelete();
- PartialRow row = del.getRow();
- row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
- session.apply(del);
- session.flush();
- if (i % 2 == 0) {
- client.emptyTabletsCacheForTable(table.getTableId());
- }
- }
- assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
- }
-
- /**
- * Regression test for KUDU-1226. Calls to session.flush() concurrent with AUTO_FLUSH_BACKGROUND
- * can end up giving ConvertBatchToListOfResponsesCB a list with nulls if a tablet was already
- * flushed. Only happens with multiple tablets.
- */
- @Test(timeout = 10000)
- public void testConcurrentFlushes() throws Exception {
- String tableName = name.getMethodName();
- CreateTableOptions builder = getBasicCreateTableOptions();
- int numTablets = 4;
- int numRowsPerTablet = 100;
-
- // Create a 4 tablets table split on 1000, 2000, and 3000.
- for (int i = 1; i < numTablets; i++) {
- PartialRow split = basicSchema.newPartialRow();
- split.addInt(0, i * numRowsPerTablet);
- builder.addSplitRow(split);
- }
- table = createTable(tableName, basicSchema, builder);
-
- // Configure the session to background flush as often as it can (every 1ms).
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
- session.setFlushInterval(1);
-
- // Fill each tablet in parallel 1 by 1 then flush. Without the fix this would quickly get an
- // NPE.
- for (int i = 0; i < numRowsPerTablet; i++) {
- for (int j = 0; j < numTablets; j++) {
- session.apply(createInsert(i + (numRowsPerTablet * j)));
- }
- session.flush();
- }
- }
-
- @Test(timeout = 10000)
- public void testOverWritingValues() throws Exception {
- String tableName = name.getMethodName();
- table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
- KuduSession session = syncClient.newSession();
- Insert insert = createInsert(0);
- PartialRow row = insert.getRow();
-
- // Overwrite all the normal columns.
- int magicNumber = 9999;
- row.addInt(1, magicNumber);
- row.addInt(2, magicNumber);
- row.addBoolean(4, false);
- // Spam the string column since it's backed by an array.
- for (int i = 0; i <= magicNumber; i++) {
- row.addString(3, i + "");
- }
- // We're supposed to keep a constant size.
- assertEquals(5, row.getVarLengthData().size());
- session.apply(insert);
-
- KuduScanner scanner = syncClient.newScannerBuilder(table).build();
- RowResult rr = scanner.nextRows().next();
- assertEquals(magicNumber, rr.getInt(1));
- assertEquals(magicNumber, rr.getInt(2));
- assertEquals(magicNumber + "", rr.getString(3));
- assertEquals(false, rr.getBoolean(4));
-
- // Test setting a value post-apply.
- try {
- row.addInt(1, 0);
- fail("Row should be frozen and throw");
- } catch (IllegalStateException ex) {
- // Ok.
- }
- }
-
- @Test(timeout = 10000)
- public void testUpsert() throws Exception {
- String tableName = name.getMethodName();
- table = createTable(tableName, basicSchema, getBasicCreateTableOptions());
- KuduSession session = syncClient.newSession();
-
- // Test an Upsert that acts as an Insert.
- assertFalse(session.apply(createUpsert(1, 1, false)).hasRowError());
-
- List<String> rowStrings = scanTableToStrings(table);
- assertEquals(1, rowStrings.size());
- assertEquals(
- "INT32 key=1, INT32 column1_i=1, INT32 column2_i=3, " +
- "STRING column3_s=a string, BOOL column4_b=true",
- rowStrings.get(0));
-
- // Test an Upsert that acts as an Update.
- assertFalse(session.apply(createUpsert(1, 2, false)).hasRowError());
- rowStrings = scanTableToStrings(table);
- assertEquals(
- "INT32 key=1, INT32 column1_i=2, INT32 column2_i=3, " +
- "STRING column3_s=a string, BOOL column4_b=true",
- rowStrings.get(0));
- }
-
- @Test(timeout = 10000)
- public void testInsertManualFlushNonCoveredRange() throws Exception {
- String tableName = name.getMethodName();
- CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange();
- createOptions.setNumReplicas(1);
- syncClient.createTable(tableName, basicSchema, createOptions);
- KuduTable table = syncClient.openTable(tableName);
-
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
-
- // Insert in reverse sorted order so that more table location lookups occur
- // (the extra results in table location lookups always occur past the inserted key).
- List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50);
- for (int key : nonCoveredKeys) {
- assertNull(session.apply(createBasicSchemaInsert(table, key)));
- }
- List<OperationResponse> results = session.flush();
- assertEquals(nonCoveredKeys.size(), results.size());
- for (OperationResponse result : results) {
- assertTrue(result.hasRowError());
- assertTrue(result.getRowError().getErrorStatus().isNotFound());
- }
-
- // Insert a batch of some valid and some invalid.
- for (int key = 90; key < 110; key++) {
- session.apply(createBasicSchemaInsert(table, key));
- }
- results = session.flush();
-
- int failures = 0;
- for (OperationResponse result : results) {
- if (result.hasRowError()) {
- failures++;
- assertTrue(result.getRowError().getErrorStatus().isNotFound());
- }
- }
- assertEquals(10, failures);
- }
-
- @Test(timeout = 10000)
- public void testInsertAutoFlushSyncNonCoveredRange() throws Exception {
- String tableName = name.getMethodName();
- CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange();
- createOptions.setNumReplicas(1);
- syncClient.createTable(tableName, basicSchema, createOptions);
- KuduTable table = syncClient.openTable(tableName);
-
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
-
- List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50);
- for (int key : nonCoveredKeys) {
- try {
- session.apply(createBasicSchemaInsert(table, key));
- fail("apply should have thrown");
- } catch (KuduException e) {
- assertTrue(e.getStatus().isNotFound());
- }
- }
- }
-
- @Test(timeout = 10000)
- public void testInsertAutoFlushBackgrounNonCoveredRange() throws Exception {
- String tableName = name.getMethodName();
- CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange();
- createOptions.setNumReplicas(1);
- syncClient.createTable(tableName, basicSchema, createOptions);
- KuduTable table = syncClient.openTable(tableName);
-
- AsyncKuduSession session = client.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
-
- List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50);
- for (int key : nonCoveredKeys) {
- OperationResponse result = session.apply(createBasicSchemaInsert(table, key)).join(5000);
- assertTrue(result.hasRowError());
- assertTrue(result.getRowError().getErrorStatus().isNotFound());
- }
-
- RowErrorsAndOverflowStatus errors = session.getPendingErrors();
- assertEquals(nonCoveredKeys.size(), errors.getRowErrors().length);
- for (RowError error : errors.getRowErrors()) {
- assertTrue(error.getErrorStatus().isNotFound());
- }
-
- // Insert a batch of some valid and some invalid.
- for (int key = 90; key < 110; key++) {
- session.apply(createBasicSchemaInsert(table, key));
- }
- session.flush();
-
- errors = session.getPendingErrors();
- assertEquals(10, errors.getRowErrors().length);
- for (RowError error : errors.getRowErrors()) {
- assertTrue(error.getErrorStatus().isNotFound());
- }
- }
-
- private Insert createInsert(int key) {
- return createBasicSchemaInsert(table, key);
- }
-
- private Upsert createUpsert(int key, int secondVal, boolean hasNull) {
- Upsert upsert = table.newUpsert();
- PartialRow row = upsert.getRow();
- row.addInt(0, key);
- row.addInt(1, secondVal);
- row.addInt(2, 3);
- if (hasNull) {
- row.setNull(3);
- } else {
- row.addString(3, "a string");
- }
- row.addBoolean(4, true);
- return upsert;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java b/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
deleted file mode 100644
index 4e41a29..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
+++ /dev/null
@@ -1,301 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.Rule;
-import org.junit.rules.TestName;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-public class TestKuduTable extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(TestKuduTable.class);
-
- @Rule
- public TestName name = new TestName();
-
- private static Schema schema = getBasicSchema();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- }
-
- @Test(timeout = 100000)
- public void testAlterTable() throws Exception {
- String tableName = name.getMethodName();
- createTable(tableName, basicSchema, getBasicCreateTableOptions());
- try {
-
- // Add a col.
- AlterTableOptions ato = new AlterTableOptions().addColumn("testaddint", Type.INT32, 4);
- submitAlterAndCheck(ato, tableName);
-
- // Rename that col.
- ato = new AlterTableOptions().renameColumn("testaddint", "newtestaddint");
- submitAlterAndCheck(ato, tableName);
-
- // Delete it.
- ato = new AlterTableOptions().dropColumn("newtestaddint");
- submitAlterAndCheck(ato, tableName);
-
- String newTableName = tableName +"new";
-
- // Rename our table.
- ato = new AlterTableOptions().renameTable(newTableName);
- submitAlterAndCheck(ato, tableName, newTableName);
-
- // Rename it back.
- ato = new AlterTableOptions().renameTable(tableName);
- submitAlterAndCheck(ato, newTableName, tableName);
-
- // Try adding two columns, where one is nullable.
- ato = new AlterTableOptions()
- .addColumn("testaddmulticolnotnull", Type.INT32, 4)
- .addNullableColumn("testaddmulticolnull", Type.STRING);
- submitAlterAndCheck(ato, tableName);
-
-
- // Try altering a table that doesn't exist.
- String nonExistingTableName = "table_does_not_exist";
- try {
- syncClient.alterTable(nonExistingTableName, ato);
- fail("Shouldn't be able to alter a table that doesn't exist");
- } catch (KuduException ex) {
- assertTrue(ex.getStatus().isNotFound());
- }
-
- try {
- syncClient.isAlterTableDone(nonExistingTableName);
- fail("Shouldn't be able to query if an alter table is done here");
- } catch (KuduException ex) {
- assertTrue(ex.getStatus().isNotFound());
- }
- } finally {
- // Normally Java tests accumulate tables without issue, deleting them all
- // when shutting down the mini cluster at the end of every test class.
- // However, testGetLocations below expects a certain table count, so
- // we'll delete our table to ensure there's no interaction between them.
- syncClient.deleteTable(tableName);
- }
- }
-
- /**
- * Helper method to submit an Alter and wait for it to happen, using the default table name to
- * check.
- */
- private void submitAlterAndCheck(AlterTableOptions ato, String tableToAlter)
- throws Exception {
- submitAlterAndCheck(ato, tableToAlter, tableToAlter);
- }
-
- private void submitAlterAndCheck(AlterTableOptions ato,
- String tableToAlter, String tableToCheck) throws
- Exception {
- if (masterHostPorts.size() > 1) {
- LOG.info("Alter table is not yet supported with multiple masters. Specify " +
- "-DnumMasters=1 on the command line to start a single-master cluster to run this test.");
- return;
- }
- AlterTableResponse alterResponse = syncClient.alterTable(tableToAlter, ato);
- boolean done = syncClient.isAlterTableDone(tableToCheck);
- assertTrue(done);
- }
-
- /**
- * Test creating tables of different sizes and see that we get the correct number of tablets back
- * @throws Exception
- */
- @Test
- public void testGetLocations() throws Exception {
- String table1 = name.getMethodName() + System.currentTimeMillis();
-
- // Test a non-existing table
- try {
- openTable(table1);
- fail("Should receive an exception since the table doesn't exist");
- } catch (Exception ex) {
- // expected
- }
- // Test with defaults
- String tableWithDefault = name.getMethodName() + "WithDefault" + System.currentTimeMillis();
- CreateTableOptions builder = getBasicCreateTableOptions();
- List<ColumnSchema> columns = new ArrayList<ColumnSchema>(schema.getColumnCount());
- int defaultInt = 30;
- String defaultString = "data";
- for (ColumnSchema columnSchema : schema.getColumns()) {
-
- Object defaultValue;
-
- if (columnSchema.getType() == Type.INT32) {
- defaultValue = defaultInt;
- } else if (columnSchema.getType() == Type.BOOL) {
- defaultValue = true;
- } else {
- defaultValue = defaultString;
- }
- columns.add(
- new ColumnSchema.ColumnSchemaBuilder(columnSchema.getName(), columnSchema.getType())
- .key(columnSchema.isKey())
- .nullable(columnSchema.isNullable())
- .defaultValue(defaultValue).build());
- }
- Schema schemaWithDefault = new Schema(columns);
- KuduTable kuduTable = createTable(tableWithDefault, schemaWithDefault, builder);
- assertEquals(defaultInt, kuduTable.getSchema().getColumnByIndex(0).getDefaultValue());
- assertEquals(defaultString,
- kuduTable.getSchema().getColumnByIndex(columns.size() - 2).getDefaultValue());
- assertEquals(true,
- kuduTable.getSchema().getColumnByIndex(columns.size() - 1).getDefaultValue());
-
- // Make sure the table's schema includes column IDs.
- assertTrue(kuduTable.getSchema().hasColumnIds());
-
- // Test we can open a table that was already created.
- openTable(tableWithDefault);
-
- // Test splitting and reading those splits
- KuduTable kuduTableWithoutDefaults = createTableWithSplitsAndTest(0);
- // finish testing read defaults
- assertNull(kuduTableWithoutDefaults.getSchema().getColumnByIndex(0).getDefaultValue());
- createTableWithSplitsAndTest(3);
- createTableWithSplitsAndTest(10);
-
- KuduTable table = createTableWithSplitsAndTest(30);
-
- List<LocatedTablet>tablets = table.getTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP);
- assertEquals(9, tablets.size());
- assertEquals(9, table.asyncGetTabletsLocations(null, getKeyInBytes(9), DEFAULT_SLEEP).join().size());
-
- tablets = table.getTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP);
- assertEquals(9, tablets.size());
- assertEquals(9, table.asyncGetTabletsLocations(getKeyInBytes(0), getKeyInBytes(9), DEFAULT_SLEEP).join().size());
-
- tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP);
- assertEquals(4, tablets.size());
- assertEquals(4, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(9), DEFAULT_SLEEP).join().size());
-
- tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP);
- assertEquals(9, tablets.size());
- assertEquals(9, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(14), DEFAULT_SLEEP).join().size());
-
- tablets = table.getTabletsLocations(getKeyInBytes(5), getKeyInBytes(31), DEFAULT_SLEEP);
- assertEquals(26, tablets.size());
- assertEquals(26, table.asyncGetTabletsLocations(getKeyInBytes(5), getKeyInBytes(31), DEFAULT_SLEEP).join().size());
-
- tablets = table.getTabletsLocations(getKeyInBytes(5), null, DEFAULT_SLEEP);
- assertEquals(26, tablets.size());
- assertEquals(26, table.asyncGetTabletsLocations(getKeyInBytes(5), null, DEFAULT_SLEEP).join().size());
-
- tablets = table.getTabletsLocations(null, getKeyInBytes(10000), DEFAULT_SLEEP);
- assertEquals(31, tablets.size());
- assertEquals(31, table.asyncGetTabletsLocations(null, getKeyInBytes(10000), DEFAULT_SLEEP).join().size());
-
- tablets = table.getTabletsLocations(getKeyInBytes(20), getKeyInBytes(10000), DEFAULT_SLEEP);
- assertEquals(11, tablets.size());
- assertEquals(11, table.asyncGetTabletsLocations(getKeyInBytes(20), getKeyInBytes(10000), DEFAULT_SLEEP).join().size());
-
- // Test listing tables.
- assertEquals(0, client.getTablesList(table1).join(DEFAULT_SLEEP).getTablesList().size());
- assertEquals(1, client.getTablesList(tableWithDefault)
- .join(DEFAULT_SLEEP).getTablesList().size());
- assertEquals(5, client.getTablesList().join(DEFAULT_SLEEP).getTablesList().size());
- assertFalse(client.getTablesList(tableWithDefault).
- join(DEFAULT_SLEEP).getTablesList().isEmpty());
-
- assertFalse(client.tableExists(table1).join(DEFAULT_SLEEP));
- assertTrue(client.tableExists(tableWithDefault).join(DEFAULT_SLEEP));
- }
-
- @Test(timeout = 100000)
- public void testLocateTableNonCoveringRange() throws Exception {
- String tableName = name.getMethodName();
- syncClient.createTable(tableName, basicSchema, getBasicTableOptionsWithNonCoveredRange());
- KuduTable table = syncClient.openTable(tableName);
-
- List<LocatedTablet> tablets;
-
- // all tablets
- tablets = table.getTabletsLocations(null, null, 100000);
- assertEquals(3, tablets.size());
- assertArrayEquals(getKeyInBytes(0), tablets.get(0).getPartition().getPartitionKeyStart());
- assertArrayEquals(getKeyInBytes(50), tablets.get(0).getPartition().getPartitionKeyEnd());
- assertArrayEquals(getKeyInBytes(50), tablets.get(1).getPartition().getPartitionKeyStart());
- assertArrayEquals(getKeyInBytes(100), tablets.get(1).getPartition().getPartitionKeyEnd());
- assertArrayEquals(getKeyInBytes(200), tablets.get(2).getPartition().getPartitionKeyStart());
- assertArrayEquals(getKeyInBytes(300), tablets.get(2).getPartition().getPartitionKeyEnd());
-
- // key < 50
- tablets = table.getTabletsLocations(null, getKeyInBytes(50), 100000);
- assertEquals(1, tablets.size());
- assertArrayEquals(getKeyInBytes(0), tablets.get(0).getPartition().getPartitionKeyStart());
- assertArrayEquals(getKeyInBytes(50), tablets.get(0).getPartition().getPartitionKeyEnd());
-
- // key >= 300
- tablets = table.getTabletsLocations(getKeyInBytes(300), null, 100000);
- assertEquals(0, tablets.size());
-
- // key >= 299
- tablets = table.getTabletsLocations(getKeyInBytes(299), null, 100000);
- assertEquals(1, tablets.size());
- assertArrayEquals(getKeyInBytes(200), tablets.get(0).getPartition().getPartitionKeyStart());
- assertArrayEquals(getKeyInBytes(300), tablets.get(0).getPartition().getPartitionKeyEnd());
-
- // key >= 150 && key < 250
- tablets = table.getTabletsLocations(getKeyInBytes(150), getKeyInBytes(250), 100000);
- assertEquals(1, tablets.size());
- assertArrayEquals(getKeyInBytes(200), tablets.get(0).getPartition().getPartitionKeyStart());
- assertArrayEquals(getKeyInBytes(300), tablets.get(0).getPartition().getPartitionKeyEnd());
- }
-
- public byte[] getKeyInBytes(int i) {
- PartialRow row = schema.newPartialRow();
- row.addInt(0, i);
- return row.encodePrimaryKey();
- }
-
- public KuduTable createTableWithSplitsAndTest(int splitsCount) throws Exception {
- String tableName = name.getMethodName() + System.currentTimeMillis();
- CreateTableOptions builder = getBasicCreateTableOptions();
-
- if (splitsCount != 0) {
- for (int i = 1; i <= splitsCount; i++) {
- PartialRow row = schema.newPartialRow();
- row.addInt(0, i);
- builder.addSplitRow(row);
- }
- }
- KuduTable table = createTable(tableName, schema, builder);
-
- List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
- assertEquals(splitsCount + 1, tablets.size());
- assertEquals(splitsCount + 1, table.asyncGetTabletsLocations(DEFAULT_SLEEP).join().size());
- for (LocatedTablet tablet : tablets) {
- assertEquals(3, tablet.getReplicas().size());
- }
- return table;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestLeaderFailover.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestLeaderFailover.java b/java/kudu-client/src/test/java/org/kududb/client/TestLeaderFailover.java
deleted file mode 100644
index 49ac502..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestLeaderFailover.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class TestLeaderFailover extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- TestLeaderFailover.class.getName() + "-" + System.currentTimeMillis();
- private static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
-
- CreateTableOptions builder = getBasicCreateTableOptions();
- createTable(TABLE_NAME, basicSchema, builder);
-
- table = openTable(TABLE_NAME);
- }
-
- /**
- * This test writes 3 rows, kills the leader, then tries to write another 3 rows. Finally it
- * counts to make sure we have 6 of them.
- *
- * This test won't run if we didn't start the cluster.
- */
- @Test(timeout = 100000)
- public void testFailover() throws Exception {
- KuduSession session = syncClient.newSession();
- for (int i = 0; i < 3; i++) {
- session.apply(createBasicSchemaInsert(table, i));
- }
-
- // Make sure the rows are in there before messing things up.
- AsyncKuduScanner scanner = client.newScannerBuilder(table).build();
- assertEquals(3, countRowsInScan(scanner));
-
- killTabletLeader(table);
-
- for (int i = 3; i < 6; i++) {
- OperationResponse resp = session.apply(createBasicSchemaInsert(table, i));
- if (resp.hasRowError()) {
- fail("Encountered a row error " + resp.getRowError());
- }
- }
-
- scanner = client.newScannerBuilder(table).build();
- assertEquals(6, countRowsInScan(scanner));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestMasterFailover.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestMasterFailover.java b/java/kudu-client/src/test/java/org/kududb/client/TestMasterFailover.java
deleted file mode 100644
index 2f91a6e..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestMasterFailover.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertEquals;
-
-
-/**
- * Tests {@link AsyncKuduClient} with multiple masters.
- */
-public class TestMasterFailover extends BaseKuduTest {
- private static final Logger LOG = LoggerFactory.getLogger(TestMasterFailover.class);
- private static final String TABLE_NAME =
- TestMasterFailover.class.getName() + "-" + System.currentTimeMillis();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions());
- }
-
- /**
- * This test is disabled as we're not supporting multi-master just yet.
- */
- @Test(timeout = 30000)
- @Ignore
- public void testKillLeader() throws Exception {
- int countMasters = masterHostPorts.size();
- if (countMasters < 3) {
- LOG.info("This test requires at least 3 master servers, but only " + countMasters +
- " are specified.");
- return;
- }
- killMasterLeader();
-
- // Test that we can open a previously created table after killing the leader master.
- KuduTable table = openTable(TABLE_NAME);
- assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
-
- // Test that we can create a new table when one of the masters is down.
- String newTableName = TABLE_NAME + "-afterLeaderIsDead";
- createTable(newTableName, basicSchema, new CreateTableOptions());
- table = openTable(newTableName);
- assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
-
- // Test that we can initialize a client when one of the masters specified in the
- // connection string is down.
- AsyncKuduClient newClient = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses).build();
- table = newClient.openTable(newTableName).join(DEFAULT_SLEEP);
- assertEquals(0, countRowsInScan(newClient.newScannerBuilder(table).build()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java b/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java
deleted file mode 100644
index 82ffacb..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestMiniKuduCluster.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.client;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.Socket;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class TestMiniKuduCluster {
-
- private static final int NUM_TABLET_SERVERS = 3;
- private static final int DEFAULT_NUM_MASTERS = 1;
-
- private MiniKuduCluster cluster;
-
- @Before
- public void before() throws Exception {
- cluster = new MiniKuduCluster.MiniKuduClusterBuilder()
- .numMasters(DEFAULT_NUM_MASTERS)
- .numTservers(NUM_TABLET_SERVERS)
- .build();
- assertTrue(cluster.waitForTabletServers(NUM_TABLET_SERVERS));
- }
-
- @After
- public void after() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @Test(timeout = 50000)
- public void test() throws Exception {
-
- assertEquals(DEFAULT_NUM_MASTERS, cluster.getMasterProcesses().size());
- assertEquals(NUM_TABLET_SERVERS, cluster.getTabletServerProcesses().size());
-
- {
- // Kill the master.
- int masterPort = cluster.getMasterProcesses().keySet().iterator().next();
- testPort(masterPort, true, 1000);
- cluster.killMasterOnPort(masterPort);
-
- testPort(masterPort, false, 2000);
-
- // Restart the master.
- cluster.restartDeadMasterOnPort(masterPort);
-
- // Test we can reach it.
- testPort(masterPort, true, 3000);
- }
-
-
- {
- // Kill the first TS.
- int tsPort = cluster.getTabletServerProcesses().keySet().iterator().next();
- testPort(tsPort, true, 1000);
- cluster.killTabletServerOnPort(tsPort);
-
- testPort(tsPort, false, 2000);
-
- // Restart it.
- cluster.restartDeadTabletServerOnPort(tsPort);
-
- testPort(tsPort, true, 3000);
- }
-
- assertEquals(DEFAULT_NUM_MASTERS, cluster.getMasterProcesses().size());
- assertEquals(NUM_TABLET_SERVERS, cluster.getTabletServerProcesses().size());
- }
-
- /**
- * Test without the specified is open or closed, waiting up to a certain time.
- * The longer you expect it might for the socket to become open or closed.
- * @param port the port to test
- * @param testIsOpen true if we should want it to be open, false if we want it closed
- * @param timeout how long we're willing to wait before it happens
- * @throws InterruptedException
- */
- private void testPort(int port, boolean testIsOpen, long timeout) throws InterruptedException {
- DeadlineTracker tracker = new DeadlineTracker();
- while (tracker.getElapsedMillis() < timeout) {
- try {
- Socket socket = new Socket(TestUtils.getUniqueLocalhost(), port);
- socket.close();
- if (testIsOpen) {
- return;
- }
- } catch (IOException e) {
- if (!testIsOpen) {
- return;
- }
- }
- Thread.sleep(200);
- }
- fail("Port " + port + " is still " + (testIsOpen ? "closed " : "open"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestOperation.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestOperation.java b/java/kudu-client/src/test/java/org/kududb/client/TestOperation.java
deleted file mode 100644
index f305fbf..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestOperation.java
+++ /dev/null
@@ -1,166 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.WireProtocol.RowOperationsPB;
-import org.kududb.client.Operation.ChangeType;
-import org.kududb.tserver.Tserver.WriteRequestPBOrBuilder;
-import org.mockito.Mockito;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.primitives.Longs;
-
-/**
- * Unit tests for Operation
- */
-public class TestOperation {
-
- private Schema createManyStringsSchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.STRING).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.STRING).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.STRING).nullable(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.STRING).nullable(true).build());
- return new Schema(columns);
- }
-
- @Test
- public void testSetStrings() {
- KuduTable table = Mockito.mock(KuduTable.class);
- Mockito.doReturn(createManyStringsSchema()).when(table).getSchema();
- Insert insert = new Insert(table);
- PartialRow row = insert.getRow();
- row.addString("c0", "c0_val");
- row.addString("c2", "c2_val");
- row.addString("c1", "c1_val");
- row.addString("c3", "c3_val");
- row.addString("c4", "c4_val");
-
- {
- WriteRequestPBOrBuilder pb =
- Operation.createAndFillWriteRequestPB(ImmutableList.<Operation>of(insert));
- RowOperationsPB rowOps = pb.getRowOperations();
- assertEquals(6 * 5, rowOps.getIndirectData().size());
- assertEquals("c0_valc1_valc2_valc3_valc4_val", rowOps.getIndirectData().toStringUtf8());
- byte[] rows = rowOps.getRows().toByteArray();
- assertEquals(ChangeType.INSERT.toEncodedByte(), rows[0]);
- // The "isset" bitset should have 5 bits set
- assertEquals(0x1f, rows[1]);
- // The "null" bitset should have no bits set
- assertEquals(0, rows[2]);
-
- // Check the strings.
- int offset = 3;
- for (int i = 0; i <= 4; i++) {
- // The offset into the indirect buffer
- assertEquals(6 * i, Bytes.getLong(rows, offset));
- offset += Longs.BYTES;
- // The length of the pointed-to string.
- assertEquals(6, Bytes.getLong(rows, offset));
- offset += Longs.BYTES;
- }
-
- // Should have used up whole buffer.
- assertEquals(rows.length, offset);
- }
-
- // Setting a field to NULL should add to the null bitmap and remove
- // the old value from the indirect buffer.
- row.setNull("c3");
- {
- WriteRequestPBOrBuilder pb =
- Operation.createAndFillWriteRequestPB(ImmutableList.<Operation>of(insert));
- RowOperationsPB rowOps = pb.getRowOperations();
- assertEquals(6 * 4, rowOps.getIndirectData().size());
- assertEquals("c0_valc1_valc2_valc4_val", rowOps.getIndirectData().toStringUtf8());
- byte[] rows = rowOps.getRows().toByteArray();
- assertEquals(ChangeType.INSERT.toEncodedByte(), rows[0]);
- // The "isset" bitset should have 5 bits set
- assertEquals(0x1f, rows[1]);
- // The "null" bitset should have 1 bit set for the null column
- assertEquals(1 << 3, rows[2]);
-
- // Check the strings.
- int offset = 3;
- int indirOffset = 0;
- for (int i = 0; i <= 4; i++) {
- if (i == 3) continue;
- // The offset into the indirect buffer
- assertEquals(indirOffset, Bytes.getLong(rows, offset));
- indirOffset += 6;
- offset += Longs.BYTES;
- // The length of the pointed-to string.
- assertEquals(6, Bytes.getLong(rows, offset));
- offset += Longs.BYTES;
- }
- // Should have used up whole buffer.
- assertEquals(rows.length, offset);
- }
- }
-
- private Schema createAllTypesKeySchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(7);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.INT8).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.INT16).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.INT32).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.INT64).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.TIMESTAMP).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c5", Type.STRING).key(true).build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("c6", Type.BINARY).key(true).build());
- return new Schema(columns);
- }
-
- @Test
- public void testRowKeyStringify() {
- KuduTable table = Mockito.mock(KuduTable.class);
- Mockito.doReturn(createAllTypesKeySchema()).when(table).getSchema();
- Insert insert = new Insert(table);
- PartialRow row = insert.getRow();
- row.addByte("c0", (byte) 1);
- row.addShort("c1", (short) 2);
- row.addInt("c2", 3);
- row.addLong("c3", 4);
- row.addLong("c4", 5);
- row.addString("c5", "c5_val");
- row.addBinary("c6", Bytes.fromString("c6_val"));
-
- assertEquals("(int8 c0=1, int16 c1=2, int32 c2=3, int64 c3=4, timestamp c4=5, string" +
- " c5=c5_val, binary c6=\"c6_val\")",
- insert.getRow().stringifyRowKey());
-
- // Test an incomplete row key.
- insert = new Insert(table);
- row = insert.getRow();
- row.addByte("c0", (byte) 1);
- try {
- row.stringifyRowKey();
- fail("Should not be able to stringifyRowKey when not all keys are specified");
- } catch (IllegalStateException ise) {
- // Expected.
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestRequestTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestRequestTracker.java b/java/kudu-client/src/test/java/org/kududb/client/TestRequestTracker.java
deleted file mode 100644
index 7528de6..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestRequestTracker.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Test;
-
-public class TestRequestTracker {
-
- @Test(timeout = 10000)
- public void test() {
- RequestTracker tracker = new RequestTracker("test");
-
- // A new tracker should have no incomplete RPCs.
- assertEquals(RequestTracker.NO_SEQ_NO, tracker.firstIncomplete());
-
- int max = 10;
-
- for (int i = 0; i < max; i++) {
- tracker.newSeqNo();
- }
-
- // The first RPC is the incomplete one.
- assertEquals(1, tracker.firstIncomplete());
-
- // Mark the first as complete, incomplete should advance by 1.
- tracker.rpcCompleted(1);
- assertEquals(2, tracker.firstIncomplete());
-
- // Mark the RPC in the middle as complete, first incomplete doesn't change.
- tracker.rpcCompleted(5);
- assertEquals(2, tracker.firstIncomplete());
-
- // Mark the first half as complete.
- // Note that we're also testing that rpcCompleted is idempotent.
- for (int i = 1; i < max / 2; i++) {
- tracker.rpcCompleted(i);
- }
-
- assertEquals(6, tracker.firstIncomplete());
-
- // Get a few more sequence numbers.
- long lastSeqNo = 0;
- for (int i = max / 2; i <= max; i++) {
- lastSeqNo = tracker.newSeqNo();
- }
-
- // Mark them all as complete except the last one.
- while (tracker.firstIncomplete() != lastSeqNo) {
- tracker.rpcCompleted(tracker.firstIncomplete());
- }
-
- assertEquals(lastSeqNo, tracker.firstIncomplete());
- tracker.rpcCompleted(lastSeqNo);
-
- // Test that we get back to NO_SEQ_NO after marking them all.
- assertEquals(RequestTracker.NO_SEQ_NO, tracker.firstIncomplete());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java b/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java
deleted file mode 100644
index 90d11aa..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestRowErrors.java
+++ /dev/null
@@ -1,98 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-import com.google.common.collect.ImmutableList;
-
-public class TestRowErrors extends BaseKuduTest {
-
- private static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
-
- }
-
- @Test(timeout = 100000)
- public void singleTabletTest() throws Exception {
- String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis();
- createTable(tableName, basicSchema, getBasicCreateTableOptions());
- table = openTable(tableName);
- AsyncKuduSession session = client.newSession();
-
- // Insert 3 rows to play with.
- for (int i = 0; i < 3; i++) {
- session.apply(createInsert(i)).join(DEFAULT_SLEEP);
- }
-
- // Try a single dupe row insert with AUTO_FLUSH_SYNC.
- Insert dupeForZero = createInsert(0);
- OperationResponse resp = session.apply(dupeForZero).join(DEFAULT_SLEEP);
- assertTrue(resp.hasRowError());
- assertTrue(resp.getRowError().getOperation() == dupeForZero);
-
- // Now try inserting two dupes and one good row, make sure we get only two errors back.
- dupeForZero = createInsert(0);
- Insert dupeForTwo = createInsert(2);
- session.setFlushMode(AsyncKuduSession.FlushMode.MANUAL_FLUSH);
- session.apply(dupeForZero);
- session.apply(dupeForTwo);
- session.apply(createInsert(4));
-
- List<OperationResponse> responses = session.flush().join(DEFAULT_SLEEP);
- List<RowError> errors = OperationResponse.collectErrors(responses);
- assertEquals(2, errors.size());
- assertTrue(errors.get(0).getOperation() == dupeForZero);
- assertTrue(errors.get(1).getOperation() == dupeForTwo);
- }
-
- /**
- * Test collecting errors from multiple tablets.
- * @throws Exception
- */
- @Test(timeout = 100000)
- public void multiTabletTest() throws Exception {
- String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis();
- createFourTabletsTableWithNineRows(tableName);
- table = openTable(tableName);
- KuduSession session = syncClient.newSession();
- session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND);
-
- int dupRows = 3;
- session.apply(createInsert(12));
- session.apply(createInsert(22));
- session.apply(createInsert(32));
-
- session.flush();
-
- RowErrorsAndOverflowStatus reos = session.getPendingErrors();
- assertEquals(dupRows, reos.getRowErrors().length);
- assertEquals(0, session.countPendingErrors());
- }
-
- private Insert createInsert(int key) {
- return createBasicSchemaInsert(table, key);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestRowResult.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestRowResult.java b/java/kudu-client/src/test/java/org/kududb/client/TestRowResult.java
deleted file mode 100644
index 1b302c1..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestRowResult.java
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.kududb.Type;
-
-import java.nio.ByteBuffer;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestRowResult extends BaseKuduTest {
-
- // Generate a unique table name
- private static final String TABLE_NAME =
- TestRowResult.class.getName() + "-" + System.currentTimeMillis();
-
- private static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- createTable(TABLE_NAME, allTypesSchema, getAllTypesCreateTableOptions());
- table = openTable(TABLE_NAME);
- }
-
- @Test(timeout = 10000)
- public void test() throws Exception {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
-
- row.addByte(0, (byte) 1);
- row.addShort(1, (short) 2);
- row.addInt(2, 3);
- row.addLong(3, 4l);
- row.addBoolean(4, true);
- row.addFloat(5, 5.6f);
- row.addDouble(6, 7.8);
- row.addString(7, "string-value");
- row.addBinary(8, "binary-array".getBytes());
- ByteBuffer bb = ByteBuffer.wrap("binary-bytebuffer".getBytes());
- bb.position(7); // We're only inserting the bytebuffer part of the original array.
- row.addBinary(9, bb);
- row.setNull(10);
- row.addLong(11, 11l);
-
- KuduSession session = syncClient.newSession();
- session.apply(insert);
-
- KuduScanner scanner = syncClient.newScannerBuilder(table).build();
- while (scanner.hasMoreRows()) {
- RowResultIterator it = scanner.nextRows();
- assertTrue(it.hasNext());
- RowResult rr = it.next();
-
- assertEquals((byte) 1, rr.getByte(0));
- assertEquals((byte) 1, rr.getByte(allTypesSchema.getColumnByIndex(0).getName()));
-
- assertEquals((short) 2, rr.getShort(1));
- assertEquals((short) 2, rr.getShort(allTypesSchema.getColumnByIndex(1).getName()));
-
- assertEquals(3, rr.getInt(2));
- assertEquals(3, rr.getInt(allTypesSchema.getColumnByIndex(2).getName()));
-
- assertEquals(4, rr.getLong(3));
- assertEquals(4, rr.getLong(allTypesSchema.getColumnByIndex(3).getName()));
-
- assertEquals(true, rr.getBoolean(4));
- assertEquals(true, rr.getBoolean(allTypesSchema.getColumnByIndex(4).getName()));
-
- assertEquals(5.6f, rr.getFloat(5), .001f);
- assertEquals(5.6f, rr.getFloat(allTypesSchema.getColumnByIndex(5).getName()), .001f);
-
- assertEquals(7.8, rr.getDouble(6), .001);
- assertEquals(7.8, rr.getDouble(allTypesSchema.getColumnByIndex(6).getName()), .001f);
-
- assertEquals("string-value", rr.getString(7));
- assertEquals("string-value", rr.getString(allTypesSchema.getColumnByIndex(7).getName()));
-
- assertArrayEquals("binary-array".getBytes(), rr.getBinaryCopy(8));
- assertArrayEquals("binary-array".getBytes(),
- rr.getBinaryCopy(allTypesSchema.getColumnByIndex(8).getName()));
-
- ByteBuffer buffer = rr.getBinary(8);
- assertEquals(buffer, rr.getBinary(allTypesSchema.getColumnByIndex(8).getName()));
- byte[] binaryValue = new byte[buffer.remaining()];
- buffer.get(binaryValue);
- assertArrayEquals("binary-array".getBytes(), binaryValue);
-
- assertArrayEquals("bytebuffer".getBytes(), rr.getBinaryCopy(9));
-
- assertEquals(true, rr.isNull(10));
- assertEquals(true, rr.isNull(allTypesSchema.getColumnByIndex(10).getName()));
-
- assertEquals(11, rr.getLong(11));
- assertEquals(11, rr.getLong(allTypesSchema.getColumnByIndex(11).getName()));
-
- // We test with the column name once since it's the same method for all types, unlike above.
- assertEquals(Type.INT8, rr.getColumnType(allTypesSchema.getColumnByIndex(0).getName()));
- assertEquals(Type.INT8, rr.getColumnType(0));
- assertEquals(Type.INT16, rr.getColumnType(1));
- assertEquals(Type.INT32, rr.getColumnType(2));
- assertEquals(Type.INT64, rr.getColumnType(3));
- assertEquals(Type.BOOL, rr.getColumnType(4));
- assertEquals(Type.FLOAT, rr.getColumnType(5));
- assertEquals(Type.DOUBLE, rr.getColumnType(6));
- assertEquals(Type.STRING, rr.getColumnType(7));
- assertEquals(Type.BINARY, rr.getColumnType(8));
- assertEquals(Type.TIMESTAMP, rr.getColumnType(11));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestScanPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestScanPredicate.java b/java/kudu-client/src/test/java/org/kududb/client/TestScanPredicate.java
deleted file mode 100644
index d67380b..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestScanPredicate.java
+++ /dev/null
@@ -1,609 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedSet;
-import org.junit.Assert;
-import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.KuduPredicate.ComparisonOp;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-
-public class TestScanPredicate extends BaseKuduTest {
-
- private Schema createTableSchema(Type type) {
- ColumnSchema key = new ColumnSchema.ColumnSchemaBuilder("key", Type.INT64).key(true).build();
- ColumnSchema val = new ColumnSchema.ColumnSchemaBuilder("value", type).nullable(true).build();
- return new Schema(ImmutableList.of(key, val));
- }
-
- private static CreateTableOptions createTableOptions() {
- return new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
- }
-
- private int countRows(KuduTable table, KuduPredicate... predicates) throws Exception {
- KuduScanner.KuduScannerBuilder scanBuilder = new KuduScanner.KuduScannerBuilder(client, table);
- for (KuduPredicate predicate : predicates) {
- scanBuilder.addPredicate(predicate);
- }
-
- KuduScanner scanner = scanBuilder.build();
-
- int count = 0;
- while (scanner.hasMoreRows()) {
- count += scanner.nextRows().getNumRows();
- }
- return count;
- }
-
- private NavigableSet<Long> createIntegerValues(Type type) {
- NavigableSet<Long> values = new TreeSet<>();
- for (long i = -50; i < 50; i++) {
- values.add(i);
- }
- values.add(KuduPredicate.minIntValue(type));
- values.add(KuduPredicate.minIntValue(type) + 1);
- values.add(KuduPredicate.maxIntValue(type) - 1);
- values.add(KuduPredicate.maxIntValue(type));
- return values;
- }
-
- private List<Long> createIntegerTestValues(Type type) {
- return ImmutableList.of(
- KuduPredicate.minIntValue(type),
- KuduPredicate.minIntValue(type) + 1,
- -51L,
- 50L,
- 0L,
- 49L,
- 50L,
- KuduPredicate.maxIntValue(type) - 1,
- KuduPredicate.maxIntValue(type));
- }
-
- private NavigableSet<Float> createFloatValues() {
- NavigableSet<Float> values = new TreeSet<>();
- for (long i = -50; i < 50; i++) {
- values.add((float) i + (float) i / 100.0F);
- }
-
- values.add(Float.NEGATIVE_INFINITY);
- values.add(-Float.MAX_VALUE);
- values.add(-Float.MIN_NORMAL);
- values.add(-Float.MIN_VALUE);
- values.add(Float.MIN_VALUE);
- values.add(Float.MIN_NORMAL);
- values.add(Float.MAX_VALUE);
- values.add(Float.POSITIVE_INFINITY);
-
- // TODO: uncomment after fixing KUDU-1386
- // values.add(Float.NaN);
- return values;
- }
-
- private List<Float> createFloatTestValues() {
- return ImmutableList.of(
- Float.NEGATIVE_INFINITY,
- -Float.MAX_VALUE,
- -100.0F,
- -1.1F,
- -1.0F,
- -Float.MIN_NORMAL,
- -Float.MIN_VALUE,
- 0.0F,
- Float.MIN_VALUE,
- Float.MIN_NORMAL,
- 1.0F,
- 1.1F,
- 100.0F,
- Float.MAX_VALUE,
- Float.POSITIVE_INFINITY
-
- // TODO: uncomment after fixing KUDU-1386
- // Float.NaN
- );
- }
-
- private NavigableSet<Double> createDoubleValues() {
- NavigableSet<Double> values = new TreeSet<>();
- for (long i = -50; i < 50; i++) {
- values.add((double) i + (double) i / 100.0);
- }
-
- values.add(Double.NEGATIVE_INFINITY);
- values.add(-Double.MAX_VALUE);
- values.add(-Double.MIN_NORMAL);
- values.add(-Double.MIN_VALUE);
- values.add(Double.MIN_VALUE);
- values.add(Double.MIN_NORMAL);
- values.add(Double.MAX_VALUE);
- values.add(Double.POSITIVE_INFINITY);
-
- // TODO: uncomment after fixing KUDU-1386
- // values.add(Double.NaN);
- return values;
- }
-
- private List<Double> createDoubleTestValues() {
- return ImmutableList.of(
- Double.NEGATIVE_INFINITY,
- -Double.MAX_VALUE,
- -100.0,
- -1.1,
- -1.0,
- -Double.MIN_NORMAL,
- -Double.MIN_VALUE,
- 0.0,
- Double.MIN_VALUE,
- Double.MIN_NORMAL,
- 1.0,
- 1.1,
- 100.0,
- Double.MAX_VALUE,
- Double.POSITIVE_INFINITY
-
- // TODO: uncomment after fixing KUDU-1386
- // Double.NaN
- );
- }
-
- private NavigableSet<String> createStringValues() {
- return ImmutableSortedSet.of("", "\0", "\0\0", "a", "a\0", "a\0a", "aa\0");
- }
-
- private List<String> createStringTestValues() {
- List<String> values = new ArrayList<>(createStringValues());
- values.add("aa");
- values.add("\1");
- values.add("a\1");
- return values;
- }
-
- private void checkIntPredicates(KuduTable table,
- NavigableSet<Long> values,
- List<Long> testValues) throws Exception {
- ColumnSchema col = table.getSchema().getColumn("value");
- Assert.assertEquals(values.size() + 1, countRows(table));
- for (long v : testValues) {
- // value = v
- KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
- Assert.assertEquals(values.contains(v) ? 1 : 0, countRows(table, equal));
-
- // value >= v
- KuduPredicate greaterEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
- Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
-
- // value <= v
- KuduPredicate lessEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
- Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
-
- // value > v
- KuduPredicate greater =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
- Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
-
- // value < v
- KuduPredicate less =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
- Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
- }
- }
-
- @Test
- public void testBoolPredicates() throws Exception {
- Schema schema = createTableSchema(Type.BOOL);
- syncClient.createTable("bool-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("bool-table");
-
- NavigableSet<Boolean> values = ImmutableSortedSet.of(false, true);
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (boolean value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addBoolean("value", value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- ColumnSchema col = table.getSchema().getColumn("value");
- Assert.assertEquals(values.size() + 1, countRows(table));
-
- for (boolean v : values) {
- // value = v
- KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
- Assert.assertEquals(values.contains(v) ? 1 : 0, countRows(table, equal));
-
- // value >= v
- KuduPredicate greaterEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
- Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
-
- // value <= v
- KuduPredicate lessEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
- Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
-
- // value > v
- KuduPredicate greater =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
- Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
-
- // value < v
- KuduPredicate less =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
- Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
- }
- }
-
- @Test
- public void testBytePredicates() throws Exception {
- Schema schema = createTableSchema(Type.INT8);
- syncClient.createTable("byte-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("byte-table");
-
- NavigableSet<Long> values = createIntegerValues(Type.INT8);
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (long value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addByte("value", (byte) value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- checkIntPredicates(table, values, createIntegerTestValues(Type.INT8));
- }
-
- @Test
- public void testShortPredicates() throws Exception {
- Schema schema = createTableSchema(Type.INT16);
- syncClient.createTable("short-table", schema,
- new CreateTableOptions().setRangePartitionColumns(
- ImmutableList.<String>of()));
- KuduTable table = syncClient.openTable("short-table");
-
- NavigableSet<Long> values = createIntegerValues(Type.INT16);
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (long value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addShort("value", (short) value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- checkIntPredicates(table, values, createIntegerTestValues(Type.INT16));
- }
-
- @Test
- public void testIntPredicates() throws Exception {
- Schema schema = createTableSchema(Type.INT32);
- syncClient.createTable("int-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("int-table");
-
- NavigableSet<Long> values = createIntegerValues(Type.INT32);
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (long value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addInt("value", (int) value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- checkIntPredicates(table, values, createIntegerTestValues(Type.INT32));
- }
-
- @Test
- public void testLongPredicates() throws Exception {
- Schema schema = createTableSchema(Type.INT64);
- syncClient.createTable("long-table", schema,
- new CreateTableOptions().setRangePartitionColumns(
- ImmutableList.<String>of()));
- KuduTable table = syncClient.openTable("long-table");
-
- NavigableSet<Long> values = createIntegerValues(Type.INT64);
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (long value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addLong("value", value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- checkIntPredicates(table, values, createIntegerTestValues(Type.INT64));
- }
-
- @Test
- public void testTimestampPredicate() throws Exception {
- Schema schema = createTableSchema(Type.INT64);
- syncClient.createTable("timestamp-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("timestamp-table");
-
- NavigableSet<Long> values = createIntegerValues(Type.INT64);
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (long value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addLong("value", value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- checkIntPredicates(table, values, createIntegerTestValues(Type.INT64));
- }
-
- @Test
- public void testFloatPredicates() throws Exception {
- Schema schema = createTableSchema(Type.FLOAT);
- syncClient.createTable("float-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("float-table");
-
- NavigableSet<Float> values = createFloatValues();
- List<Float> testValues = createFloatTestValues();
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (float value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addFloat("value", value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- ColumnSchema col = table.getSchema().getColumn("value");
- Assert.assertEquals(values.size() + 1, countRows(table));
-
- for (float v : testValues) {
- // value = v
- KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
- Assert.assertEquals(values.subSet(v, true, v, true).size(), countRows(table, equal));
-
- // value >= v
- KuduPredicate greaterEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
- Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
-
- // value <= v
- KuduPredicate lessEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
- Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
-
- // value > v
- KuduPredicate greater =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
- Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
-
- // value < v
- KuduPredicate less =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
- Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
- }
- }
-
- @Test
- public void testDoublePredicates() throws Exception {
- Schema schema = createTableSchema(Type.DOUBLE);
- syncClient.createTable("double-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("double-table");
-
- NavigableSet<Double> values = createDoubleValues();
- List<Double> testValues = createDoubleTestValues();
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (double value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addDouble("value", value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- ColumnSchema col = table.getSchema().getColumn("value");
- Assert.assertEquals(values.size() + 1, countRows(table));
-
- for (double v : testValues) {
- // value = v
- KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
- Assert.assertEquals(values.subSet(v, true, v, true).size(), countRows(table, equal));
-
- // value >= v
- KuduPredicate greaterEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
- Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
-
- // value <= v
- KuduPredicate lessEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
- Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
-
- // value > v
- KuduPredicate greater =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
- Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
-
- // value < v
- KuduPredicate less =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
- Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
- }
- }
-
- @Test
- public void testStringPredicates() throws Exception {
- Schema schema = createTableSchema(Type.STRING);
- syncClient.createTable("string-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("string-table");
-
- NavigableSet<String> values = createStringValues();
- List<String> testValues = createStringTestValues();
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (String value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addString("value", value);
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- ColumnSchema col = table.getSchema().getColumn("value");
- Assert.assertEquals(values.size() + 1, countRows(table));
-
- for (String v : testValues) {
- // value = v
- KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
- Assert.assertEquals(values.subSet(v, true, v, true).size(), countRows(table, equal));
-
- // value >= v
- KuduPredicate greaterEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
- Assert.assertEquals(values.tailSet(v).size(), countRows(table, greaterEqual));
-
- // value <= v
- KuduPredicate lessEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
- Assert.assertEquals(values.headSet(v, true).size(), countRows(table, lessEqual));
-
- // value > v
- KuduPredicate greater =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
- Assert.assertEquals(values.tailSet(v, false).size(), countRows(table, greater));
-
- // value < v
- KuduPredicate less =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
- Assert.assertEquals(values.headSet(v).size(), countRows(table, less));
- }
- }
-
- @Test
- public void testBinaryPredicates() throws Exception {
- Schema schema = createTableSchema(Type.BINARY);
- syncClient.createTable("binary-table", schema, createTableOptions());
- KuduTable table = syncClient.openTable("binary-table");
-
- NavigableSet<String> values = createStringValues();
- List<String> testValues = createStringTestValues();
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- long i = 0;
- for (String value : values) {
- Insert insert = table.newInsert();
- insert.getRow().addLong("key", i++);
- insert.getRow().addBinary("value", Bytes.fromString(value));
- session.apply(insert);
- }
- Insert nullInsert = table.newInsert();
- nullInsert.getRow().addLong("key", i++);
- nullInsert.getRow().setNull("value");
- session.apply(nullInsert);
- session.flush();
-
- ColumnSchema col = table.getSchema().getColumn("value");
- Assert.assertEquals(values.size() + 1, countRows(table));
-
- for (String s : testValues) {
- byte[] v = Bytes.fromString(s);
- // value = v
- KuduPredicate equal = KuduPredicate.newComparisonPredicate(col, ComparisonOp.EQUAL, v);
- Assert.assertEquals(values.subSet(s, true, s, true).size(), countRows(table, equal));
-
- // value >= v
- KuduPredicate greaterEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER_EQUAL, v);
- Assert.assertEquals(values.tailSet(s).size(), countRows(table, greaterEqual));
-
- // value <= v
- KuduPredicate lessEqual =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS_EQUAL, v);
- Assert.assertEquals(values.headSet(s, true).size(), countRows(table, lessEqual));
-
- // value > v
- KuduPredicate greater =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.GREATER, v);
- Assert.assertEquals(values.tailSet(s, false).size(), countRows(table, greater));
-
- // value < v
- KuduPredicate less =
- KuduPredicate.newComparisonPredicate(col, ComparisonOp.LESS, v);
- Assert.assertEquals(values.headSet(s).size(), countRows(table, less));
- }
- }
-}
[25/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
new file mode 100644
index 0000000..dea94c4
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduPredicate.java
@@ -0,0 +1,683 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.kududb.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.UnsignedBytes;
+import com.google.protobuf.ByteString;
+import org.kududb.ColumnSchema;
+import org.kududb.Common;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.util.Arrays;
+
+/**
+ * A predicate which can be used to filter rows based on the value of a column.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduPredicate {
+
+ /**
+ * The predicate type.
+ */
+ @InterfaceAudience.Private
+ enum PredicateType {
+ /** A predicate which filters all rows. */
+ NONE,
+ /** A predicate which filters all rows not equal to a value. */
+ EQUALITY,
+ /** A predicate which filters all rows not in a range. */
+ RANGE,
+ /** A predicate which filters all null rows. */
+ IS_NOT_NULL,
+ }
+
+ /**
+ * The comparison operator of a predicate.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public enum ComparisonOp {
+ GREATER,
+ GREATER_EQUAL,
+ EQUAL,
+ LESS,
+ LESS_EQUAL,
+ }
+
+ private final PredicateType type;
+ private final ColumnSchema column;
+
+ /**
+ * The inclusive lower bound value if this is a Range predicate, or
+ * the createEquality value if this is an Equality predicate.
+ */
+ private final byte[] lower;
+
+ /** The exclusive upper bound value if this is a Range predicate. */
+ private final byte[] upper;
+
+ /**
+ * Creates a new {@code KuduPredicate} on a boolean column.
+ * @param column the column schema
+ * @param op the comparison operation
+ * @param value the value to compare against
+ */
+ public static KuduPredicate newComparisonPredicate(ColumnSchema column,
+ ComparisonOp op,
+ boolean value) {
+ checkColumn(column, Type.BOOL);
+ // Create the comparison predicate. Range predicates on boolean values can
+ // always be converted to either an equality, an IS NOT NULL (filtering only
+ // null values), or NONE (filtering all values).
+ switch (op) {
+ case GREATER: {
+ // b > true -> b NONE
+ // b > false -> b = true
+ if (value) {
+ return none(column);
+ } else {
+ return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(true), null);
+ }
+ }
+ case GREATER_EQUAL: {
+ // b >= true -> b = true
+ // b >= false -> b IS NOT NULL
+ if (value) {
+ return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(true), null);
+ } else {
+ return newIsNotNullPredicate(column);
+ }
+ }
+ case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column,
+ Bytes.fromBoolean(value), null);
+ case LESS: {
+ // b < true -> b NONE
+ // b < false -> b = true
+ if (value) {
+ return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(false), null);
+ } else {
+ return none(column);
+ }
+ }
+ case LESS_EQUAL: {
+ // b <= true -> b IS NOT NULL
+ // b <= false -> b = false
+ if (value) {
+ return newIsNotNullPredicate(column);
+ } else {
+ return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(false), null);
+ }
+ }
+ default: throw new RuntimeException("unknown comparison op");
+ }
+ }
+
+ /**
+ * Creates a new comparison predicate on an integer or timestamp column.
+ * @param column the column schema
+ * @param op the comparison operation
+ * @param value the value to compare against
+ */
+ public static KuduPredicate newComparisonPredicate(ColumnSchema column,
+ ComparisonOp op,
+ long value) {
+ checkColumn(column, Type.INT8, Type.INT16, Type.INT32, Type.INT64, Type.TIMESTAMP);
+ Preconditions.checkArgument(value <= maxIntValue(column.getType()) &&
+ value >= minIntValue(column.getType()),
+ "integer value out of range for %s column: %s",
+ column.getType(), value);
+
+ if (op == ComparisonOp.LESS_EQUAL) {
+ if (value == maxIntValue(column.getType())) {
+ // If the value can't be incremented because it is at the top end of the
+ // range, then substitute the predicate with an IS NOT NULL predicate.
+ // This has the same effect as an inclusive upper bound on the maximum
+ // value. If the column is not nullable then the IS NOT NULL predicate
+ // is ignored.
+ return newIsNotNullPredicate(column);
+ }
+ value += 1;
+ } else if (op == ComparisonOp.GREATER) {
+ if (value == maxIntValue(column.getType())) {
+ return none(column);
+ }
+ value += 1;
+ }
+
+ byte[] bytes;
+ switch (column.getType()) {
+ case INT8: {
+ bytes = new byte[] { (byte) value };
+ break;
+ }
+ case INT16: {
+ bytes = Bytes.fromShort((short) value);
+ break;
+ }
+ case INT32: {
+ bytes = Bytes.fromInt((int) value);
+ break;
+ }
+ case INT64:
+ case TIMESTAMP: {
+ bytes = Bytes.fromLong(value);
+ break;
+ }
+ default: throw new RuntimeException("already checked");
+ }
+ switch (op) {
+ case GREATER:
+ case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
+ case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
+ case LESS:
+ case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
+ default: throw new RuntimeException("unknown comparison op");
+ }
+ }
+
+ /**
+ * Creates a new comparison predicate on a float column.
+ * @param column the column schema
+ * @param op the comparison operation
+ * @param value the value to compare against
+ */
+ public static KuduPredicate newComparisonPredicate(ColumnSchema column,
+ ComparisonOp op,
+ float value) {
+ checkColumn(column, Type.FLOAT);
+ if (op == ComparisonOp.LESS_EQUAL) {
+ if (value == Float.POSITIVE_INFINITY) {
+ return newIsNotNullPredicate(column);
+ }
+ value = Math.nextAfter(value, Float.POSITIVE_INFINITY);
+ } else if (op == ComparisonOp.GREATER) {
+ if (value == Float.POSITIVE_INFINITY) {
+ return none(column);
+ }
+ value = Math.nextAfter(value, Float.POSITIVE_INFINITY);
+ }
+
+ byte[] bytes = Bytes.fromFloat(value);
+ switch (op) {
+ case GREATER:
+ case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
+ case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
+ case LESS:
+ case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
+ default: throw new RuntimeException("unknown comparison op");
+ }
+ }
+
+ /**
+ * Creates a new comparison predicate on a double column.
+ * @param column the column schema
+ * @param op the comparison operation
+ * @param value the value to compare against
+ */
+ public static KuduPredicate newComparisonPredicate(ColumnSchema column,
+ ComparisonOp op,
+ double value) {
+ checkColumn(column, Type.DOUBLE);
+ if (op == ComparisonOp.LESS_EQUAL) {
+ if (value == Double.POSITIVE_INFINITY) {
+ return newIsNotNullPredicate(column);
+ }
+ value = Math.nextAfter(value, Double.POSITIVE_INFINITY);
+ } else if (op == ComparisonOp.GREATER) {
+ if (value == Double.POSITIVE_INFINITY) {
+ return none(column);
+ }
+ value = Math.nextAfter(value, Double.POSITIVE_INFINITY);
+ }
+
+ byte[] bytes = Bytes.fromDouble(value);
+ switch (op) {
+ case GREATER:
+ case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
+ case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
+ case LESS:
+ case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
+ default: throw new RuntimeException("unknown comparison op");
+ }
+ }
+
+ /**
+ * Creates a new comparison predicate on a string column.
+ * @param column the column schema
+ * @param op the comparison operation
+ * @param value the value to compare against
+ */
+ public static KuduPredicate newComparisonPredicate(ColumnSchema column,
+ ComparisonOp op,
+ String value) {
+ checkColumn(column, Type.STRING);
+
+ byte[] bytes = Bytes.fromString(value);
+ if (op == ComparisonOp.LESS_EQUAL || op == ComparisonOp.GREATER) {
+ bytes = Arrays.copyOf(bytes, bytes.length + 1);
+ }
+
+ switch (op) {
+ case GREATER:
+ case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
+ case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
+ case LESS:
+ case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
+ default: throw new RuntimeException("unknown comparison op");
+ }
+ }
+
+ /**
+ * Creates a new comparison predicate on a binary column.
+ * @param column the column schema
+ * @param op the comparison operation
+ * @param value the value to compare against
+ */
+ public static KuduPredicate newComparisonPredicate(ColumnSchema column,
+ ComparisonOp op,
+ byte[] value) {
+ checkColumn(column, Type.BINARY);
+
+ if (op == ComparisonOp.LESS_EQUAL || op == ComparisonOp.GREATER) {
+ value = Arrays.copyOf(value, value.length + 1);
+ }
+
+ switch (op) {
+ case GREATER:
+ case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, value, null);
+ case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, value, null);
+ case LESS:
+ case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, value);
+ default: throw new RuntimeException("unknown comparison op");
+ }
+ }
+
+ /**
+ * @param type the predicate type
+ * @param column the column to which the predicate applies
+ * @param lower the lower bound serialized value if this is a Range predicate,
+ * or the equality value if this is an Equality predicate
+ * @param upper the upper bound serialized value if this is an Equality predicate
+ */
+ @VisibleForTesting
+ KuduPredicate(PredicateType type, ColumnSchema column, byte[] lower, byte[] upper) {
+ this.type = type;
+ this.column = column;
+ this.lower = lower;
+ this.upper = upper;
+ }
+
+ /**
+ * Factory function for a {@code None} predicate.
+ * @param column the column to which the predicate applies
+ * @return a None predicate
+ */
+ @VisibleForTesting
+ static KuduPredicate none(ColumnSchema column) {
+ return new KuduPredicate(PredicateType.NONE, column, null, null);
+ }
+
+ /**
+ * Factory function for an {@code IS NOT NULL} predicate.
+ * @param column the column to which the predicate applies
+ * @return a {@code IS NOT NULL} predicate
+ */
+ @VisibleForTesting
+ static KuduPredicate newIsNotNullPredicate(ColumnSchema column) {
+ return new KuduPredicate(PredicateType.IS_NOT_NULL, column, null, null);
+ }
+
+ /**
+ * @return the type of this predicate
+ */
+ PredicateType getType() {
+ return type;
+ }
+
+ /**
+ * Merges another {@code ColumnPredicate} into this one, returning a new
+ * {@code ColumnPredicate} which matches the logical intersection ({@code AND})
+ * of the input predicates.
+ * @param other the predicate to merge with this predicate
+ * @return a new predicate that is the logical intersection
+ */
+ KuduPredicate merge(KuduPredicate other) {
+ Preconditions.checkArgument(column.equals(other.column),
+ "predicates from different columns may not be merged");
+ if (type == PredicateType.NONE || other.type == PredicateType.NONE) {
+ return none(column);
+ }
+
+ if (type == PredicateType.IS_NOT_NULL) {
+ // NOT NULL is less selective than all other predicate types, so the
+ // intersection of NOT NULL with any other predicate is just the other
+ // predicate.
+ //
+ // Note: this will no longer be true when an IS NULL predicate type is
+ // added.
+ return other;
+ }
+
+ if (type == PredicateType.EQUALITY) {
+ if (other.type == PredicateType.EQUALITY) {
+ if (compare(lower, other.lower) != 0) {
+ return none(this.column);
+ } else {
+ return this;
+ }
+ } else {
+ if ((other.lower == null || compare(lower, other.lower) >= 0) &&
+ (other.upper == null || compare(lower, other.upper) < 0)) {
+ return this;
+ } else {
+ return none(this.column);
+ }
+ }
+ } else {
+ if (other.type == PredicateType.EQUALITY) {
+ return other.merge(this);
+ } else {
+ byte[] newLower = other.lower == null ||
+ (lower != null && compare(lower, other.lower) >= 0) ? lower : other.lower;
+ byte[] newUpper = other.upper == null ||
+ (upper != null && compare(upper, other.upper) <= 0) ? upper : other.upper;
+ if (newLower != null && newUpper != null && compare(newLower, newUpper) >= 0) {
+ return none(column);
+ } else {
+ if (newLower != null && newUpper != null && areConsecutive(newLower, newUpper)) {
+ return new KuduPredicate(PredicateType.EQUALITY, column, newLower, null);
+ } else {
+ return new KuduPredicate(PredicateType.RANGE, column, newLower, newUpper);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * @return the schema of the predicate column
+ */
+ ColumnSchema getColumn() {
+ return column;
+ }
+
+ /**
+ * Convert the predicate to the protobuf representation.
+ * @return the protobuf message for this predicate.
+ */
+ @InterfaceAudience.Private
+ public Common.ColumnPredicatePB toPB() {
+ Common.ColumnPredicatePB.Builder builder = Common.ColumnPredicatePB.newBuilder();
+ builder.setColumn(column.getName());
+
+ switch (type) {
+ case EQUALITY: {
+ builder.getEqualityBuilder().setValue(ByteString.copyFrom(lower));
+ break;
+ }
+ case RANGE: {
+ Common.ColumnPredicatePB.Range.Builder b = builder.getRangeBuilder();
+ if (lower != null) {
+ b.setLower(ByteString.copyFrom(lower));
+ }
+ if (upper != null) {
+ b.setUpper(ByteString.copyFrom(upper));
+ }
+ break;
+ }
+ case IS_NOT_NULL: {
+ builder.setIsNotNull(builder.getIsNotNullBuilder());
+ break;
+ }
+ case NONE: throw new IllegalStateException(
+ "can not convert None predicate to protobuf message");
+ default: throw new IllegalArgumentException(
+ String.format("unknown predicate type: %s", type));
+ }
+ return builder.build();
+ }
+
+ /**
+ * Convert a column predicate protobuf message into a predicate.
+ * @return a predicate
+ */
+ @InterfaceAudience.Private
+ public static KuduPredicate fromPB(Schema schema, Common.ColumnPredicatePB pb) {
+ ColumnSchema column = schema.getColumn(pb.getColumn());
+ switch (pb.getPredicateCase()) {
+ case EQUALITY: {
+ return new KuduPredicate(PredicateType.EQUALITY, column,
+ pb.getEquality().getValue().toByteArray(), null);
+
+ }
+ case RANGE: {
+ Common.ColumnPredicatePB.Range range = pb.getRange();
+ return new KuduPredicate(PredicateType.RANGE, column,
+ range.hasLower() ? range.getLower().toByteArray() : null,
+ range.hasUpper() ? range.getUpper().toByteArray() : null);
+ }
+ default: throw new IllegalArgumentException("unknown predicate type");
+ }
+ }
+
+ /**
+ * Compares two bounds based on the type of this predicate's column.
+ * @param a the first serialized value
+ * @param b the second serialized value
+ * @return the comparison of the serialized values based on the column type
+ */
+ private int compare(byte[] a, byte[] b) {
+ switch (column.getType().getDataType()) {
+ case BOOL:
+ return Boolean.compare(Bytes.getBoolean(a), Bytes.getBoolean(b));
+ case INT8:
+ return Byte.compare(Bytes.getByte(a), Bytes.getByte(b));
+ case INT16:
+ return Short.compare(Bytes.getShort(a), Bytes.getShort(b));
+ case INT32:
+ return Integer.compare(Bytes.getInt(a), Bytes.getInt(b));
+ case INT64:
+ case TIMESTAMP:
+ return Long.compare(Bytes.getLong(a), Bytes.getLong(b));
+ case FLOAT:
+ return Float.compare(Bytes.getFloat(a), Bytes.getFloat(b));
+ case DOUBLE:
+ return Double.compare(Bytes.getDouble(a), Bytes.getDouble(b));
+ case STRING:
+ case BINARY:
+ return UnsignedBytes.lexicographicalComparator().compare(a, b);
+ default:
+ throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
+ }
+ }
+
+ /**
+ * Returns true if increment(a) == b.
+ * @param a the value which would be incremented
+ * @param b the target value
+ * @return true if increment(a) == b
+ */
+ private boolean areConsecutive(byte[] a, byte[] b) {
+ switch (column.getType().getDataType()) {
+ case BOOL: return false;
+ case INT8: {
+ byte m = Bytes.getByte(a);
+ byte n = Bytes.getByte(b);
+ return m < n && m + 1 == n;
+ }
+ case INT16: {
+ short m = Bytes.getShort(a);
+ short n = Bytes.getShort(b);
+ return m < n && m + 1 == n;
+ }
+ case INT32: {
+ int m = Bytes.getInt(a);
+ int n = Bytes.getInt(b);
+ return m < n && m + 1 == n;
+ }
+ case INT64:
+ case TIMESTAMP: {
+ long m = Bytes.getLong(a);
+ long n = Bytes.getLong(b);
+ return m < n && m + 1 == n;
+ }
+ case FLOAT: {
+ float m = Bytes.getFloat(a);
+ float n = Bytes.getFloat(b);
+ return m < n && Math.nextAfter(m, Float.POSITIVE_INFINITY) == n;
+ }
+ case DOUBLE: {
+ double m = Bytes.getDouble(a);
+ double n = Bytes.getDouble(b);
+ return m < n && Math.nextAfter(m, Double.POSITIVE_INFINITY) == n;
+ }
+ case STRING:
+ case BINARY: {
+ if (a.length + 1 != b.length || b[a.length] != 0) return false;
+ for (int i = 0; i < a.length; i++) {
+ if (a[i] != b[i]) return false;
+ }
+ return true;
+ }
+ default:
+ throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
+ }
+ }
+
+ /**
+ * Returns the maximum value for the integer type.
+ * @param type an integer type
+ * @return the maximum value
+ */
+ @VisibleForTesting
+ static long maxIntValue(Type type) {
+ switch (type) {
+ case INT8: return Byte.MAX_VALUE;
+ case INT16: return Short.MAX_VALUE;
+ case INT32: return Integer.MAX_VALUE;
+ case TIMESTAMP:
+ case INT64: return Long.MAX_VALUE;
+ default: throw new IllegalArgumentException("type must be an integer type");
+ }
+ }
+
+ /**
+ * Returns the minimum value for the integer type.
+ * @param type an integer type
+ * @return the minimum value
+ */
+ @VisibleForTesting
+ static long minIntValue(Type type) {
+ switch (type) {
+ case INT8: return Byte.MIN_VALUE;
+ case INT16: return Short.MIN_VALUE;
+ case INT32: return Integer.MIN_VALUE;
+ case TIMESTAMP:
+ case INT64: return Long.MIN_VALUE;
+ default: throw new IllegalArgumentException("type must be an integer type");
+ }
+ }
+
+
+ /**
+ * Checks that the column is one of the expected types.
+ * @param column the column being checked
+ * @param passedTypes the expected types (logical OR)
+ */
+ private static void checkColumn(ColumnSchema column, Type... passedTypes) {
+ for (Type type : passedTypes) {
+ if (column.getType().equals(type)) return;
+ }
+ throw new IllegalArgumentException(String.format("%s's type isn't %s, it's %s",
+ column.getName(), Arrays.toString(passedTypes),
+ column.getType().getName()));
+ }
+
+ /**
+ * Returns the string value of serialized value according to the type of column.
+ * @param value the value
+ * @return the text representation of the value
+ */
+ private String valueToString(byte[] value) {
+ switch (column.getType().getDataType()) {
+ case BOOL: return Boolean.toString(Bytes.getBoolean(value));
+ case INT8: return Byte.toString(Bytes.getByte(value));
+ case INT16: return Short.toString(Bytes.getShort(value));
+ case INT32: return Integer.toString(Bytes.getInt(value));
+ case INT64: return Long.toString(Bytes.getLong(value));
+ case TIMESTAMP: return RowResult.timestampToString(Bytes.getLong(value));
+ case FLOAT: return Float.toString(Bytes.getFloat(value));
+ case DOUBLE: return Double.toString(Bytes.getDouble(value));
+ case STRING: {
+ String v = Bytes.getString(value);
+ StringBuilder sb = new StringBuilder(2 + v.length());
+ sb.append('"');
+ sb.append(v);
+ sb.append('"');
+ return sb.toString();
+ }
+ case BINARY: return Bytes.hex(value);
+ default:
+ throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
+ }
+ }
+
+ @Override
+ public String toString() {
+ switch (type) {
+ case EQUALITY: return String.format("`%s` = %s", column.getName(), valueToString(lower));
+ case RANGE: {
+ if (lower == null) {
+ return String.format("`%s` < %s", column.getName(), valueToString(upper));
+ } else if (upper == null) {
+ return String.format("`%s` >= %s", column.getName(), valueToString(lower));
+ } else {
+ return String.format("`%s` >= %s AND `%s` < %s",
+ column.getName(), valueToString(lower),
+ column.getName(), valueToString(upper));
+ }
+ }
+ case IS_NOT_NULL: return String.format("`%s` IS NOT NULL", column.getName());
+ case NONE: return String.format("`%s` NONE", column.getName());
+ default: throw new IllegalArgumentException(String.format("unknown predicate type %s", type));
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ KuduPredicate that = (KuduPredicate) o;
+ return type == that.type &&
+ column.equals(that.column) &&
+ Arrays.equals(lower, that.lower) &&
+ Arrays.equals(upper, that.upper);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(type, column, Arrays.hashCode(lower), Arrays.hashCode(upper));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java
new file mode 100644
index 0000000..8ebd89d
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpc.java
@@ -0,0 +1,363 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.Message;
+import com.stumbleupon.async.Deferred;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.util.Pair;
+import org.kududb.util.Slice;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+
+/**
+ * Abstract base class for all RPC requests going out to Kudu.
+ * <p>
+ * Implementations of this class are <b>not</b> expected to be synchronized.
+ *
+ * <h1>A note on passing {@code byte} arrays in argument</h1>
+ * None of the method that receive a {@code byte[]} in argument will copy it.
+ * If you change the contents of any byte array you give to an instance of
+ * this class, you <em>may</em> affect the behavior of the request in an
+ * <strong>unpredictable</strong> way. If you need to change the byte array,
+ * {@link Object#clone() clone} it before giving it to this class. For those
+ * familiar with the term "defensive copy", we don't do it in order to avoid
+ * unnecessary memory copies when you know you won't be changing (or event
+ * holding a reference to) the byte array, which is frequently the case.
+ */
+@InterfaceAudience.Private
+public abstract class KuduRpc<R> {
+
+ // Service names.
+ protected static final String MASTER_SERVICE_NAME = "kudu.master.MasterService";
+ protected static final String TABLET_SERVER_SERVICE_NAME = "kudu.tserver.TabletServerService";
+
+ private static final Logger LOG = LoggerFactory.getLogger(KuduRpc.class);
+
+ /**
+ * Returns the partition key this RPC is for, or {@code null} if the RPC is
+ * not tablet specific.
+ * <p>
+ * <strong>DO NOT MODIFY THE CONTENTS OF THE RETURNED ARRAY.</strong>
+ */
+ byte[] partitionKey() {
+ return null;
+ }
+
+ /**
+ * The Deferred that will be invoked when this RPC completes or fails.
+ * In case of a successful completion, this Deferred's first callback
+ * will be invoked with an {@link Object} containing the de-serialized
+ * RPC response in argument.
+ * Once an RPC has been used, we create a new Deferred for it, in case
+ * the user wants to re-use it.
+ */
+ private Deferred<R> deferred;
+
+ private AsyncKuduClient.RemoteTablet tablet;
+
+ final KuduTable table;
+
+ final DeadlineTracker deadlineTracker;
+
+ protected long propagatedTimestamp = -1;
+ protected ExternalConsistencyMode externalConsistencyMode = CLIENT_PROPAGATED;
+
+ /**
+ * How many times have we retried this RPC?.
+ * Proper synchronization is required, although in practice most of the code
+ * that access this attribute will have a happens-before relationship with
+ * the rest of the code, due to other existing synchronization.
+ */
+ byte attempt; // package-private for TabletClient and AsyncKuduClient only.
+
+ /**
+ * Set by TabletClient when isRequestTracked returns true to identify this RPC in the sequence of
+ * RPCs sent by this client. Once it is set it should never change unless the RPC is reused.
+ */
+ long sequenceId = RequestTracker.NO_SEQ_NO;
+
+ KuduRpc(KuduTable table) {
+ this.table = table;
+ this.deadlineTracker = new DeadlineTracker();
+ }
+
+ /**
+ * To be implemented by the concrete sub-type.
+ *
+ * Notice that this method is package-private, so only classes within this
+ * package can use this as a base class.
+ */
+ abstract ChannelBuffer serialize(Message header);
+
+ /**
+ * Package private way of getting the name of the RPC service.
+ */
+ abstract String serviceName();
+
+ /**
+ * Package private way of getting the name of the RPC method.
+ */
+ abstract String method();
+
+ /**
+ * Returns the set of application-specific feature flags required to service the RPC.
+ * @return the feature flags required to complete the RPC
+ */
+ Collection<Integer> getRequiredFeatures() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * To be implemented by the concrete sub-type.
+ * This method is expected to de-serialize a response received for the
+ * current RPC.
+ *
+ * Notice that this method is package-private, so only classes within this
+ * package can use this as a base class.
+ *
+ * @param callResponse The call response from which to deserialize.
+ * @param tsUUID A string that contains the UUID of the server that answered the RPC.
+ * @return An Object of type R that will be sent to callback and an Object that will be an Error
+ * of type TabletServerErrorPB or MasterErrorPB that will be converted into an exception and
+ * sent to errback.
+ * @throws Exception An exception that will be sent to errback.
+ */
+ abstract Pair<R, Object> deserialize(CallResponse callResponse, String tsUUID) throws Exception;
+
+ /**
+ * Update the statistics information before this rpc is called back. This method should not throw
+ * any exception, including RuntimeException. This method does nothing by default.
+ *
+ * @param statistics object to update
+ * @param response of this rpc
+ */
+ void updateStatistics(Statistics statistics, R response){
+ // default do nothing
+ }
+
+ /**
+ * Sets the external consistency mode for this RPC.
+ * TODO make this cover most if not all RPCs (right now only scans and writes use this).
+ * @param externalConsistencyMode the mode to set
+ */
+ public void setExternalConsistencyMode(ExternalConsistencyMode externalConsistencyMode) {
+ this.externalConsistencyMode = externalConsistencyMode;
+ }
+
+ public ExternalConsistencyMode getExternalConsistencyMode() {
+ return this.externalConsistencyMode;
+ }
+
+ /**
+ * Sets the propagated timestamp for this RPC.
+ * @param propagatedTimestamp the timestamp to propagate
+ */
+ public void setPropagatedTimestamp(long propagatedTimestamp) {
+ this.propagatedTimestamp = propagatedTimestamp;
+ }
+
+ private void handleCallback(final Object result) {
+ final Deferred<R> d = deferred;
+ if (d == null) {
+ return;
+ }
+ deferred = null;
+ attempt = 0;
+ if (isRequestTracked()) {
+ table.getAsyncClient().getRequestTracker().rpcCompleted(sequenceId);
+ sequenceId = RequestTracker.NO_SEQ_NO;
+ }
+ deadlineTracker.reset();
+ d.callback(result);
+ }
+
+ /**
+ * Package private way of making an RPC complete by giving it its result.
+ * If this RPC has no {@link Deferred} associated to it, nothing will
+ * happen. This may happen if the RPC was already called back.
+ * <p>
+ * Once this call to this method completes, this object can be re-used to
+ * re-send the same RPC, provided that no other thread still believes this
+ * RPC to be in-flight (guaranteeing this may be hard in error cases).
+ */
+ final void callback(final R result) {
+ handleCallback(result);
+ }
+
+ /**
+ * Same as callback, except that it accepts an Exception.
+ */
+ final void errback(final Exception e) {
+ handleCallback(e);
+ }
+
+ /** Package private way of accessing / creating the Deferred of this RPC. */
+ final Deferred<R> getDeferred() {
+ if (deferred == null) {
+ deferred = new Deferred<R>();
+ }
+ return deferred;
+ }
+
+ AsyncKuduClient.RemoteTablet getTablet() {
+ return this.tablet;
+ }
+
+ void setTablet(AsyncKuduClient.RemoteTablet tablet) {
+ this.tablet = tablet;
+ }
+
+ public KuduTable getTable() {
+ return table;
+ }
+
+ void setTimeoutMillis(long timeout) {
+ deadlineTracker.setDeadline(timeout);
+ }
+
+ /**
+ * If this RPC needs to be tracked on the client and server-side. Some RPCs require exactly-once
+ * semantics which is enabled by tracking them.
+ * @return true if the request has to be tracked, else false
+ */
+ boolean isRequestTracked() {
+ return false;
+ }
+
+ long getSequenceId() {
+ return sequenceId;
+ }
+
+ void setSequenceId(long sequenceId) {
+ assert (this.sequenceId == RequestTracker.NO_SEQ_NO);
+ this.sequenceId = sequenceId;
+ }
+
+ public String toString() {
+
+ final StringBuilder buf = new StringBuilder();
+ buf.append("KuduRpc(method=");
+ buf.append(method());
+ buf.append(", tablet=");
+ if (tablet == null) {
+ buf.append("null");
+ } else {
+ buf.append(tablet.getTabletIdAsString());
+ }
+ buf.append(", attempt=").append(attempt);
+ buf.append(", ").append(deadlineTracker);
+ // Cheating a bit, we're not actually logging but we'll augment the information provided by
+ // this method if DEBUG is enabled.
+ if (LOG.isDebugEnabled()) {
+ buf.append(", ").append(deferred);
+ }
+ buf.append(')');
+ return buf.toString();
+ }
+
+ static void readProtobuf(final Slice slice,
+ final com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ final int length = slice.length();
+ final byte[] payload = slice.getRawArray();
+ final int offset = slice.getRawOffset();
+ try {
+ builder.mergeFrom(payload, offset, length);
+ if (!builder.isInitialized()) {
+ throw new RuntimeException("Could not deserialize the response," +
+ " incompatible RPC? Error is: " + builder.getInitializationErrorString());
+ }
+ } catch (InvalidProtocolBufferException e) {
+ throw new RuntimeException("Invalid RPC response: length=" + length +
+ ", payload=" + Bytes.pretty(payload));
+ }
+ }
+
+ static ChannelBuffer toChannelBuffer(Message header, Message pb) {
+ int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(header, pb);
+ byte[] buf = new byte[totalSize+4];
+ ChannelBuffer chanBuf = ChannelBuffers.wrappedBuffer(buf);
+ chanBuf.clear();
+ chanBuf.writeInt(totalSize);
+ final CodedOutputStream out = CodedOutputStream.newInstance(buf, 4, totalSize);
+ try {
+ out.writeRawVarint32(header.getSerializedSize());
+ header.writeTo(out);
+
+ out.writeRawVarint32(pb.getSerializedSize());
+ pb.writeTo(out);
+ out.checkNoSpaceLeft();
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot serialize the following message " + pb);
+ }
+ chanBuf.writerIndex(buf.length);
+ return chanBuf;
+ }
+
+ /**
+ * Upper bound on the size of a byte array we de-serialize.
+ * This is to prevent Kudu from OOM'ing us, should there be a bug or
+ * undetected corruption of an RPC on the network, which would turn a
+ * an innocuous RPC into something allocating a ton of memory.
+ * The Hadoop RPC protocol doesn't do any checksumming as they probably
+ * assumed that TCP checksums would be sufficient (they're not).
+ */
+ static final long MAX_BYTE_ARRAY_MASK =
+ 0xFFFFFFFFF0000000L; // => max = 256MB
+
+ /**
+ * Verifies that the given length looks like a reasonable array length.
+ * This method accepts 0 as a valid length.
+ * @param buf The buffer from which the length was read.
+ * @param length The length to validate.
+ * @throws IllegalArgumentException if the length is negative or
+ * suspiciously large.
+ */
+ static void checkArrayLength(final ChannelBuffer buf, final long length) {
+ // 2 checks in 1. If any of the high bits are set, we know the value is
+ // either too large, or is negative (if the most-significant bit is set).
+ if ((length & MAX_BYTE_ARRAY_MASK) != 0) {
+ if (length < 0) {
+ throw new IllegalArgumentException("Read negative byte array length: "
+ + length + " in buf=" + buf + '=' + Bytes.pretty(buf));
+ } else {
+ throw new IllegalArgumentException("Read byte array length that's too"
+ + " large: " + length + " > " + ~MAX_BYTE_ARRAY_MASK + " in buf="
+ + buf + '=' + Bytes.pretty(buf));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java
new file mode 100644
index 0000000..981e04a
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduRpcResponse.java
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+
+/**
+ * Base class for RPC responses.
+ */
+@InterfaceAudience.Private
+abstract class KuduRpcResponse {
+ private final long elapsedMillis;
+ private final String tsUUID;
+
+ /**
+ * Constructor with information common to all RPCs.
+ * @param elapsedMillis time in milliseconds since RPC creation to now
+ * @param tsUUID a string that contains the UUID of the server that answered the RPC
+ */
+ KuduRpcResponse(long elapsedMillis, String tsUUID) {
+ this.elapsedMillis = elapsedMillis;
+ this.tsUUID = tsUUID;
+ }
+
+ /**
+ * Get the number of milliseconds elapsed since the RPC was created up to the moment when this
+ * response was created.
+ * @return elapsed time in milliseconds
+ */
+ public long getElapsedMillis() {
+ return elapsedMillis;
+ }
+
+ /**
+ * Get the identifier of the tablet server that sent the response. May be
+ * {@code null} if the RPC failed before tablet location lookup succeeded.
+ * @return a string containing a UUID
+ */
+ public String getTsUUID() {
+ return tsUUID;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
new file mode 100644
index 0000000..382a4f0
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanToken.java
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.kududb.client;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+import org.kududb.ColumnSchema;
+import org.kududb.Common;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.Client.ScanTokenPB;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A scan token describes a partial scan of a Kudu table limited to a single
+ * contiguous physical location. Using the {@link KuduScanTokenBuilder}, clients can
+ * describe the desired scan, including predicates, bounds, timestamps, and
+ * caching, and receive back a collection of scan tokens.
+ *
+ * Each scan token may be separately turned into a scanner using
+ * {@link #intoScanner}, with each scanner responsible for a disjoint section
+ * of the table.
+ *
+ * Scan tokens may be serialized using the {@link #serialize} method and
+ * deserialized back into a scanner using the {@link #deserializeIntoScanner}
+ * method. This allows use cases such as generating scan tokens in the planner
+ * component of a query engine, then sending the tokens to execution nodes based
+ * on locality, and then instantiating the scanners on those nodes.
+ *
+ * Scan token locality information can be inspected using the {@link #getTablet}
+ * method.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class KuduScanToken implements Comparable<KuduScanToken> {
+ private final LocatedTablet tablet;
+ private final ScanTokenPB message;
+
+ private KuduScanToken(LocatedTablet tablet, ScanTokenPB message) {
+ this.tablet = tablet;
+ this.message = message;
+ }
+
+ /**
+ * Returns the tablet which the scanner created from this token will access.
+ * @return the located tablet
+ */
+ public LocatedTablet getTablet() {
+ return tablet;
+ }
+
+ /**
+ * Creates a {@link KuduScanner} from this scan token.
+ * @param client a Kudu client for the cluster
+ * @return a scanner for the scan token
+ */
+ public KuduScanner intoScanner(KuduClient client) throws Exception {
+ return pbIntoScanner(message, client);
+ }
+
+ /**
+ * Serializes this {@code KuduScanToken} into a byte array.
+ * @return the serialized scan token
+ * @throws IOException
+ */
+ public byte[] serialize() throws IOException {
+ byte[] buf = new byte[message.getSerializedSize()];
+ CodedOutputStream cos = CodedOutputStream.newInstance(buf);
+ message.writeTo(cos);
+ cos.flush();
+ return buf;
+ }
+
+ /**
+ * Deserializes a {@code KuduScanToken} into a {@link KuduScanner}.
+ * @param buf a byte array containing the serialized scan token.
+ * @param client a Kudu client for the cluster
+ * @return a scanner for the serialized scan token
+ * @throws Exception
+ */
+ public static KuduScanner deserializeIntoScanner(byte[] buf, KuduClient client) throws Exception {
+ return pbIntoScanner(ScanTokenPB.parseFrom(CodedInputStream.newInstance(buf)), client);
+ }
+
+ private static KuduScanner pbIntoScanner(ScanTokenPB message,
+ KuduClient client) throws Exception {
+ Preconditions.checkArgument(
+ !message.getFeatureFlagsList().contains(ScanTokenPB.Feature.Unknown),
+ "Scan token requires an unsupported feature. This Kudu client must be updated.");
+
+ KuduTable table = client.openTable(message.getTableName());
+ KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table);
+
+ List<Integer> columns = new ArrayList<>(message.getProjectedColumnsCount());
+ for (Common.ColumnSchemaPB column : message.getProjectedColumnsList()) {
+ int columnIdx = table.getSchema().getColumnIndex(column.getName());
+ ColumnSchema schema = table.getSchema().getColumnByIndex(columnIdx);
+ Preconditions.checkArgument(column.getType() == schema.getType().getDataType(),
+ String.format("Column types do not match for column %s",
+ column.getName()));
+ columns.add(columnIdx);
+ }
+ builder.setProjectedColumnIndexes(columns);
+
+ for (Common.ColumnPredicatePB pred : message.getColumnPredicatesList()) {
+ builder.addPredicate(KuduPredicate.fromPB(table.getSchema(), pred));
+ }
+
+ if (message.hasLowerBoundPrimaryKey()) {
+ builder.lowerBoundRaw(message.getLowerBoundPrimaryKey().toByteArray());
+ }
+ if (message.hasUpperBoundPrimaryKey()) {
+ builder.exclusiveUpperBoundRaw(message.getUpperBoundPrimaryKey().toByteArray());
+ }
+
+ if (message.hasLowerBoundPartitionKey()) {
+ builder.lowerBoundPartitionKeyRaw(message.getLowerBoundPartitionKey().toByteArray());
+ }
+ if (message.hasUpperBoundPartitionKey()) {
+ builder.exclusiveUpperBoundPartitionKeyRaw(message.getUpperBoundPartitionKey().toByteArray());
+ }
+
+ if (message.hasLimit()) {
+ builder.limit(message.getLimit());
+ }
+
+ if (message.hasFaultTolerant()) {
+ // TODO(KUDU-1040)
+ }
+
+ if (message.hasReadMode()) {
+ switch (message.getReadMode()) {
+ case READ_AT_SNAPSHOT: {
+ builder.readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT);
+ if (message.hasSnapTimestamp()) {
+ builder.snapshotTimestampRaw(message.getSnapTimestamp());
+ }
+ break;
+ }
+ case READ_LATEST: {
+ builder.readMode(AsyncKuduScanner.ReadMode.READ_LATEST);
+ break;
+ }
+ default: throw new IllegalArgumentException("unknown read mode");
+ }
+ }
+
+ if (message.hasPropagatedTimestamp()) {
+ // TODO (KUDU-1411)
+ }
+
+ if (message.hasCacheBlocks()) {
+ builder.cacheBlocks(message.getCacheBlocks());
+ }
+
+ return builder.build();
+ }
+
+ @Override
+ public int compareTo(KuduScanToken other) {
+ if (!message.getTableName().equals(other.message.getTableName())) {
+ throw new IllegalArgumentException("Scan tokens from different tables may not be compared");
+ }
+
+ return tablet.getPartition().compareTo(other.getTablet().getPartition());
+ }
+
+ /**
+ * Builds a sequence of scan tokens.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public static class KuduScanTokenBuilder
+ extends AbstractKuduScannerBuilder<KuduScanTokenBuilder, List<KuduScanToken>> {
+
+ private long timeout;
+
+ KuduScanTokenBuilder(AsyncKuduClient client, KuduTable table) {
+ super(client, table);
+ timeout = client.getDefaultOperationTimeoutMs();
+ }
+
+ /**
+ * Sets a timeout value to use when building the list of scan tokens. If
+ * unset, the client operation timeout will be used.
+ * @param timeoutMs the timeout in milliseconds.
+ */
+ public KuduScanTokenBuilder setTimeout(long timeoutMs) {
+ timeout = timeoutMs;
+ return this;
+ }
+
+ @Override
+ public List<KuduScanToken> build() {
+ if (lowerBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY ||
+ upperBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY) {
+ throw new IllegalArgumentException(
+ "Partition key bounds may not be set on KuduScanTokenBuilder");
+ }
+
+ // If the scan is short-circuitable, then return no tokens.
+ for (KuduPredicate predicate : predicates.values()) {
+ if (predicate.getType() == KuduPredicate.PredicateType.NONE) {
+ return ImmutableList.of();
+ }
+ }
+
+ Client.ScanTokenPB.Builder proto = Client.ScanTokenPB.newBuilder();
+
+ proto.setTableName(table.getName());
+
+ // Map the column names or indices to actual columns in the table schema.
+ // If the user did not set either projection, then scan all columns.
+ if (projectedColumnNames != null) {
+ for (String columnName : projectedColumnNames) {
+ ColumnSchema columnSchema = table.getSchema().getColumn(columnName);
+ Preconditions.checkArgument(columnSchema != null, "unknown column %s", columnName);
+ ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), columnSchema);
+ }
+ } else if (projectedColumnIndexes != null) {
+ for (int columnIdx : projectedColumnIndexes) {
+ ColumnSchema columnSchema = table.getSchema().getColumnByIndex(columnIdx);
+ Preconditions.checkArgument(columnSchema != null, "unknown column index %s", columnIdx);
+ ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), columnSchema);
+ }
+ } else {
+ for (ColumnSchema column : table.getSchema().getColumns()) {
+ ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), column);
+ }
+ }
+
+ for (KuduPredicate predicate : predicates.values()) {
+ proto.addColumnPredicates(predicate.toPB());
+ }
+
+ if (lowerBoundPrimaryKey != AsyncKuduClient.EMPTY_ARRAY && lowerBoundPrimaryKey.length > 0) {
+ proto.setLowerBoundPrimaryKey(ZeroCopyLiteralByteString.copyFrom(lowerBoundPrimaryKey));
+ }
+ if (upperBoundPrimaryKey != AsyncKuduClient.EMPTY_ARRAY && upperBoundPrimaryKey.length > 0) {
+ proto.setUpperBoundPrimaryKey(ZeroCopyLiteralByteString.copyFrom(upperBoundPrimaryKey));
+ }
+ if (lowerBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY &&
+ lowerBoundPartitionKey.length > 0) {
+ proto.setLowerBoundPartitionKey(ZeroCopyLiteralByteString.copyFrom(lowerBoundPartitionKey));
+ }
+ if (upperBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY &&
+ upperBoundPartitionKey.length > 0) {
+ proto.setUpperBoundPartitionKey(ZeroCopyLiteralByteString.copyFrom(upperBoundPartitionKey));
+ }
+
+ proto.setLimit(limit);
+ proto.setReadMode(readMode.pbVersion());
+
+ // If the last propagated timestamp is set send it with the scan.
+ if (table.getAsyncClient().getLastPropagatedTimestamp() != AsyncKuduClient.NO_TIMESTAMP) {
+ proto.setPropagatedTimestamp(client.getLastPropagatedTimestamp());
+ }
+
+ // If the mode is set to read on snapshot set the snapshot timestamp.
+ if (readMode == AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT &&
+ htTimestamp != AsyncKuduClient.NO_TIMESTAMP) {
+ proto.setSnapTimestamp(htTimestamp);
+ }
+
+ proto.setCacheBlocks(cacheBlocks);
+
+ try {
+ List<LocatedTablet> tablets;
+ if (table.getPartitionSchema().isSimpleRangePartitioning()) {
+ // TODO: replace this with proper partition pruning.
+ tablets = table.getTabletsLocations(
+ lowerBoundPrimaryKey.length == 0 ? null : lowerBoundPrimaryKey,
+ upperBoundPrimaryKey.length == 0 ? null : upperBoundPrimaryKey,
+ timeout);
+ } else {
+ tablets = table.getTabletsLocations(timeout);
+ }
+
+ List<KuduScanToken> tokens = new ArrayList<>(tablets.size());
+ for (LocatedTablet tablet : tablets) {
+ Client.ScanTokenPB.Builder builder = proto.clone();
+ builder.setLowerBoundPartitionKey(
+ ZeroCopyLiteralByteString.wrap(tablet.getPartition().partitionKeyStart));
+ builder.setUpperBoundPartitionKey(
+ ZeroCopyLiteralByteString.wrap(tablet.getPartition().partitionKeyEnd));
+ tokens.add(new KuduScanToken(tablet, builder.build()));
+ }
+ return tokens;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java
new file mode 100644
index 0000000..87db768
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduScanner.java
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.stumbleupon.async.Deferred;
+
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.AsyncKuduScanner.ReadMode;
+
+/**
+ * Synchronous version of {@link AsyncKuduScanner}. Offers the same API but with blocking methods.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduScanner {
+
+ private final AsyncKuduScanner asyncScanner;
+
+ KuduScanner(AsyncKuduScanner asyncScanner) {
+ this.asyncScanner = asyncScanner;
+ }
+
+ /**
+ * Tells if the last rpc returned that there might be more rows to scan.
+ * @return true if there might be more data to scan, else false
+ */
+ public boolean hasMoreRows() {
+ return asyncScanner.hasMoreRows();
+ }
+
+ /**
+ * Scans a number of rows.
+ * <p>
+ * Once this method returns {@code null} once (which indicates that this
+ * {@code Scanner} is done scanning), calling it again leads to an undefined
+ * behavior.
+ * @return a list of rows.
+ * @throws KuduException if anything went wrong
+ */
+ public RowResultIterator nextRows() throws KuduException {
+ Deferred<RowResultIterator> d = asyncScanner.nextRows();
+ try {
+ return d.join(asyncScanner.scanRequestTimeout);
+ } catch (Exception e) {
+ throw KuduException.transformException(e);
+ }
+ }
+
+ /**
+ * Closes this scanner (don't forget to call this when you're done with it!).
+ * <p>
+ * Closing a scanner already closed has no effect.
+ * @return a deferred object that indicates the completion of the request
+ * @throws KuduException if anything went wrong
+ */
+ public RowResultIterator close() throws KuduException {
+ Deferred<RowResultIterator> d = asyncScanner.close();
+ try {
+ return d.join(asyncScanner.scanRequestTimeout);
+ } catch (Exception e) {
+ throw KuduException.transformException(e);
+ }
+ }
+
+ /**
+ * Returns the maximum number of rows that this scanner was configured to return.
+ * @return a long representing the maximum number of rows that can be returned
+ */
+ public long getLimit() {
+ return asyncScanner.getLimit();
+ }
+
+ /**
+ * Returns if this scanner was configured to cache data blocks or not.
+ * @return true if this scanner will cache blocks, else else.
+ */
+ public boolean getCacheBlocks() {
+ return asyncScanner.getCacheBlocks();
+ }
+
+ /**
+ * Returns the maximum number of bytes returned by the scanner, on each batch.
+ * @return a long representing the maximum number of bytes that a scanner can receive at once
+ * from a tablet server
+ */
+ public long getBatchSizeBytes() {
+ return asyncScanner.getBatchSizeBytes();
+ }
+
+ /**
+ * Returns the ReadMode for this scanner.
+ * @return the configured read mode for this scanner
+ */
+ public ReadMode getReadMode() {
+ return asyncScanner.getReadMode();
+ }
+
+ /**
+ * Returns the projection schema of this scanner. If specific columns were
+ * not specified during scanner creation, the table schema is returned.
+ * @return the projection schema for this scanner
+ */
+ public Schema getProjectionSchema() {
+ return asyncScanner.getProjectionSchema();
+ }
+
+ /**
+ * A Builder class to build {@link KuduScanner}.
+ * Use {@link KuduClient#newScannerBuilder} in order to get a builder instance.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public static class KuduScannerBuilder
+ extends AbstractKuduScannerBuilder<KuduScannerBuilder, KuduScanner> {
+
+ KuduScannerBuilder(AsyncKuduClient client, KuduTable table) {
+ super(client, table);
+ }
+
+ /**
+ * Builds a {@link KuduScanner} using the passed configurations.
+ * @return a new {@link KuduScanner}
+ */
+ public KuduScanner build() {
+ return new KuduScanner(new AsyncKuduScanner(
+ client, table, projectedColumnNames, projectedColumnIndexes, readMode, orderMode,
+ scanRequestTimeout, predicates, limit, cacheBlocks,
+ prefetching, lowerBoundPrimaryKey, upperBoundPrimaryKey,
+ lowerBoundPartitionKey, upperBoundPartitionKey,
+ htTimestamp, batchSizeBytes));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java
new file mode 100644
index 0000000..61718b2
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduSession.java
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.stumbleupon.async.DeferredGroupException;
+import com.stumbleupon.async.TimeoutException;
+import org.kududb.annotations.*;
+
+import com.stumbleupon.async.Deferred;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Synchronous version of {@link AsyncKuduSession}.
+ * Offers the same API but with blocking methods.<p>
+ *
+ * This class is <b>not</b> thread-safe.<p>
+ *
+ * A major difference with {@link AsyncKuduSession} is that the time spent waiting on operations is
+ * defined by {@link #setTimeoutMillis(long)} which defaults to getting it from
+ * {@link KuduClient#getDefaultOperationTimeoutMs()}.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduSession implements SessionConfiguration {
+
+ public static final Logger LOG = LoggerFactory.getLogger(KuduSession.class);
+
+ private final AsyncKuduSession session;
+
+ KuduSession(AsyncKuduSession session) {
+ this.session = session;
+ }
+
+ /**
+ * Blocking call with a different behavior based on the flush mode. PleaseThrottleException is
+ * managed by this method and will not be thrown, unlike {@link AsyncKuduSession#apply}.
+ * <p>
+ * <ul>
+ * <li>AUTO_FLUSH_SYNC: the call returns when the operation is persisted,
+ * else it throws an exception.
+ * <li>AUTO_FLUSH_BACKGROUND: the call returns when the operation has been added to the buffer.
+ * This call should normally perform only fast in-memory operations but
+ * it may have to wait when the buffer is full and there's another buffer being flushed. Row
+ * errors can be checked by calling {@link #countPendingErrors()} and can be retrieved by calling
+ * {@link #getPendingErrors()}.
+ * <li>MANUAL_FLUSH: the call returns when the operation has been added to the buffer,
+ * else it throws a KuduException if the buffer is full.
+ * </ul>
+ *
+ * @param operation operation to apply
+ * @return an OperationResponse for the applied Operation
+ * @throws KuduException if anything went wrong
+ */
+ public OperationResponse apply(Operation operation) throws KuduException {
+ while (true) {
+ try {
+ Deferred<OperationResponse> d = session.apply(operation);
+ if (getFlushMode() == FlushMode.AUTO_FLUSH_SYNC) {
+ return d.join(getTimeoutMillis());
+ }
+ break;
+ } catch (PleaseThrottleException ex) {
+ try {
+ ex.getDeferred().join(getTimeoutMillis());
+ } catch (Exception e) {
+ // This is the error response from the buffer that was flushing,
+ // we can't do much with it at this point.
+ LOG.error("Previous batch had this exception", e);
+ }
+ } catch (Exception e) {
+ throw KuduException.transformException(e);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Blocking call that force flushes this session's buffers. Data is persisted when this call
+ * returns, else it will throw an exception.
+ * @return a list of OperationResponse, one per operation that was flushed
+ * @throws KuduException if anything went wrong
+ */
+ public List<OperationResponse> flush() throws KuduException {
+ try {
+ return session.flush().join(getTimeoutMillis());
+ } catch (Exception e) {
+ throw KuduException.transformException(e);
+ }
+ }
+
+ /**
+ * Blocking call that flushes the buffers (see {@link #flush()} and closes the sessions.
+ * @return List of OperationResponse, one per operation that was flushed
+ * @throws KuduException if anything went wrong
+ */
+ public List<OperationResponse> close() throws KuduException {
+ try {
+ return session.close().join(getTimeoutMillis());
+ } catch (Exception e) {
+ throw KuduException.transformException(e);
+ }
+ }
+
+ @Override
+ public FlushMode getFlushMode() {
+ return session.getFlushMode();
+ }
+
+ @Override
+ public void setFlushMode(AsyncKuduSession.FlushMode flushMode) {
+ session.setFlushMode(flushMode);
+ }
+
+ @Override
+ public void setMutationBufferSpace(int size) {
+ session.setMutationBufferSpace(size);
+ }
+
+ @Override
+ public void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage) {
+ session.setMutationBufferLowWatermark(mutationBufferLowWatermarkPercentage);
+ }
+
+ @Override
+ public void setFlushInterval(int interval) {
+ session.setFlushInterval(interval);
+ }
+
+ @Override
+ public long getTimeoutMillis() {
+ return session.getTimeoutMillis();
+ }
+
+ @Override
+ public void setTimeoutMillis(long timeout) {
+ session.setTimeoutMillis(timeout);
+ }
+
+ @Override
+ public boolean isClosed() {
+ return session.isClosed();
+ }
+
+ @Override
+ public boolean hasPendingOperations() {
+ return session.hasPendingOperations();
+ }
+
+ @Override
+ public void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode) {
+ session.setExternalConsistencyMode(consistencyMode);
+ }
+
+ @Override
+ public boolean isIgnoreAllDuplicateRows() {
+ return session.isIgnoreAllDuplicateRows();
+ }
+
+ @Override
+ public void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows) {
+ session.setIgnoreAllDuplicateRows(ignoreAllDuplicateRows);
+ }
+
+ @Override
+ public int countPendingErrors() {
+ return session.countPendingErrors();
+ }
+
+ @Override
+ public RowErrorsAndOverflowStatus getPendingErrors() {
+ return session.getPendingErrors();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java
new file mode 100644
index 0000000..7f59e47
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduTable.java
@@ -0,0 +1,203 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import com.stumbleupon.async.Deferred;
+
+import java.util.List;
+
+/**
+ * A KuduTable represents a table on a particular cluster. It holds the current
+ * schema of the table. Any given KuduTable instance belongs to a specific AsyncKuduClient
+ * instance.
+ *
+ * Upon construction, the table is looked up in the catalog (or catalog cache),
+ * and the schema fetched for introspection. The schema is not kept in sync with the master.
+ *
+ * This class is thread-safe.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduTable {
+
+ private final Schema schema;
+ private final PartitionSchema partitionSchema;
+ private final AsyncKuduClient client;
+ private final String name;
+ private final String tableId;
+
+ /**
+ * Package-private constructor, use {@link KuduClient#openTable(String)} to get an instance.
+ * @param client the client this instance belongs to
+ * @param name this table's name
+ * @param schema this table's schema
+ */
+ KuduTable(AsyncKuduClient client, String name, String tableId,
+ Schema schema, PartitionSchema partitionSchema) {
+ this.schema = schema;
+ this.partitionSchema = partitionSchema;
+ this.client = client;
+ this.name = name;
+ this.tableId = tableId;
+ }
+
+ /**
+ * Get this table's schema, as of the moment this instance was created.
+ * @return this table's schema
+ */
+ public Schema getSchema() {
+ return this.schema;
+ }
+
+ /**
+ * Gets the table's partition schema.
+ *
+ * This method is new, and not considered stable or suitable for public use.
+ *
+ * @return the table's partition schema.
+ */
+ @InterfaceAudience.LimitedPrivate("Impala")
+ @InterfaceStability.Unstable
+ public PartitionSchema getPartitionSchema() {
+ return partitionSchema;
+ }
+
+ /**
+ * Get this table's name.
+ * @return this table's name
+ */
+ public String getName() {
+ return this.name;
+ }
+
+ /**
+ * Get this table's unique identifier.
+ * @return this table's tableId
+ */
+ public String getTableId() {
+ return tableId;
+ }
+
+ /**
+ * Get the async client that created this instance.
+ * @return an async kudu client
+ */
+ public AsyncKuduClient getAsyncClient() {
+ return this.client;
+ }
+
+ /**
+ * Get a new insert configured with this table's schema. The returned object should not be reused.
+ * @return an insert with this table's schema
+ */
+ public Insert newInsert() {
+ return new Insert(this);
+ }
+
+ /**
+ * Get a new update configured with this table's schema. The returned object should not be reused.
+ * @return an update with this table's schema
+ */
+ public Update newUpdate() {
+ return new Update(this);
+ }
+
+ /**
+ * Get a new delete configured with this table's schema. The returned object should not be reused.
+ * @return a delete with this table's schema
+ */
+ public Delete newDelete() {
+ return new Delete(this);
+ }
+
+ /**
+ * Get a new upsert configured with this table's schema. The returned object should not be reused.
+ * @return an upsert with this table's schema
+ */
+ public Upsert newUpsert() {
+ return new Upsert(this);
+ }
+
+ /**
+ * Get all the tablets for this table. This may query the master multiple times if there
+ * are a lot of tablets.
+ * @param deadline deadline in milliseconds for this method to finish
+ * @return a list containing the metadata and locations for each of the tablets in the
+ * table
+ * @throws Exception
+ * @deprecated use the {@link KuduScanToken} API
+ */
+ @Deprecated
+ public List<LocatedTablet> getTabletsLocations(long deadline) throws Exception {
+ return getTabletsLocations(null, null, deadline);
+ }
+
+ /**
+ * Asynchronously get all the tablets for this table.
+ * @param deadline max time spent in milliseconds for the deferred result of this method to
+ * get called back, if deadline is reached, the deferred result will get erred back
+ * @return a {@link Deferred} object that yields a list containing the metadata and
+ * locations for each of the tablets in the table
+ * @deprecated use the {@link KuduScanToken} API
+ */
+ @Deprecated
+ public Deferred<List<LocatedTablet>> asyncGetTabletsLocations(long deadline) {
+ return asyncGetTabletsLocations(null, null, deadline);
+ }
+
+ /**
+ * Get all or some tablets for this table. This may query the master multiple times if there
+ * are a lot of tablets.
+ * This method blocks until it gets all the tablets.
+ * @param startKey where to start in the table, pass null to start at the beginning
+ * @param endKey where to stop in the table (exclusive), pass null to get all the tablets until
+ * the end of the table
+ * @param deadline deadline in milliseconds for this method to finish
+ * @return a list containing the metadata and locations for each of the tablets in the
+ * table
+ * @throws Exception
+ * @deprecated use the {@link KuduScanToken} API
+ */
+ @Deprecated
+ public List<LocatedTablet> getTabletsLocations(byte[] startKey,
+ byte[] endKey,
+ long deadline) throws Exception{
+ return client.syncLocateTable(this, startKey, endKey, deadline);
+ }
+
+ /**
+ * Asynchronously get all or some tablets for this table.
+ * @param startKey where to start in the table, pass null to start at the beginning
+ * @param endKey where to stop in the table (exclusive), pass null to get all the tablets until
+ * the end of the table
+ * @param deadline max time spent in milliseconds for the deferred result of this method to
+ * get called back, if deadline is reached, the deferred result will get erred back
+ * @return a {@link Deferred} object that yields a list containing the metadata and locations
+ * for each of the tablets in the table
+ * @deprecated use the {@link KuduScanToken} API
+ */
+ @Deprecated
+ public Deferred<List<LocatedTablet>> asyncGetTabletsLocations(byte[] startKey,
+ byte[] endKey,
+ long deadline) {
+ return client.locateTable(this, startKey, endKey, deadline);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java
new file mode 100644
index 0000000..78725f0
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesRequest.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.master.Master;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@InterfaceAudience.Private
+class ListTablesRequest extends KuduRpc<ListTablesResponse> {
+
+ private final String nameFilter;
+
+ ListTablesRequest(KuduTable masterTable, String nameFilter) {
+ super(masterTable);
+ this.nameFilter = nameFilter;
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ final Master.ListTablesRequestPB.Builder builder =
+ Master.ListTablesRequestPB.newBuilder();
+ if (nameFilter != null) {
+ builder.setNameFilter(nameFilter);
+ }
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return "ListTables";
+ }
+
+ @Override
+ Pair<ListTablesResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final Master.ListTablesResponsePB.Builder respBuilder =
+ Master.ListTablesResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), respBuilder);
+ int serversCount = respBuilder.getTablesCount();
+ List<String> tables = new ArrayList<String>(serversCount);
+ for (Master.ListTablesResponsePB.TableInfo info : respBuilder.getTablesList()) {
+ tables.add(info.getName());
+ }
+ ListTablesResponse response = new ListTablesResponse(deadlineTracker.getElapsedMillis(),
+ tsUUID, tables);
+ return new Pair<ListTablesResponse, Object>(
+ response, respBuilder.hasError() ? respBuilder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java
new file mode 100644
index 0000000..70daee2
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTablesResponse.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.util.List;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ListTablesResponse extends KuduRpcResponse {
+
+ private final List<String> tablesList;
+
+ ListTablesResponse(long ellapsedMillis, String tsUUID, List<String> tablesList) {
+ super(ellapsedMillis, tsUUID);
+ this.tablesList = tablesList;
+ }
+
+ /**
+ * Get the list of tables as specified in the request.
+ * @return a list of table names
+ */
+ public List<String> getTablesList() {
+ return tablesList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java
new file mode 100644
index 0000000..bf26626
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersRequest.java
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import static org.kududb.master.Master.*;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@InterfaceAudience.Private
+public class ListTabletServersRequest extends KuduRpc<ListTabletServersResponse> {
+
+ public ListTabletServersRequest(KuduTable masterTable) {
+ super(masterTable);
+ }
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ final ListTabletServersRequestPB.Builder builder =
+ ListTabletServersRequestPB.newBuilder();
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return "ListTabletServers";
+ }
+
+ @Override
+ Pair<ListTabletServersResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final ListTabletServersResponsePB.Builder respBuilder =
+ ListTabletServersResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), respBuilder);
+ int serversCount = respBuilder.getServersCount();
+ List<String> servers = new ArrayList<String>(serversCount);
+ for (ListTabletServersResponsePB.Entry entry : respBuilder.getServersList()) {
+ servers.add(entry.getRegistration().getRpcAddresses(0).getHost());
+ }
+ ListTabletServersResponse response = new ListTabletServersResponse(deadlineTracker
+ .getElapsedMillis(), tsUUID, serversCount, servers);
+ return new Pair<ListTabletServersResponse, Object>(
+ response, respBuilder.hasError() ? respBuilder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java
new file mode 100644
index 0000000..373a14d
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ListTabletServersResponse.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.util.List;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ListTabletServersResponse extends KuduRpcResponse {
+
+ private final int tabletServersCount;
+ private final List<String> tabletServersList;
+
+ /**
+ * @param ellapsedMillis Time in milliseconds since RPC creation to now.
+ * @param tabletServersCount How many tablet servers the master is reporting.
+ * @param tabletServersList List of tablet servers.
+ */
+ ListTabletServersResponse(long ellapsedMillis, String tsUUID,
+ int tabletServersCount, List<String> tabletServersList) {
+ super(ellapsedMillis, tsUUID);
+ this.tabletServersCount = tabletServersCount;
+ this.tabletServersList = tabletServersList;
+ }
+
+ /**
+ * Get the count of tablet servers as reported by the master.
+ * @return TS count.
+ */
+ public int getTabletServersCount() {
+ return tabletServersCount;
+ }
+
+ /**
+ * Get the list of tablet servers, as represented by their hostname.
+ * @return List of hostnames, one per TS.
+ */
+ public List<String> getTabletServersList() {
+ return tabletServersList;
+ }
+}
[16/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduPredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduPredicate.java b/java/kudu-client/src/main/java/org/kududb/client/KuduPredicate.java
deleted file mode 100644
index dea94c4..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduPredicate.java
+++ /dev/null
@@ -1,683 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.kududb.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.UnsignedBytes;
-import com.google.protobuf.ByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.util.Arrays;
-
-/**
- * A predicate which can be used to filter rows based on the value of a column.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduPredicate {
-
- /**
- * The predicate type.
- */
- @InterfaceAudience.Private
- enum PredicateType {
- /** A predicate which filters all rows. */
- NONE,
- /** A predicate which filters all rows not equal to a value. */
- EQUALITY,
- /** A predicate which filters all rows not in a range. */
- RANGE,
- /** A predicate which filters all null rows. */
- IS_NOT_NULL,
- }
-
- /**
- * The comparison operator of a predicate.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public enum ComparisonOp {
- GREATER,
- GREATER_EQUAL,
- EQUAL,
- LESS,
- LESS_EQUAL,
- }
-
- private final PredicateType type;
- private final ColumnSchema column;
-
- /**
- * The inclusive lower bound value if this is a Range predicate, or
- * the createEquality value if this is an Equality predicate.
- */
- private final byte[] lower;
-
- /** The exclusive upper bound value if this is a Range predicate. */
- private final byte[] upper;
-
- /**
- * Creates a new {@code KuduPredicate} on a boolean column.
- * @param column the column schema
- * @param op the comparison operation
- * @param value the value to compare against
- */
- public static KuduPredicate newComparisonPredicate(ColumnSchema column,
- ComparisonOp op,
- boolean value) {
- checkColumn(column, Type.BOOL);
- // Create the comparison predicate. Range predicates on boolean values can
- // always be converted to either an equality, an IS NOT NULL (filtering only
- // null values), or NONE (filtering all values).
- switch (op) {
- case GREATER: {
- // b > true -> b NONE
- // b > false -> b = true
- if (value) {
- return none(column);
- } else {
- return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(true), null);
- }
- }
- case GREATER_EQUAL: {
- // b >= true -> b = true
- // b >= false -> b IS NOT NULL
- if (value) {
- return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(true), null);
- } else {
- return newIsNotNullPredicate(column);
- }
- }
- case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column,
- Bytes.fromBoolean(value), null);
- case LESS: {
- // b < true -> b NONE
- // b < false -> b = true
- if (value) {
- return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(false), null);
- } else {
- return none(column);
- }
- }
- case LESS_EQUAL: {
- // b <= true -> b IS NOT NULL
- // b <= false -> b = false
- if (value) {
- return newIsNotNullPredicate(column);
- } else {
- return new KuduPredicate(PredicateType.EQUALITY, column, Bytes.fromBoolean(false), null);
- }
- }
- default: throw new RuntimeException("unknown comparison op");
- }
- }
-
- /**
- * Creates a new comparison predicate on an integer or timestamp column.
- * @param column the column schema
- * @param op the comparison operation
- * @param value the value to compare against
- */
- public static KuduPredicate newComparisonPredicate(ColumnSchema column,
- ComparisonOp op,
- long value) {
- checkColumn(column, Type.INT8, Type.INT16, Type.INT32, Type.INT64, Type.TIMESTAMP);
- Preconditions.checkArgument(value <= maxIntValue(column.getType()) &&
- value >= minIntValue(column.getType()),
- "integer value out of range for %s column: %s",
- column.getType(), value);
-
- if (op == ComparisonOp.LESS_EQUAL) {
- if (value == maxIntValue(column.getType())) {
- // If the value can't be incremented because it is at the top end of the
- // range, then substitute the predicate with an IS NOT NULL predicate.
- // This has the same effect as an inclusive upper bound on the maximum
- // value. If the column is not nullable then the IS NOT NULL predicate
- // is ignored.
- return newIsNotNullPredicate(column);
- }
- value += 1;
- } else if (op == ComparisonOp.GREATER) {
- if (value == maxIntValue(column.getType())) {
- return none(column);
- }
- value += 1;
- }
-
- byte[] bytes;
- switch (column.getType()) {
- case INT8: {
- bytes = new byte[] { (byte) value };
- break;
- }
- case INT16: {
- bytes = Bytes.fromShort((short) value);
- break;
- }
- case INT32: {
- bytes = Bytes.fromInt((int) value);
- break;
- }
- case INT64:
- case TIMESTAMP: {
- bytes = Bytes.fromLong(value);
- break;
- }
- default: throw new RuntimeException("already checked");
- }
- switch (op) {
- case GREATER:
- case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
- case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
- case LESS:
- case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
- default: throw new RuntimeException("unknown comparison op");
- }
- }
-
- /**
- * Creates a new comparison predicate on a float column.
- * @param column the column schema
- * @param op the comparison operation
- * @param value the value to compare against
- */
- public static KuduPredicate newComparisonPredicate(ColumnSchema column,
- ComparisonOp op,
- float value) {
- checkColumn(column, Type.FLOAT);
- if (op == ComparisonOp.LESS_EQUAL) {
- if (value == Float.POSITIVE_INFINITY) {
- return newIsNotNullPredicate(column);
- }
- value = Math.nextAfter(value, Float.POSITIVE_INFINITY);
- } else if (op == ComparisonOp.GREATER) {
- if (value == Float.POSITIVE_INFINITY) {
- return none(column);
- }
- value = Math.nextAfter(value, Float.POSITIVE_INFINITY);
- }
-
- byte[] bytes = Bytes.fromFloat(value);
- switch (op) {
- case GREATER:
- case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
- case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
- case LESS:
- case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
- default: throw new RuntimeException("unknown comparison op");
- }
- }
-
- /**
- * Creates a new comparison predicate on a double column.
- * @param column the column schema
- * @param op the comparison operation
- * @param value the value to compare against
- */
- public static KuduPredicate newComparisonPredicate(ColumnSchema column,
- ComparisonOp op,
- double value) {
- checkColumn(column, Type.DOUBLE);
- if (op == ComparisonOp.LESS_EQUAL) {
- if (value == Double.POSITIVE_INFINITY) {
- return newIsNotNullPredicate(column);
- }
- value = Math.nextAfter(value, Double.POSITIVE_INFINITY);
- } else if (op == ComparisonOp.GREATER) {
- if (value == Double.POSITIVE_INFINITY) {
- return none(column);
- }
- value = Math.nextAfter(value, Double.POSITIVE_INFINITY);
- }
-
- byte[] bytes = Bytes.fromDouble(value);
- switch (op) {
- case GREATER:
- case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
- case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
- case LESS:
- case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
- default: throw new RuntimeException("unknown comparison op");
- }
- }
-
- /**
- * Creates a new comparison predicate on a string column.
- * @param column the column schema
- * @param op the comparison operation
- * @param value the value to compare against
- */
- public static KuduPredicate newComparisonPredicate(ColumnSchema column,
- ComparisonOp op,
- String value) {
- checkColumn(column, Type.STRING);
-
- byte[] bytes = Bytes.fromString(value);
- if (op == ComparisonOp.LESS_EQUAL || op == ComparisonOp.GREATER) {
- bytes = Arrays.copyOf(bytes, bytes.length + 1);
- }
-
- switch (op) {
- case GREATER:
- case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, bytes, null);
- case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, bytes, null);
- case LESS:
- case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, bytes);
- default: throw new RuntimeException("unknown comparison op");
- }
- }
-
- /**
- * Creates a new comparison predicate on a binary column.
- * @param column the column schema
- * @param op the comparison operation
- * @param value the value to compare against
- */
- public static KuduPredicate newComparisonPredicate(ColumnSchema column,
- ComparisonOp op,
- byte[] value) {
- checkColumn(column, Type.BINARY);
-
- if (op == ComparisonOp.LESS_EQUAL || op == ComparisonOp.GREATER) {
- value = Arrays.copyOf(value, value.length + 1);
- }
-
- switch (op) {
- case GREATER:
- case GREATER_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, value, null);
- case EQUAL: return new KuduPredicate(PredicateType.EQUALITY, column, value, null);
- case LESS:
- case LESS_EQUAL: return new KuduPredicate(PredicateType.RANGE, column, null, value);
- default: throw new RuntimeException("unknown comparison op");
- }
- }
-
- /**
- * @param type the predicate type
- * @param column the column to which the predicate applies
- * @param lower the lower bound serialized value if this is a Range predicate,
- * or the equality value if this is an Equality predicate
- * @param upper the upper bound serialized value if this is an Equality predicate
- */
- @VisibleForTesting
- KuduPredicate(PredicateType type, ColumnSchema column, byte[] lower, byte[] upper) {
- this.type = type;
- this.column = column;
- this.lower = lower;
- this.upper = upper;
- }
-
- /**
- * Factory function for a {@code None} predicate.
- * @param column the column to which the predicate applies
- * @return a None predicate
- */
- @VisibleForTesting
- static KuduPredicate none(ColumnSchema column) {
- return new KuduPredicate(PredicateType.NONE, column, null, null);
- }
-
- /**
- * Factory function for an {@code IS NOT NULL} predicate.
- * @param column the column to which the predicate applies
- * @return a {@code IS NOT NULL} predicate
- */
- @VisibleForTesting
- static KuduPredicate newIsNotNullPredicate(ColumnSchema column) {
- return new KuduPredicate(PredicateType.IS_NOT_NULL, column, null, null);
- }
-
- /**
- * @return the type of this predicate
- */
- PredicateType getType() {
- return type;
- }
-
- /**
- * Merges another {@code ColumnPredicate} into this one, returning a new
- * {@code ColumnPredicate} which matches the logical intersection ({@code AND})
- * of the input predicates.
- * @param other the predicate to merge with this predicate
- * @return a new predicate that is the logical intersection
- */
- KuduPredicate merge(KuduPredicate other) {
- Preconditions.checkArgument(column.equals(other.column),
- "predicates from different columns may not be merged");
- if (type == PredicateType.NONE || other.type == PredicateType.NONE) {
- return none(column);
- }
-
- if (type == PredicateType.IS_NOT_NULL) {
- // NOT NULL is less selective than all other predicate types, so the
- // intersection of NOT NULL with any other predicate is just the other
- // predicate.
- //
- // Note: this will no longer be true when an IS NULL predicate type is
- // added.
- return other;
- }
-
- if (type == PredicateType.EQUALITY) {
- if (other.type == PredicateType.EQUALITY) {
- if (compare(lower, other.lower) != 0) {
- return none(this.column);
- } else {
- return this;
- }
- } else {
- if ((other.lower == null || compare(lower, other.lower) >= 0) &&
- (other.upper == null || compare(lower, other.upper) < 0)) {
- return this;
- } else {
- return none(this.column);
- }
- }
- } else {
- if (other.type == PredicateType.EQUALITY) {
- return other.merge(this);
- } else {
- byte[] newLower = other.lower == null ||
- (lower != null && compare(lower, other.lower) >= 0) ? lower : other.lower;
- byte[] newUpper = other.upper == null ||
- (upper != null && compare(upper, other.upper) <= 0) ? upper : other.upper;
- if (newLower != null && newUpper != null && compare(newLower, newUpper) >= 0) {
- return none(column);
- } else {
- if (newLower != null && newUpper != null && areConsecutive(newLower, newUpper)) {
- return new KuduPredicate(PredicateType.EQUALITY, column, newLower, null);
- } else {
- return new KuduPredicate(PredicateType.RANGE, column, newLower, newUpper);
- }
- }
- }
- }
- }
-
- /**
- * @return the schema of the predicate column
- */
- ColumnSchema getColumn() {
- return column;
- }
-
- /**
- * Convert the predicate to the protobuf representation.
- * @return the protobuf message for this predicate.
- */
- @InterfaceAudience.Private
- public Common.ColumnPredicatePB toPB() {
- Common.ColumnPredicatePB.Builder builder = Common.ColumnPredicatePB.newBuilder();
- builder.setColumn(column.getName());
-
- switch (type) {
- case EQUALITY: {
- builder.getEqualityBuilder().setValue(ByteString.copyFrom(lower));
- break;
- }
- case RANGE: {
- Common.ColumnPredicatePB.Range.Builder b = builder.getRangeBuilder();
- if (lower != null) {
- b.setLower(ByteString.copyFrom(lower));
- }
- if (upper != null) {
- b.setUpper(ByteString.copyFrom(upper));
- }
- break;
- }
- case IS_NOT_NULL: {
- builder.setIsNotNull(builder.getIsNotNullBuilder());
- break;
- }
- case NONE: throw new IllegalStateException(
- "can not convert None predicate to protobuf message");
- default: throw new IllegalArgumentException(
- String.format("unknown predicate type: %s", type));
- }
- return builder.build();
- }
-
- /**
- * Convert a column predicate protobuf message into a predicate.
- * @return a predicate
- */
- @InterfaceAudience.Private
- public static KuduPredicate fromPB(Schema schema, Common.ColumnPredicatePB pb) {
- ColumnSchema column = schema.getColumn(pb.getColumn());
- switch (pb.getPredicateCase()) {
- case EQUALITY: {
- return new KuduPredicate(PredicateType.EQUALITY, column,
- pb.getEquality().getValue().toByteArray(), null);
-
- }
- case RANGE: {
- Common.ColumnPredicatePB.Range range = pb.getRange();
- return new KuduPredicate(PredicateType.RANGE, column,
- range.hasLower() ? range.getLower().toByteArray() : null,
- range.hasUpper() ? range.getUpper().toByteArray() : null);
- }
- default: throw new IllegalArgumentException("unknown predicate type");
- }
- }
-
- /**
- * Compares two bounds based on the type of this predicate's column.
- * @param a the first serialized value
- * @param b the second serialized value
- * @return the comparison of the serialized values based on the column type
- */
- private int compare(byte[] a, byte[] b) {
- switch (column.getType().getDataType()) {
- case BOOL:
- return Boolean.compare(Bytes.getBoolean(a), Bytes.getBoolean(b));
- case INT8:
- return Byte.compare(Bytes.getByte(a), Bytes.getByte(b));
- case INT16:
- return Short.compare(Bytes.getShort(a), Bytes.getShort(b));
- case INT32:
- return Integer.compare(Bytes.getInt(a), Bytes.getInt(b));
- case INT64:
- case TIMESTAMP:
- return Long.compare(Bytes.getLong(a), Bytes.getLong(b));
- case FLOAT:
- return Float.compare(Bytes.getFloat(a), Bytes.getFloat(b));
- case DOUBLE:
- return Double.compare(Bytes.getDouble(a), Bytes.getDouble(b));
- case STRING:
- case BINARY:
- return UnsignedBytes.lexicographicalComparator().compare(a, b);
- default:
- throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
- }
- }
-
- /**
- * Returns true if increment(a) == b.
- * @param a the value which would be incremented
- * @param b the target value
- * @return true if increment(a) == b
- */
- private boolean areConsecutive(byte[] a, byte[] b) {
- switch (column.getType().getDataType()) {
- case BOOL: return false;
- case INT8: {
- byte m = Bytes.getByte(a);
- byte n = Bytes.getByte(b);
- return m < n && m + 1 == n;
- }
- case INT16: {
- short m = Bytes.getShort(a);
- short n = Bytes.getShort(b);
- return m < n && m + 1 == n;
- }
- case INT32: {
- int m = Bytes.getInt(a);
- int n = Bytes.getInt(b);
- return m < n && m + 1 == n;
- }
- case INT64:
- case TIMESTAMP: {
- long m = Bytes.getLong(a);
- long n = Bytes.getLong(b);
- return m < n && m + 1 == n;
- }
- case FLOAT: {
- float m = Bytes.getFloat(a);
- float n = Bytes.getFloat(b);
- return m < n && Math.nextAfter(m, Float.POSITIVE_INFINITY) == n;
- }
- case DOUBLE: {
- double m = Bytes.getDouble(a);
- double n = Bytes.getDouble(b);
- return m < n && Math.nextAfter(m, Double.POSITIVE_INFINITY) == n;
- }
- case STRING:
- case BINARY: {
- if (a.length + 1 != b.length || b[a.length] != 0) return false;
- for (int i = 0; i < a.length; i++) {
- if (a[i] != b[i]) return false;
- }
- return true;
- }
- default:
- throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
- }
- }
-
- /**
- * Returns the maximum value for the integer type.
- * @param type an integer type
- * @return the maximum value
- */
- @VisibleForTesting
- static long maxIntValue(Type type) {
- switch (type) {
- case INT8: return Byte.MAX_VALUE;
- case INT16: return Short.MAX_VALUE;
- case INT32: return Integer.MAX_VALUE;
- case TIMESTAMP:
- case INT64: return Long.MAX_VALUE;
- default: throw new IllegalArgumentException("type must be an integer type");
- }
- }
-
- /**
- * Returns the minimum value for the integer type.
- * @param type an integer type
- * @return the minimum value
- */
- @VisibleForTesting
- static long minIntValue(Type type) {
- switch (type) {
- case INT8: return Byte.MIN_VALUE;
- case INT16: return Short.MIN_VALUE;
- case INT32: return Integer.MIN_VALUE;
- case TIMESTAMP:
- case INT64: return Long.MIN_VALUE;
- default: throw new IllegalArgumentException("type must be an integer type");
- }
- }
-
-
- /**
- * Checks that the column is one of the expected types.
- * @param column the column being checked
- * @param passedTypes the expected types (logical OR)
- */
- private static void checkColumn(ColumnSchema column, Type... passedTypes) {
- for (Type type : passedTypes) {
- if (column.getType().equals(type)) return;
- }
- throw new IllegalArgumentException(String.format("%s's type isn't %s, it's %s",
- column.getName(), Arrays.toString(passedTypes),
- column.getType().getName()));
- }
-
- /**
- * Returns the string value of serialized value according to the type of column.
- * @param value the value
- * @return the text representation of the value
- */
- private String valueToString(byte[] value) {
- switch (column.getType().getDataType()) {
- case BOOL: return Boolean.toString(Bytes.getBoolean(value));
- case INT8: return Byte.toString(Bytes.getByte(value));
- case INT16: return Short.toString(Bytes.getShort(value));
- case INT32: return Integer.toString(Bytes.getInt(value));
- case INT64: return Long.toString(Bytes.getLong(value));
- case TIMESTAMP: return RowResult.timestampToString(Bytes.getLong(value));
- case FLOAT: return Float.toString(Bytes.getFloat(value));
- case DOUBLE: return Double.toString(Bytes.getDouble(value));
- case STRING: {
- String v = Bytes.getString(value);
- StringBuilder sb = new StringBuilder(2 + v.length());
- sb.append('"');
- sb.append(v);
- sb.append('"');
- return sb.toString();
- }
- case BINARY: return Bytes.hex(value);
- default:
- throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
- }
- }
-
- @Override
- public String toString() {
- switch (type) {
- case EQUALITY: return String.format("`%s` = %s", column.getName(), valueToString(lower));
- case RANGE: {
- if (lower == null) {
- return String.format("`%s` < %s", column.getName(), valueToString(upper));
- } else if (upper == null) {
- return String.format("`%s` >= %s", column.getName(), valueToString(lower));
- } else {
- return String.format("`%s` >= %s AND `%s` < %s",
- column.getName(), valueToString(lower),
- column.getName(), valueToString(upper));
- }
- }
- case IS_NOT_NULL: return String.format("`%s` IS NOT NULL", column.getName());
- case NONE: return String.format("`%s` NONE", column.getName());
- default: throw new IllegalArgumentException(String.format("unknown predicate type %s", type));
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- KuduPredicate that = (KuduPredicate) o;
- return type == that.type &&
- column.equals(that.column) &&
- Arrays.equals(lower, that.lower) &&
- Arrays.equals(upper, that.upper);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(type, column, Arrays.hashCode(lower), Arrays.hashCode(upper));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java b/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
deleted file mode 100644
index 8ebd89d..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
+++ /dev/null
@@ -1,363 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.Message;
-import com.stumbleupon.async.Deferred;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
-
-/**
- * Abstract base class for all RPC requests going out to Kudu.
- * <p>
- * Implementations of this class are <b>not</b> expected to be synchronized.
- *
- * <h1>A note on passing {@code byte} arrays in argument</h1>
- * None of the method that receive a {@code byte[]} in argument will copy it.
- * If you change the contents of any byte array you give to an instance of
- * this class, you <em>may</em> affect the behavior of the request in an
- * <strong>unpredictable</strong> way. If you need to change the byte array,
- * {@link Object#clone() clone} it before giving it to this class. For those
- * familiar with the term "defensive copy", we don't do it in order to avoid
- * unnecessary memory copies when you know you won't be changing (or event
- * holding a reference to) the byte array, which is frequently the case.
- */
-@InterfaceAudience.Private
-public abstract class KuduRpc<R> {
-
- // Service names.
- protected static final String MASTER_SERVICE_NAME = "kudu.master.MasterService";
- protected static final String TABLET_SERVER_SERVICE_NAME = "kudu.tserver.TabletServerService";
-
- private static final Logger LOG = LoggerFactory.getLogger(KuduRpc.class);
-
- /**
- * Returns the partition key this RPC is for, or {@code null} if the RPC is
- * not tablet specific.
- * <p>
- * <strong>DO NOT MODIFY THE CONTENTS OF THE RETURNED ARRAY.</strong>
- */
- byte[] partitionKey() {
- return null;
- }
-
- /**
- * The Deferred that will be invoked when this RPC completes or fails.
- * In case of a successful completion, this Deferred's first callback
- * will be invoked with an {@link Object} containing the de-serialized
- * RPC response in argument.
- * Once an RPC has been used, we create a new Deferred for it, in case
- * the user wants to re-use it.
- */
- private Deferred<R> deferred;
-
- private AsyncKuduClient.RemoteTablet tablet;
-
- final KuduTable table;
-
- final DeadlineTracker deadlineTracker;
-
- protected long propagatedTimestamp = -1;
- protected ExternalConsistencyMode externalConsistencyMode = CLIENT_PROPAGATED;
-
- /**
- * How many times have we retried this RPC?.
- * Proper synchronization is required, although in practice most of the code
- * that access this attribute will have a happens-before relationship with
- * the rest of the code, due to other existing synchronization.
- */
- byte attempt; // package-private for TabletClient and AsyncKuduClient only.
-
- /**
- * Set by TabletClient when isRequestTracked returns true to identify this RPC in the sequence of
- * RPCs sent by this client. Once it is set it should never change unless the RPC is reused.
- */
- long sequenceId = RequestTracker.NO_SEQ_NO;
-
- KuduRpc(KuduTable table) {
- this.table = table;
- this.deadlineTracker = new DeadlineTracker();
- }
-
- /**
- * To be implemented by the concrete sub-type.
- *
- * Notice that this method is package-private, so only classes within this
- * package can use this as a base class.
- */
- abstract ChannelBuffer serialize(Message header);
-
- /**
- * Package private way of getting the name of the RPC service.
- */
- abstract String serviceName();
-
- /**
- * Package private way of getting the name of the RPC method.
- */
- abstract String method();
-
- /**
- * Returns the set of application-specific feature flags required to service the RPC.
- * @return the feature flags required to complete the RPC
- */
- Collection<Integer> getRequiredFeatures() {
- return ImmutableList.of();
- }
-
- /**
- * To be implemented by the concrete sub-type.
- * This method is expected to de-serialize a response received for the
- * current RPC.
- *
- * Notice that this method is package-private, so only classes within this
- * package can use this as a base class.
- *
- * @param callResponse The call response from which to deserialize.
- * @param tsUUID A string that contains the UUID of the server that answered the RPC.
- * @return An Object of type R that will be sent to callback and an Object that will be an Error
- * of type TabletServerErrorPB or MasterErrorPB that will be converted into an exception and
- * sent to errback.
- * @throws Exception An exception that will be sent to errback.
- */
- abstract Pair<R, Object> deserialize(CallResponse callResponse, String tsUUID) throws Exception;
-
- /**
- * Update the statistics information before this rpc is called back. This method should not throw
- * any exception, including RuntimeException. This method does nothing by default.
- *
- * @param statistics object to update
- * @param response of this rpc
- */
- void updateStatistics(Statistics statistics, R response){
- // default do nothing
- }
-
- /**
- * Sets the external consistency mode for this RPC.
- * TODO make this cover most if not all RPCs (right now only scans and writes use this).
- * @param externalConsistencyMode the mode to set
- */
- public void setExternalConsistencyMode(ExternalConsistencyMode externalConsistencyMode) {
- this.externalConsistencyMode = externalConsistencyMode;
- }
-
- public ExternalConsistencyMode getExternalConsistencyMode() {
- return this.externalConsistencyMode;
- }
-
- /**
- * Sets the propagated timestamp for this RPC.
- * @param propagatedTimestamp the timestamp to propagate
- */
- public void setPropagatedTimestamp(long propagatedTimestamp) {
- this.propagatedTimestamp = propagatedTimestamp;
- }
-
- private void handleCallback(final Object result) {
- final Deferred<R> d = deferred;
- if (d == null) {
- return;
- }
- deferred = null;
- attempt = 0;
- if (isRequestTracked()) {
- table.getAsyncClient().getRequestTracker().rpcCompleted(sequenceId);
- sequenceId = RequestTracker.NO_SEQ_NO;
- }
- deadlineTracker.reset();
- d.callback(result);
- }
-
- /**
- * Package private way of making an RPC complete by giving it its result.
- * If this RPC has no {@link Deferred} associated to it, nothing will
- * happen. This may happen if the RPC was already called back.
- * <p>
- * Once this call to this method completes, this object can be re-used to
- * re-send the same RPC, provided that no other thread still believes this
- * RPC to be in-flight (guaranteeing this may be hard in error cases).
- */
- final void callback(final R result) {
- handleCallback(result);
- }
-
- /**
- * Same as callback, except that it accepts an Exception.
- */
- final void errback(final Exception e) {
- handleCallback(e);
- }
-
- /** Package private way of accessing / creating the Deferred of this RPC. */
- final Deferred<R> getDeferred() {
- if (deferred == null) {
- deferred = new Deferred<R>();
- }
- return deferred;
- }
-
- AsyncKuduClient.RemoteTablet getTablet() {
- return this.tablet;
- }
-
- void setTablet(AsyncKuduClient.RemoteTablet tablet) {
- this.tablet = tablet;
- }
-
- public KuduTable getTable() {
- return table;
- }
-
- void setTimeoutMillis(long timeout) {
- deadlineTracker.setDeadline(timeout);
- }
-
- /**
- * If this RPC needs to be tracked on the client and server-side. Some RPCs require exactly-once
- * semantics which is enabled by tracking them.
- * @return true if the request has to be tracked, else false
- */
- boolean isRequestTracked() {
- return false;
- }
-
- long getSequenceId() {
- return sequenceId;
- }
-
- void setSequenceId(long sequenceId) {
- assert (this.sequenceId == RequestTracker.NO_SEQ_NO);
- this.sequenceId = sequenceId;
- }
-
- public String toString() {
-
- final StringBuilder buf = new StringBuilder();
- buf.append("KuduRpc(method=");
- buf.append(method());
- buf.append(", tablet=");
- if (tablet == null) {
- buf.append("null");
- } else {
- buf.append(tablet.getTabletIdAsString());
- }
- buf.append(", attempt=").append(attempt);
- buf.append(", ").append(deadlineTracker);
- // Cheating a bit, we're not actually logging but we'll augment the information provided by
- // this method if DEBUG is enabled.
- if (LOG.isDebugEnabled()) {
- buf.append(", ").append(deferred);
- }
- buf.append(')');
- return buf.toString();
- }
-
- static void readProtobuf(final Slice slice,
- final com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- final int length = slice.length();
- final byte[] payload = slice.getRawArray();
- final int offset = slice.getRawOffset();
- try {
- builder.mergeFrom(payload, offset, length);
- if (!builder.isInitialized()) {
- throw new RuntimeException("Could not deserialize the response," +
- " incompatible RPC? Error is: " + builder.getInitializationErrorString());
- }
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException("Invalid RPC response: length=" + length +
- ", payload=" + Bytes.pretty(payload));
- }
- }
-
- static ChannelBuffer toChannelBuffer(Message header, Message pb) {
- int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(header, pb);
- byte[] buf = new byte[totalSize+4];
- ChannelBuffer chanBuf = ChannelBuffers.wrappedBuffer(buf);
- chanBuf.clear();
- chanBuf.writeInt(totalSize);
- final CodedOutputStream out = CodedOutputStream.newInstance(buf, 4, totalSize);
- try {
- out.writeRawVarint32(header.getSerializedSize());
- header.writeTo(out);
-
- out.writeRawVarint32(pb.getSerializedSize());
- pb.writeTo(out);
- out.checkNoSpaceLeft();
- } catch (IOException e) {
- throw new RuntimeException("Cannot serialize the following message " + pb);
- }
- chanBuf.writerIndex(buf.length);
- return chanBuf;
- }
-
- /**
- * Upper bound on the size of a byte array we de-serialize.
- * This is to prevent Kudu from OOM'ing us, should there be a bug or
- * undetected corruption of an RPC on the network, which would turn a
- * an innocuous RPC into something allocating a ton of memory.
- * The Hadoop RPC protocol doesn't do any checksumming as they probably
- * assumed that TCP checksums would be sufficient (they're not).
- */
- static final long MAX_BYTE_ARRAY_MASK =
- 0xFFFFFFFFF0000000L; // => max = 256MB
-
- /**
- * Verifies that the given length looks like a reasonable array length.
- * This method accepts 0 as a valid length.
- * @param buf The buffer from which the length was read.
- * @param length The length to validate.
- * @throws IllegalArgumentException if the length is negative or
- * suspiciously large.
- */
- static void checkArrayLength(final ChannelBuffer buf, final long length) {
- // 2 checks in 1. If any of the high bits are set, we know the value is
- // either too large, or is negative (if the most-significant bit is set).
- if ((length & MAX_BYTE_ARRAY_MASK) != 0) {
- if (length < 0) {
- throw new IllegalArgumentException("Read negative byte array length: "
- + length + " in buf=" + buf + '=' + Bytes.pretty(buf));
- } else {
- throw new IllegalArgumentException("Read byte array length that's too"
- + " large: " + length + " > " + ~MAX_BYTE_ARRAY_MASK + " in buf="
- + buf + '=' + Bytes.pretty(buf));
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduRpcResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduRpcResponse.java b/java/kudu-client/src/main/java/org/kududb/client/KuduRpcResponse.java
deleted file mode 100644
index 981e04a..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduRpcResponse.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-
-/**
- * Base class for RPC responses.
- */
-@InterfaceAudience.Private
-abstract class KuduRpcResponse {
- private final long elapsedMillis;
- private final String tsUUID;
-
- /**
- * Constructor with information common to all RPCs.
- * @param elapsedMillis time in milliseconds since RPC creation to now
- * @param tsUUID a string that contains the UUID of the server that answered the RPC
- */
- KuduRpcResponse(long elapsedMillis, String tsUUID) {
- this.elapsedMillis = elapsedMillis;
- this.tsUUID = tsUUID;
- }
-
- /**
- * Get the number of milliseconds elapsed since the RPC was created up to the moment when this
- * response was created.
- * @return elapsed time in milliseconds
- */
- public long getElapsedMillis() {
- return elapsedMillis;
- }
-
- /**
- * Get the identifier of the tablet server that sent the response. May be
- * {@code null} if the RPC failed before tablet location lookup succeeded.
- * @return a string containing a UUID
- */
- public String getTsUUID() {
- return tsUUID;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduScanToken.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduScanToken.java b/java/kudu-client/src/main/java/org/kududb/client/KuduScanToken.java
deleted file mode 100644
index 382a4f0..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduScanToken.java
+++ /dev/null
@@ -1,315 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.kududb.client;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Client.ScanTokenPB;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * A scan token describes a partial scan of a Kudu table limited to a single
- * contiguous physical location. Using the {@link KuduScanTokenBuilder}, clients can
- * describe the desired scan, including predicates, bounds, timestamps, and
- * caching, and receive back a collection of scan tokens.
- *
- * Each scan token may be separately turned into a scanner using
- * {@link #intoScanner}, with each scanner responsible for a disjoint section
- * of the table.
- *
- * Scan tokens may be serialized using the {@link #serialize} method and
- * deserialized back into a scanner using the {@link #deserializeIntoScanner}
- * method. This allows use cases such as generating scan tokens in the planner
- * component of a query engine, then sending the tokens to execution nodes based
- * on locality, and then instantiating the scanners on those nodes.
- *
- * Scan token locality information can be inspected using the {@link #getTablet}
- * method.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class KuduScanToken implements Comparable<KuduScanToken> {
- private final LocatedTablet tablet;
- private final ScanTokenPB message;
-
- private KuduScanToken(LocatedTablet tablet, ScanTokenPB message) {
- this.tablet = tablet;
- this.message = message;
- }
-
- /**
- * Returns the tablet which the scanner created from this token will access.
- * @return the located tablet
- */
- public LocatedTablet getTablet() {
- return tablet;
- }
-
- /**
- * Creates a {@link KuduScanner} from this scan token.
- * @param client a Kudu client for the cluster
- * @return a scanner for the scan token
- */
- public KuduScanner intoScanner(KuduClient client) throws Exception {
- return pbIntoScanner(message, client);
- }
-
- /**
- * Serializes this {@code KuduScanToken} into a byte array.
- * @return the serialized scan token
- * @throws IOException
- */
- public byte[] serialize() throws IOException {
- byte[] buf = new byte[message.getSerializedSize()];
- CodedOutputStream cos = CodedOutputStream.newInstance(buf);
- message.writeTo(cos);
- cos.flush();
- return buf;
- }
-
- /**
- * Deserializes a {@code KuduScanToken} into a {@link KuduScanner}.
- * @param buf a byte array containing the serialized scan token.
- * @param client a Kudu client for the cluster
- * @return a scanner for the serialized scan token
- * @throws Exception
- */
- public static KuduScanner deserializeIntoScanner(byte[] buf, KuduClient client) throws Exception {
- return pbIntoScanner(ScanTokenPB.parseFrom(CodedInputStream.newInstance(buf)), client);
- }
-
- private static KuduScanner pbIntoScanner(ScanTokenPB message,
- KuduClient client) throws Exception {
- Preconditions.checkArgument(
- !message.getFeatureFlagsList().contains(ScanTokenPB.Feature.Unknown),
- "Scan token requires an unsupported feature. This Kudu client must be updated.");
-
- KuduTable table = client.openTable(message.getTableName());
- KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table);
-
- List<Integer> columns = new ArrayList<>(message.getProjectedColumnsCount());
- for (Common.ColumnSchemaPB column : message.getProjectedColumnsList()) {
- int columnIdx = table.getSchema().getColumnIndex(column.getName());
- ColumnSchema schema = table.getSchema().getColumnByIndex(columnIdx);
- Preconditions.checkArgument(column.getType() == schema.getType().getDataType(),
- String.format("Column types do not match for column %s",
- column.getName()));
- columns.add(columnIdx);
- }
- builder.setProjectedColumnIndexes(columns);
-
- for (Common.ColumnPredicatePB pred : message.getColumnPredicatesList()) {
- builder.addPredicate(KuduPredicate.fromPB(table.getSchema(), pred));
- }
-
- if (message.hasLowerBoundPrimaryKey()) {
- builder.lowerBoundRaw(message.getLowerBoundPrimaryKey().toByteArray());
- }
- if (message.hasUpperBoundPrimaryKey()) {
- builder.exclusiveUpperBoundRaw(message.getUpperBoundPrimaryKey().toByteArray());
- }
-
- if (message.hasLowerBoundPartitionKey()) {
- builder.lowerBoundPartitionKeyRaw(message.getLowerBoundPartitionKey().toByteArray());
- }
- if (message.hasUpperBoundPartitionKey()) {
- builder.exclusiveUpperBoundPartitionKeyRaw(message.getUpperBoundPartitionKey().toByteArray());
- }
-
- if (message.hasLimit()) {
- builder.limit(message.getLimit());
- }
-
- if (message.hasFaultTolerant()) {
- // TODO(KUDU-1040)
- }
-
- if (message.hasReadMode()) {
- switch (message.getReadMode()) {
- case READ_AT_SNAPSHOT: {
- builder.readMode(AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT);
- if (message.hasSnapTimestamp()) {
- builder.snapshotTimestampRaw(message.getSnapTimestamp());
- }
- break;
- }
- case READ_LATEST: {
- builder.readMode(AsyncKuduScanner.ReadMode.READ_LATEST);
- break;
- }
- default: throw new IllegalArgumentException("unknown read mode");
- }
- }
-
- if (message.hasPropagatedTimestamp()) {
- // TODO (KUDU-1411)
- }
-
- if (message.hasCacheBlocks()) {
- builder.cacheBlocks(message.getCacheBlocks());
- }
-
- return builder.build();
- }
-
- @Override
- public int compareTo(KuduScanToken other) {
- if (!message.getTableName().equals(other.message.getTableName())) {
- throw new IllegalArgumentException("Scan tokens from different tables may not be compared");
- }
-
- return tablet.getPartition().compareTo(other.getTablet().getPartition());
- }
-
- /**
- * Builds a sequence of scan tokens.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Unstable
- public static class KuduScanTokenBuilder
- extends AbstractKuduScannerBuilder<KuduScanTokenBuilder, List<KuduScanToken>> {
-
- private long timeout;
-
- KuduScanTokenBuilder(AsyncKuduClient client, KuduTable table) {
- super(client, table);
- timeout = client.getDefaultOperationTimeoutMs();
- }
-
- /**
- * Sets a timeout value to use when building the list of scan tokens. If
- * unset, the client operation timeout will be used.
- * @param timeoutMs the timeout in milliseconds.
- */
- public KuduScanTokenBuilder setTimeout(long timeoutMs) {
- timeout = timeoutMs;
- return this;
- }
-
- @Override
- public List<KuduScanToken> build() {
- if (lowerBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY ||
- upperBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY) {
- throw new IllegalArgumentException(
- "Partition key bounds may not be set on KuduScanTokenBuilder");
- }
-
- // If the scan is short-circuitable, then return no tokens.
- for (KuduPredicate predicate : predicates.values()) {
- if (predicate.getType() == KuduPredicate.PredicateType.NONE) {
- return ImmutableList.of();
- }
- }
-
- Client.ScanTokenPB.Builder proto = Client.ScanTokenPB.newBuilder();
-
- proto.setTableName(table.getName());
-
- // Map the column names or indices to actual columns in the table schema.
- // If the user did not set either projection, then scan all columns.
- if (projectedColumnNames != null) {
- for (String columnName : projectedColumnNames) {
- ColumnSchema columnSchema = table.getSchema().getColumn(columnName);
- Preconditions.checkArgument(columnSchema != null, "unknown column %s", columnName);
- ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), columnSchema);
- }
- } else if (projectedColumnIndexes != null) {
- for (int columnIdx : projectedColumnIndexes) {
- ColumnSchema columnSchema = table.getSchema().getColumnByIndex(columnIdx);
- Preconditions.checkArgument(columnSchema != null, "unknown column index %s", columnIdx);
- ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), columnSchema);
- }
- } else {
- for (ColumnSchema column : table.getSchema().getColumns()) {
- ProtobufHelper.columnToPb(proto.addProjectedColumnsBuilder(), column);
- }
- }
-
- for (KuduPredicate predicate : predicates.values()) {
- proto.addColumnPredicates(predicate.toPB());
- }
-
- if (lowerBoundPrimaryKey != AsyncKuduClient.EMPTY_ARRAY && lowerBoundPrimaryKey.length > 0) {
- proto.setLowerBoundPrimaryKey(ZeroCopyLiteralByteString.copyFrom(lowerBoundPrimaryKey));
- }
- if (upperBoundPrimaryKey != AsyncKuduClient.EMPTY_ARRAY && upperBoundPrimaryKey.length > 0) {
- proto.setUpperBoundPrimaryKey(ZeroCopyLiteralByteString.copyFrom(upperBoundPrimaryKey));
- }
- if (lowerBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY &&
- lowerBoundPartitionKey.length > 0) {
- proto.setLowerBoundPartitionKey(ZeroCopyLiteralByteString.copyFrom(lowerBoundPartitionKey));
- }
- if (upperBoundPartitionKey != AsyncKuduClient.EMPTY_ARRAY &&
- upperBoundPartitionKey.length > 0) {
- proto.setUpperBoundPartitionKey(ZeroCopyLiteralByteString.copyFrom(upperBoundPartitionKey));
- }
-
- proto.setLimit(limit);
- proto.setReadMode(readMode.pbVersion());
-
- // If the last propagated timestamp is set send it with the scan.
- if (table.getAsyncClient().getLastPropagatedTimestamp() != AsyncKuduClient.NO_TIMESTAMP) {
- proto.setPropagatedTimestamp(client.getLastPropagatedTimestamp());
- }
-
- // If the mode is set to read on snapshot set the snapshot timestamp.
- if (readMode == AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT &&
- htTimestamp != AsyncKuduClient.NO_TIMESTAMP) {
- proto.setSnapTimestamp(htTimestamp);
- }
-
- proto.setCacheBlocks(cacheBlocks);
-
- try {
- List<LocatedTablet> tablets;
- if (table.getPartitionSchema().isSimpleRangePartitioning()) {
- // TODO: replace this with proper partition pruning.
- tablets = table.getTabletsLocations(
- lowerBoundPrimaryKey.length == 0 ? null : lowerBoundPrimaryKey,
- upperBoundPrimaryKey.length == 0 ? null : upperBoundPrimaryKey,
- timeout);
- } else {
- tablets = table.getTabletsLocations(timeout);
- }
-
- List<KuduScanToken> tokens = new ArrayList<>(tablets.size());
- for (LocatedTablet tablet : tablets) {
- Client.ScanTokenPB.Builder builder = proto.clone();
- builder.setLowerBoundPartitionKey(
- ZeroCopyLiteralByteString.wrap(tablet.getPartition().partitionKeyStart));
- builder.setUpperBoundPartitionKey(
- ZeroCopyLiteralByteString.wrap(tablet.getPartition().partitionKeyEnd));
- tokens.add(new KuduScanToken(tablet, builder.build()));
- }
- return tokens;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduScanner.java b/java/kudu-client/src/main/java/org/kududb/client/KuduScanner.java
deleted file mode 100644
index 87db768..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduScanner.java
+++ /dev/null
@@ -1,149 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.stumbleupon.async.Deferred;
-
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduScanner.ReadMode;
-
-/**
- * Synchronous version of {@link AsyncKuduScanner}. Offers the same API but with blocking methods.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduScanner {
-
- private final AsyncKuduScanner asyncScanner;
-
- KuduScanner(AsyncKuduScanner asyncScanner) {
- this.asyncScanner = asyncScanner;
- }
-
- /**
- * Tells if the last rpc returned that there might be more rows to scan.
- * @return true if there might be more data to scan, else false
- */
- public boolean hasMoreRows() {
- return asyncScanner.hasMoreRows();
- }
-
- /**
- * Scans a number of rows.
- * <p>
- * Once this method returns {@code null} once (which indicates that this
- * {@code Scanner} is done scanning), calling it again leads to an undefined
- * behavior.
- * @return a list of rows.
- * @throws KuduException if anything went wrong
- */
- public RowResultIterator nextRows() throws KuduException {
- Deferred<RowResultIterator> d = asyncScanner.nextRows();
- try {
- return d.join(asyncScanner.scanRequestTimeout);
- } catch (Exception e) {
- throw KuduException.transformException(e);
- }
- }
-
- /**
- * Closes this scanner (don't forget to call this when you're done with it!).
- * <p>
- * Closing a scanner already closed has no effect.
- * @return a deferred object that indicates the completion of the request
- * @throws KuduException if anything went wrong
- */
- public RowResultIterator close() throws KuduException {
- Deferred<RowResultIterator> d = asyncScanner.close();
- try {
- return d.join(asyncScanner.scanRequestTimeout);
- } catch (Exception e) {
- throw KuduException.transformException(e);
- }
- }
-
- /**
- * Returns the maximum number of rows that this scanner was configured to return.
- * @return a long representing the maximum number of rows that can be returned
- */
- public long getLimit() {
- return asyncScanner.getLimit();
- }
-
- /**
- * Returns if this scanner was configured to cache data blocks or not.
- * @return true if this scanner will cache blocks, else else.
- */
- public boolean getCacheBlocks() {
- return asyncScanner.getCacheBlocks();
- }
-
- /**
- * Returns the maximum number of bytes returned by the scanner, on each batch.
- * @return a long representing the maximum number of bytes that a scanner can receive at once
- * from a tablet server
- */
- public long getBatchSizeBytes() {
- return asyncScanner.getBatchSizeBytes();
- }
-
- /**
- * Returns the ReadMode for this scanner.
- * @return the configured read mode for this scanner
- */
- public ReadMode getReadMode() {
- return asyncScanner.getReadMode();
- }
-
- /**
- * Returns the projection schema of this scanner. If specific columns were
- * not specified during scanner creation, the table schema is returned.
- * @return the projection schema for this scanner
- */
- public Schema getProjectionSchema() {
- return asyncScanner.getProjectionSchema();
- }
-
- /**
- * A Builder class to build {@link KuduScanner}.
- * Use {@link KuduClient#newScannerBuilder} in order to get a builder instance.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public static class KuduScannerBuilder
- extends AbstractKuduScannerBuilder<KuduScannerBuilder, KuduScanner> {
-
- KuduScannerBuilder(AsyncKuduClient client, KuduTable table) {
- super(client, table);
- }
-
- /**
- * Builds a {@link KuduScanner} using the passed configurations.
- * @return a new {@link KuduScanner}
- */
- public KuduScanner build() {
- return new KuduScanner(new AsyncKuduScanner(
- client, table, projectedColumnNames, projectedColumnIndexes, readMode, orderMode,
- scanRequestTimeout, predicates, limit, cacheBlocks,
- prefetching, lowerBoundPrimaryKey, upperBoundPrimaryKey,
- lowerBoundPartitionKey, upperBoundPartitionKey,
- htTimestamp, batchSizeBytes));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java b/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java
deleted file mode 100644
index 61718b2..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduSession.java
+++ /dev/null
@@ -1,190 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.stumbleupon.async.DeferredGroupException;
-import com.stumbleupon.async.TimeoutException;
-import org.kududb.annotations.*;
-
-import com.stumbleupon.async.Deferred;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Synchronous version of {@link AsyncKuduSession}.
- * Offers the same API but with blocking methods.<p>
- *
- * This class is <b>not</b> thread-safe.<p>
- *
- * A major difference with {@link AsyncKuduSession} is that the time spent waiting on operations is
- * defined by {@link #setTimeoutMillis(long)} which defaults to getting it from
- * {@link KuduClient#getDefaultOperationTimeoutMs()}.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduSession implements SessionConfiguration {
-
- public static final Logger LOG = LoggerFactory.getLogger(KuduSession.class);
-
- private final AsyncKuduSession session;
-
- KuduSession(AsyncKuduSession session) {
- this.session = session;
- }
-
- /**
- * Blocking call with a different behavior based on the flush mode. PleaseThrottleException is
- * managed by this method and will not be thrown, unlike {@link AsyncKuduSession#apply}.
- * <p>
- * <ul>
- * <li>AUTO_FLUSH_SYNC: the call returns when the operation is persisted,
- * else it throws an exception.
- * <li>AUTO_FLUSH_BACKGROUND: the call returns when the operation has been added to the buffer.
- * This call should normally perform only fast in-memory operations but
- * it may have to wait when the buffer is full and there's another buffer being flushed. Row
- * errors can be checked by calling {@link #countPendingErrors()} and can be retrieved by calling
- * {@link #getPendingErrors()}.
- * <li>MANUAL_FLUSH: the call returns when the operation has been added to the buffer,
- * else it throws a KuduException if the buffer is full.
- * </ul>
- *
- * @param operation operation to apply
- * @return an OperationResponse for the applied Operation
- * @throws KuduException if anything went wrong
- */
- public OperationResponse apply(Operation operation) throws KuduException {
- while (true) {
- try {
- Deferred<OperationResponse> d = session.apply(operation);
- if (getFlushMode() == FlushMode.AUTO_FLUSH_SYNC) {
- return d.join(getTimeoutMillis());
- }
- break;
- } catch (PleaseThrottleException ex) {
- try {
- ex.getDeferred().join(getTimeoutMillis());
- } catch (Exception e) {
- // This is the error response from the buffer that was flushing,
- // we can't do much with it at this point.
- LOG.error("Previous batch had this exception", e);
- }
- } catch (Exception e) {
- throw KuduException.transformException(e);
- }
- }
- return null;
- }
-
- /**
- * Blocking call that force flushes this session's buffers. Data is persisted when this call
- * returns, else it will throw an exception.
- * @return a list of OperationResponse, one per operation that was flushed
- * @throws KuduException if anything went wrong
- */
- public List<OperationResponse> flush() throws KuduException {
- try {
- return session.flush().join(getTimeoutMillis());
- } catch (Exception e) {
- throw KuduException.transformException(e);
- }
- }
-
- /**
- * Blocking call that flushes the buffers (see {@link #flush()} and closes the sessions.
- * @return List of OperationResponse, one per operation that was flushed
- * @throws KuduException if anything went wrong
- */
- public List<OperationResponse> close() throws KuduException {
- try {
- return session.close().join(getTimeoutMillis());
- } catch (Exception e) {
- throw KuduException.transformException(e);
- }
- }
-
- @Override
- public FlushMode getFlushMode() {
- return session.getFlushMode();
- }
-
- @Override
- public void setFlushMode(AsyncKuduSession.FlushMode flushMode) {
- session.setFlushMode(flushMode);
- }
-
- @Override
- public void setMutationBufferSpace(int size) {
- session.setMutationBufferSpace(size);
- }
-
- @Override
- public void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage) {
- session.setMutationBufferLowWatermark(mutationBufferLowWatermarkPercentage);
- }
-
- @Override
- public void setFlushInterval(int interval) {
- session.setFlushInterval(interval);
- }
-
- @Override
- public long getTimeoutMillis() {
- return session.getTimeoutMillis();
- }
-
- @Override
- public void setTimeoutMillis(long timeout) {
- session.setTimeoutMillis(timeout);
- }
-
- @Override
- public boolean isClosed() {
- return session.isClosed();
- }
-
- @Override
- public boolean hasPendingOperations() {
- return session.hasPendingOperations();
- }
-
- @Override
- public void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode) {
- session.setExternalConsistencyMode(consistencyMode);
- }
-
- @Override
- public boolean isIgnoreAllDuplicateRows() {
- return session.isIgnoreAllDuplicateRows();
- }
-
- @Override
- public void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows) {
- session.setIgnoreAllDuplicateRows(ignoreAllDuplicateRows);
- }
-
- @Override
- public int countPendingErrors() {
- return session.countPendingErrors();
- }
-
- @Override
- public RowErrorsAndOverflowStatus getPendingErrors() {
- return session.getPendingErrors();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java b/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java
deleted file mode 100644
index 7f59e47..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/KuduTable.java
+++ /dev/null
@@ -1,203 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import com.stumbleupon.async.Deferred;
-
-import java.util.List;
-
-/**
- * A KuduTable represents a table on a particular cluster. It holds the current
- * schema of the table. Any given KuduTable instance belongs to a specific AsyncKuduClient
- * instance.
- *
- * Upon construction, the table is looked up in the catalog (or catalog cache),
- * and the schema fetched for introspection. The schema is not kept in sync with the master.
- *
- * This class is thread-safe.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduTable {
-
- private final Schema schema;
- private final PartitionSchema partitionSchema;
- private final AsyncKuduClient client;
- private final String name;
- private final String tableId;
-
- /**
- * Package-private constructor, use {@link KuduClient#openTable(String)} to get an instance.
- * @param client the client this instance belongs to
- * @param name this table's name
- * @param schema this table's schema
- */
- KuduTable(AsyncKuduClient client, String name, String tableId,
- Schema schema, PartitionSchema partitionSchema) {
- this.schema = schema;
- this.partitionSchema = partitionSchema;
- this.client = client;
- this.name = name;
- this.tableId = tableId;
- }
-
- /**
- * Get this table's schema, as of the moment this instance was created.
- * @return this table's schema
- */
- public Schema getSchema() {
- return this.schema;
- }
-
- /**
- * Gets the table's partition schema.
- *
- * This method is new, and not considered stable or suitable for public use.
- *
- * @return the table's partition schema.
- */
- @InterfaceAudience.LimitedPrivate("Impala")
- @InterfaceStability.Unstable
- public PartitionSchema getPartitionSchema() {
- return partitionSchema;
- }
-
- /**
- * Get this table's name.
- * @return this table's name
- */
- public String getName() {
- return this.name;
- }
-
- /**
- * Get this table's unique identifier.
- * @return this table's tableId
- */
- public String getTableId() {
- return tableId;
- }
-
- /**
- * Get the async client that created this instance.
- * @return an async kudu client
- */
- public AsyncKuduClient getAsyncClient() {
- return this.client;
- }
-
- /**
- * Get a new insert configured with this table's schema. The returned object should not be reused.
- * @return an insert with this table's schema
- */
- public Insert newInsert() {
- return new Insert(this);
- }
-
- /**
- * Get a new update configured with this table's schema. The returned object should not be reused.
- * @return an update with this table's schema
- */
- public Update newUpdate() {
- return new Update(this);
- }
-
- /**
- * Get a new delete configured with this table's schema. The returned object should not be reused.
- * @return a delete with this table's schema
- */
- public Delete newDelete() {
- return new Delete(this);
- }
-
- /**
- * Get a new upsert configured with this table's schema. The returned object should not be reused.
- * @return an upsert with this table's schema
- */
- public Upsert newUpsert() {
- return new Upsert(this);
- }
-
- /**
- * Get all the tablets for this table. This may query the master multiple times if there
- * are a lot of tablets.
- * @param deadline deadline in milliseconds for this method to finish
- * @return a list containing the metadata and locations for each of the tablets in the
- * table
- * @throws Exception
- * @deprecated use the {@link KuduScanToken} API
- */
- @Deprecated
- public List<LocatedTablet> getTabletsLocations(long deadline) throws Exception {
- return getTabletsLocations(null, null, deadline);
- }
-
- /**
- * Asynchronously get all the tablets for this table.
- * @param deadline max time spent in milliseconds for the deferred result of this method to
- * get called back, if deadline is reached, the deferred result will get erred back
- * @return a {@link Deferred} object that yields a list containing the metadata and
- * locations for each of the tablets in the table
- * @deprecated use the {@link KuduScanToken} API
- */
- @Deprecated
- public Deferred<List<LocatedTablet>> asyncGetTabletsLocations(long deadline) {
- return asyncGetTabletsLocations(null, null, deadline);
- }
-
- /**
- * Get all or some tablets for this table. This may query the master multiple times if there
- * are a lot of tablets.
- * This method blocks until it gets all the tablets.
- * @param startKey where to start in the table, pass null to start at the beginning
- * @param endKey where to stop in the table (exclusive), pass null to get all the tablets until
- * the end of the table
- * @param deadline deadline in milliseconds for this method to finish
- * @return a list containing the metadata and locations for each of the tablets in the
- * table
- * @throws Exception
- * @deprecated use the {@link KuduScanToken} API
- */
- @Deprecated
- public List<LocatedTablet> getTabletsLocations(byte[] startKey,
- byte[] endKey,
- long deadline) throws Exception{
- return client.syncLocateTable(this, startKey, endKey, deadline);
- }
-
- /**
- * Asynchronously get all or some tablets for this table.
- * @param startKey where to start in the table, pass null to start at the beginning
- * @param endKey where to stop in the table (exclusive), pass null to get all the tablets until
- * the end of the table
- * @param deadline max time spent in milliseconds for the deferred result of this method to
- * get called back, if deadline is reached, the deferred result will get erred back
- * @return a {@link Deferred} object that yields a list containing the metadata and locations
- * for each of the tablets in the table
- * @deprecated use the {@link KuduScanToken} API
- */
- @Deprecated
- public Deferred<List<LocatedTablet>> asyncGetTabletsLocations(byte[] startKey,
- byte[] endKey,
- long deadline) {
- return client.locateTable(this, startKey, endKey, deadline);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ListTablesRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ListTablesRequest.java b/java/kudu-client/src/main/java/org/kududb/client/ListTablesRequest.java
deleted file mode 100644
index 78725f0..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ListTablesRequest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@InterfaceAudience.Private
-class ListTablesRequest extends KuduRpc<ListTablesResponse> {
-
- private final String nameFilter;
-
- ListTablesRequest(KuduTable masterTable, String nameFilter) {
- super(masterTable);
- this.nameFilter = nameFilter;
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- final Master.ListTablesRequestPB.Builder builder =
- Master.ListTablesRequestPB.newBuilder();
- if (nameFilter != null) {
- builder.setNameFilter(nameFilter);
- }
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return "ListTables";
- }
-
- @Override
- Pair<ListTablesResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- final Master.ListTablesResponsePB.Builder respBuilder =
- Master.ListTablesResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), respBuilder);
- int serversCount = respBuilder.getTablesCount();
- List<String> tables = new ArrayList<String>(serversCount);
- for (Master.ListTablesResponsePB.TableInfo info : respBuilder.getTablesList()) {
- tables.add(info.getName());
- }
- ListTablesResponse response = new ListTablesResponse(deadlineTracker.getElapsedMillis(),
- tsUUID, tables);
- return new Pair<ListTablesResponse, Object>(
- response, respBuilder.hasError() ? respBuilder.getError() : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ListTablesResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ListTablesResponse.java b/java/kudu-client/src/main/java/org/kududb/client/ListTablesResponse.java
deleted file mode 100644
index 70daee2..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ListTablesResponse.java
+++ /dev/null
@@ -1,42 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.util.List;
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class ListTablesResponse extends KuduRpcResponse {
-
- private final List<String> tablesList;
-
- ListTablesResponse(long ellapsedMillis, String tsUUID, List<String> tablesList) {
- super(ellapsedMillis, tsUUID);
- this.tablesList = tablesList;
- }
-
- /**
- * Get the list of tables as specified in the request.
- * @return a list of table names
- */
- public List<String> getTablesList() {
- return tablesList;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersRequest.java b/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersRequest.java
deleted file mode 100644
index bf26626..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersRequest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import static org.kududb.master.Master.*;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@InterfaceAudience.Private
-public class ListTabletServersRequest extends KuduRpc<ListTabletServersResponse> {
-
- public ListTabletServersRequest(KuduTable masterTable) {
- super(masterTable);
- }
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- final ListTabletServersRequestPB.Builder builder =
- ListTabletServersRequestPB.newBuilder();
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return "ListTabletServers";
- }
-
- @Override
- Pair<ListTabletServersResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- final ListTabletServersResponsePB.Builder respBuilder =
- ListTabletServersResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), respBuilder);
- int serversCount = respBuilder.getServersCount();
- List<String> servers = new ArrayList<String>(serversCount);
- for (ListTabletServersResponsePB.Entry entry : respBuilder.getServersList()) {
- servers.add(entry.getRegistration().getRpcAddresses(0).getHost());
- }
- ListTabletServersResponse response = new ListTabletServersResponse(deadlineTracker
- .getElapsedMillis(), tsUUID, serversCount, servers);
- return new Pair<ListTabletServersResponse, Object>(
- response, respBuilder.hasError() ? respBuilder.getError() : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersResponse.java b/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersResponse.java
deleted file mode 100644
index 373a14d..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ListTabletServersResponse.java
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.util.List;
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class ListTabletServersResponse extends KuduRpcResponse {
-
- private final int tabletServersCount;
- private final List<String> tabletServersList;
-
- /**
- * @param ellapsedMillis Time in milliseconds since RPC creation to now.
- * @param tabletServersCount How many tablet servers the master is reporting.
- * @param tabletServersList List of tablet servers.
- */
- ListTabletServersResponse(long ellapsedMillis, String tsUUID,
- int tabletServersCount, List<String> tabletServersList) {
- super(ellapsedMillis, tsUUID);
- this.tabletServersCount = tabletServersCount;
- this.tabletServersList = tabletServersList;
- }
-
- /**
- * Get the count of tablet servers as reported by the master.
- * @return TS count.
- */
- public int getTabletServersCount() {
- return tabletServersCount;
- }
-
- /**
- * Get the list of tablet servers, as represented by their hostname.
- * @return List of hostnames, one per TS.
- */
- public List<String> getTabletServersList() {
- return tabletServersList;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ListTabletsRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ListTabletsRequest.java b/java/kudu-client/src/main/java/org/kududb/client/ListTabletsRequest.java
deleted file mode 100644
index 8a29b7b..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ListTabletsRequest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.tablet.Tablet;
-import org.kududb.tserver.Tserver;
-import org.kududb.tserver.TserverService;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@InterfaceAudience.Private
-class ListTabletsRequest extends KuduRpc<ListTabletsResponse> {
-
- ListTabletsRequest() {
- super(null);
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- final Tserver.ListTabletsRequestPB.Builder builder =
- Tserver.ListTabletsRequestPB.newBuilder();
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() { return TABLET_SERVER_SERVICE_NAME; }
-
- @Override
- String method() {
- return "ListTablets";
- }
-
- @Override
- Pair<ListTabletsResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- final Tserver.ListTabletsResponsePB.Builder respBuilder =
- Tserver.ListTabletsResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), respBuilder);
- int serversCount = respBuilder.getStatusAndSchemaCount();
- List<String> tablets = new ArrayList<String>(serversCount);
- for (Tserver.ListTabletsResponsePB.StatusAndSchemaPB info
- : respBuilder.getStatusAndSchemaList()) {
- tablets.add(info.getTabletStatus().getTabletId());
- }
- ListTabletsResponse response = new ListTabletsResponse(deadlineTracker.getElapsedMillis(),
- tsUUID, tablets);
- return new Pair<ListTabletsResponse, Object>(
- response, respBuilder.hasError() ? respBuilder.getError() : null);
- }
-}
[29/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
new file mode 100644
index 0000000..c1ccb3f
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -0,0 +1,2437 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.kududb.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.net.HostAndPort;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.Message;
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.socket.nio.NioWorkerPool;
+import org.kududb.Common;
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.consensus.Metadata;
+import org.kududb.master.Master;
+import org.kududb.master.Master.GetTableLocationsResponsePB;
+import org.kududb.util.AsyncUtil;
+import org.kududb.util.NetUtil;
+import org.kududb.util.Pair;
+import org.kududb.util.Slice;
+import org.jboss.netty.channel.ChannelEvent;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.DefaultChannelPipeline;
+import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
+import org.jboss.netty.channel.socket.SocketChannel;
+import org.jboss.netty.channel.socket.SocketChannelConfig;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.handler.timeout.ReadTimeoutHandler;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.jboss.netty.util.Timeout;
+import org.jboss.netty.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.GuardedBy;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+
+/**
+ * A fully asynchronous and thread-safe client for Kudu.
+ * <p>
+ * This client should be
+ * instantiated only once. You can use it with any number of tables at the
+ * same time. The only case where you should have multiple instances is when
+ * you want to use multiple different clusters at the same time.
+ * <p>
+ * If you play by the rules, this client is completely
+ * thread-safe. Read the documentation carefully to know what the requirements
+ * are for this guarantee to apply.
+ * <p>
+ * This client is fully non-blocking, any blocking operation will return a
+ * {@link Deferred} instance to which you can attach a {@link Callback} chain
+ * that will execute when the asynchronous operation completes.
+ *
+ * <h1>Note regarding {@code KuduRpc} instances passed to this class</h1>
+ * Every {@link KuduRpc} passed to a method of this class should not be
+ * changed or re-used until the {@code Deferred} returned by that method
+ * calls you back. <strong>Changing or re-using any {@link KuduRpc} for
+ * an RPC in flight will lead to <em>unpredictable</em> results and voids
+ * your warranty</strong>.
+ *
+ * <h1>{@code throws} clauses</h1>
+ * None of the asynchronous methods in this API are expected to throw an
+ * exception. But the {@link Deferred} object they return to you can carry an
+ * exception that you should handle (using "errbacks", see the javadoc of
+ * {@link Deferred}). In order to be able to do proper asynchronous error
+ * handling, you need to know what types of exceptions you're expected to face
+ * in your errbacks. In order to document that, the methods of this API use
+ * javadoc's {@code @throws} to spell out the exception types you should
+ * handle in your errback. Asynchronous exceptions will be indicated as such
+ * in the javadoc with "(deferred)".
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class AsyncKuduClient implements AutoCloseable {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AsyncKuduClient.class);
+ public static final int SLEEP_TIME = 500;
+ public static final byte[] EMPTY_ARRAY = new byte[0];
+ public static final long NO_TIMESTAMP = -1;
+ public static final long DEFAULT_OPERATION_TIMEOUT_MS = 30000;
+ public static final long DEFAULT_SOCKET_READ_TIMEOUT_MS = 10000;
+ private static final long MAX_RPC_ATTEMPTS = 100;
+
+ private final ClientSocketChannelFactory channelFactory;
+
+ /**
+ * This map and the next 2 maps contain the same data, but indexed
+ * differently. There is no consistency guarantee across the maps.
+ * They are not updated all at the same time atomically. This map
+ * is always the first to be updated, because that's the map from
+ * which all the lookups are done in the fast-path of the requests
+ * that need to locate a tablet. The second map to be updated is
+ * tablet2client, because it comes second in the fast-path
+ * of every requests that need to locate a tablet. The third map
+ * is only used to handle TabletServer disconnections gracefully.
+ *
+ * This map is keyed by table ID.
+ */
+ private final ConcurrentHashMap<String, ConcurrentSkipListMap<byte[],
+ RemoteTablet>> tabletsCache = new ConcurrentHashMap<>();
+
+ /**
+ * Maps a tablet ID to the RemoteTablet that knows where all the replicas are served.
+ */
+ private final ConcurrentHashMap<Slice, RemoteTablet> tablet2client = new ConcurrentHashMap<>();
+
+ /**
+ * Maps a client connected to a TabletServer to the list of tablets we know
+ * it's serving so far.
+ */
+ private final ConcurrentHashMap<TabletClient, ArrayList<RemoteTablet>> client2tablets =
+ new ConcurrentHashMap<>();
+
+ /**
+ * Map of table ID to non-covered range cache.
+ *
+ * TODO: Currently once a non-covered range is added to the cache, it is never
+ * removed. Once adding range partitions becomes possible entries will need to
+ * be expired.
+ */
+ private final ConcurrentMap<String, NonCoveredRangeCache> nonCoveredRangeCaches =
+ new ConcurrentHashMap<>();
+
+ /**
+ * Cache that maps a TabletServer address ("ip:port") to the clients
+ * connected to it.
+ * <p>
+ * Access to this map must be synchronized by locking its monitor.
+ * Lock ordering: when locking both this map and a TabletClient, the
+ * TabletClient must always be locked first to avoid deadlocks. Logging
+ * the contents of this map (or calling toString) requires copying it first.
+ * <p>
+ * This isn't a {@link ConcurrentHashMap} because we don't use it frequently
+ * (just when connecting to / disconnecting from TabletClients) and when we
+ * add something to it, we want to do an atomic get-and-put, but
+ * {@code putIfAbsent} isn't a good fit for us since it requires to create
+ * an object that may be "wasted" in case another thread wins the insertion
+ * race, and we don't want to create unnecessary connections.
+ * <p>
+ * Upon disconnection, clients are automatically removed from this map.
+ * We don't use a {@code ChannelGroup} because a {@code ChannelGroup} does
+ * the clean-up on the {@code channelClosed} event, which is actually the
+ * 3rd and last event to be fired when a channel gets disconnected. The
+ * first one to get fired is, {@code channelDisconnected}. This matters to
+ * us because we want to purge disconnected clients from the cache as
+ * quickly as possible after the disconnection, to avoid handing out clients
+ * that are going to cause unnecessary errors.
+ * @see TabletClientPipeline#handleDisconnect
+ */
+ private final HashMap<String, TabletClient> ip2client =
+ new HashMap<String, TabletClient>();
+
+ @GuardedBy("sessions")
+ private final Set<AsyncKuduSession> sessions = new HashSet<AsyncKuduSession>();
+
+ // Since the masters also go through TabletClient, we need to treat them as if they were a normal
+ // table. We'll use the following fake table name to identify places where we need special
+ // handling.
+ static final String MASTER_TABLE_NAME_PLACEHOLDER = "Kudu Master";
+ final KuduTable masterTable;
+ private final List<HostAndPort> masterAddresses;
+
+ private final HashedWheelTimer timer;
+
+ /**
+ * Timestamp required for HybridTime external consistency through timestamp
+ * propagation.
+ * @see src/kudu/common/common.proto
+ */
+ private long lastPropagatedTimestamp = NO_TIMESTAMP;
+
+ // A table is considered not served when we get an empty list of locations but know
+ // that a tablet exists. This is currently only used for new tables. The objects stored are
+ // table IDs.
+ private final Set<String> tablesNotServed = Collections.newSetFromMap(new
+ ConcurrentHashMap<String, Boolean>());
+
+ /**
+ * Semaphore used to rate-limit master lookups
+ * Once we have more than this number of concurrent master lookups, we'll
+ * start to throttle ourselves slightly.
+ * @see #acquireMasterLookupPermit
+ */
+ private final Semaphore masterLookups = new Semaphore(50);
+
+ private final Random sleepRandomizer = new Random();
+
+ private final long defaultOperationTimeoutMs;
+
+ private final long defaultAdminOperationTimeoutMs;
+
+ private final long defaultSocketReadTimeoutMs;
+
+ private final Statistics statistics;
+
+ private final boolean statisticsDisabled;
+
+ private final RequestTracker requestTracker;
+
+ private volatile boolean closed;
+
+ private AsyncKuduClient(AsyncKuduClientBuilder b) {
+ this.channelFactory = b.createChannelFactory();
+ this.masterAddresses = b.masterAddresses;
+ this.masterTable = new KuduTable(this, MASTER_TABLE_NAME_PLACEHOLDER,
+ MASTER_TABLE_NAME_PLACEHOLDER, null, null);
+ this.defaultOperationTimeoutMs = b.defaultOperationTimeoutMs;
+ this.defaultAdminOperationTimeoutMs = b.defaultAdminOperationTimeoutMs;
+ this.defaultSocketReadTimeoutMs = b.defaultSocketReadTimeoutMs;
+ this.statisticsDisabled = b.statisticsDisabled;
+ statistics = statisticsDisabled ? null : new Statistics();
+ this.timer = b.timer;
+ String clientId = UUID.randomUUID().toString().replace("-", "");
+ this.requestTracker = new RequestTracker(clientId);
+ }
+
+ /**
+ * Updates the last timestamp received from a server. Used for CLIENT_PROPAGATED
+ * external consistency. This is only publicly visible so that it can be set
+ * on tests, users should generally disregard this method.
+ *
+ * @param lastPropagatedTimestamp the last timestamp received from a server
+ */
+ @VisibleForTesting
+ public synchronized void updateLastPropagatedTimestamp(long lastPropagatedTimestamp) {
+ if (this.lastPropagatedTimestamp == -1 ||
+ this.lastPropagatedTimestamp < lastPropagatedTimestamp) {
+ this.lastPropagatedTimestamp = lastPropagatedTimestamp;
+ }
+ }
+
+ @VisibleForTesting
+ public synchronized long getLastPropagatedTimestamp() {
+ return lastPropagatedTimestamp;
+ }
+
+ /**
+ * Returns a synchronous {@link KuduClient} which wraps this asynchronous client.
+ * Calling {@link KuduClient#close} on the returned client will close this client.
+ * If this asynchronous client should outlive the returned synchronous client,
+ * then do not close the synchronous client.
+ * @return a new synchronous {@code KuduClient}
+ */
+ public KuduClient syncClient() {
+ return new KuduClient(this);
+ }
+
+ /**
+ * Create a table on the cluster with the specified name, schema, and table configurations.
+ * @param name the table's name
+ * @param schema the table's schema
+ * @param builder a builder containing the table's configurations
+ * @return a deferred object to track the progress of the createTable command that gives
+ * an object to communicate with the created table
+ */
+ public Deferred<KuduTable> createTable(final String name, Schema schema,
+ CreateTableOptions builder) {
+ checkIsClosed();
+ if (builder == null) {
+ throw new IllegalArgumentException("CreateTableOptions may not be null");
+ }
+ if (!builder.getBuilder().getPartitionSchema().hasRangeSchema() &&
+ builder.getBuilder().getPartitionSchema().getHashBucketSchemasCount() == 0) {
+ throw new IllegalArgumentException("Table partitioning must be specified using " +
+ "setRangePartitionColumns or addHashPartitions");
+
+ }
+ CreateTableRequest create = new CreateTableRequest(this.masterTable, name, schema, builder);
+ create.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ return sendRpcToTablet(create).addCallbackDeferring(
+ new Callback<Deferred<KuduTable>, CreateTableResponse>() {
+ @Override
+ public Deferred<KuduTable> call(CreateTableResponse createTableResponse) throws Exception {
+ return openTable(name);
+ }
+ });
+ }
+
+ /**
+ * Delete a table on the cluster with the specified name.
+ * @param name the table's name
+ * @return a deferred object to track the progress of the deleteTable command
+ */
+ public Deferred<DeleteTableResponse> deleteTable(String name) {
+ checkIsClosed();
+ DeleteTableRequest delete = new DeleteTableRequest(this.masterTable, name);
+ delete.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ return sendRpcToTablet(delete);
+ }
+
+ /**
+ * Alter a table on the cluster as specified by the builder.
+ *
+ * When the returned deferred completes it only indicates that the master accepted the alter
+ * command, use {@link AsyncKuduClient#isAlterTableDone(String)} to know when the alter finishes.
+ * @param name the table's name, if this is a table rename then the old table name must be passed
+ * @param ato the alter table builder
+ * @return a deferred object to track the progress of the alter command
+ */
+ public Deferred<AlterTableResponse> alterTable(String name, AlterTableOptions ato) {
+ checkIsClosed();
+ AlterTableRequest alter = new AlterTableRequest(this.masterTable, name, ato);
+ alter.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ return sendRpcToTablet(alter);
+ }
+
+ /**
+ * Helper method that checks and waits until the completion of an alter command.
+ * It will block until the alter command is done or the deadline is reached.
+ * @param name the table's name, if the table was renamed then that name must be checked against
+ * @return a deferred object to track the progress of the isAlterTableDone command
+ */
+ public Deferred<IsAlterTableDoneResponse> isAlterTableDone(String name) {
+ checkIsClosed();
+ IsAlterTableDoneRequest request = new IsAlterTableDoneRequest(this.masterTable, name);
+ request.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ return sendRpcToTablet(request);
+ }
+
+ /**
+ * Get the list of running tablet servers.
+ * @return a deferred object that yields a list of tablet servers
+ */
+ public Deferred<ListTabletServersResponse> listTabletServers() {
+ checkIsClosed();
+ ListTabletServersRequest rpc = new ListTabletServersRequest(this.masterTable);
+ rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ return sendRpcToTablet(rpc);
+ }
+
+ Deferred<GetTableSchemaResponse> getTableSchema(String name) {
+ GetTableSchemaRequest rpc = new GetTableSchemaRequest(this.masterTable, name);
+ rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ return sendRpcToTablet(rpc);
+ }
+
+ /**
+ * Get the list of all the tables.
+ * @return a deferred object that yields a list of all the tables
+ */
+ public Deferred<ListTablesResponse> getTablesList() {
+ return getTablesList(null);
+ }
+
+ /**
+ * Get a list of table names. Passing a null filter returns all the tables. When a filter is
+ * specified, it only returns tables that satisfy a substring match.
+ * @param nameFilter an optional table name filter
+ * @return a deferred that yields the list of table names
+ */
+ public Deferred<ListTablesResponse> getTablesList(String nameFilter) {
+ ListTablesRequest rpc = new ListTablesRequest(this.masterTable, nameFilter);
+ rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ return sendRpcToTablet(rpc);
+ }
+
+ /**
+ * Test if a table exists.
+ * @param name a non-null table name
+ * @return true if the table exists, else false
+ */
+ public Deferred<Boolean> tableExists(final String name) {
+ if (name == null) {
+ throw new IllegalArgumentException("The table name cannot be null");
+ }
+ return getTablesList().addCallbackDeferring(new Callback<Deferred<Boolean>,
+ ListTablesResponse>() {
+ @Override
+ public Deferred<Boolean> call(ListTablesResponse listTablesResponse) throws Exception {
+ for (String tableName : listTablesResponse.getTablesList()) {
+ if (name.equals(tableName)) {
+ return Deferred.fromResult(true);
+ }
+ }
+ return Deferred.fromResult(false);
+ }
+ });
+ }
+
+ /**
+ * Open the table with the given name. If the table was just created, the Deferred will only get
+ * called back when all the tablets have been successfully created.
+ * @param name table to open
+ * @return a KuduTable if the table exists, else a MasterErrorException
+ */
+ public Deferred<KuduTable> openTable(final String name) {
+ checkIsClosed();
+
+ // We create an RPC that we're never going to send, and will instead use it to keep track of
+ // timeouts and use its Deferred.
+ final KuduRpc<KuduTable> fakeRpc = new KuduRpc<KuduTable>(null) {
+ @Override
+ ChannelBuffer serialize(Message header) { return null; }
+
+ @Override
+ String serviceName() { return null; }
+
+ @Override
+ String method() {
+ return "IsCreateTableDone";
+ }
+
+ @Override
+ Pair<KuduTable, Object> deserialize(CallResponse callResponse, String tsUUID)
+ throws Exception { return null; }
+ };
+ fakeRpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+
+ return getTableSchema(name).addCallbackDeferring(new Callback<Deferred<KuduTable>,
+ GetTableSchemaResponse>() {
+ @Override
+ public Deferred<KuduTable> call(GetTableSchemaResponse response) throws Exception {
+ KuduTable table = new KuduTable(AsyncKuduClient.this,
+ name,
+ response.getTableId(),
+ response.getSchema(),
+ response.getPartitionSchema());
+ // We grab the Deferred first because calling callback on the RPC will reset it and we'd
+ // return a different, non-triggered Deferred.
+ Deferred<KuduTable> d = fakeRpc.getDeferred();
+ if (response.isCreateTableDone()) {
+ LOG.debug("Opened table {}", name);
+ fakeRpc.callback(table);
+ } else {
+ LOG.debug("Delaying opening table {}, its tablets aren't fully created", name);
+ fakeRpc.attempt++;
+ delayedIsCreateTableDone(
+ table,
+ fakeRpc,
+ getOpenTableCB(fakeRpc, table),
+ getDelayedIsCreateTableDoneErrback(fakeRpc));
+ }
+ return d;
+ }
+ });
+ }
+
+ /**
+ * This callback will be repeatadly used when opening a table until it is done being created.
+ */
+ Callback<Deferred<KuduTable>, Master.IsCreateTableDoneResponsePB> getOpenTableCB(
+ final KuduRpc<KuduTable> rpc, final KuduTable table) {
+ return new Callback<Deferred<KuduTable>, Master.IsCreateTableDoneResponsePB>() {
+ @Override
+ public Deferred<KuduTable> call(
+ Master.IsCreateTableDoneResponsePB isCreateTableDoneResponsePB) throws Exception {
+ String tableName = table.getName();
+ Deferred<KuduTable> d = rpc.getDeferred();
+ if (isCreateTableDoneResponsePB.getDone()) {
+ LOG.debug("Table {}'s tablets are now created", tableName);
+ rpc.callback(table);
+ } else {
+ rpc.attempt++;
+ LOG.debug("Table {}'s tablets are still not created, further delaying opening it",
+ tableName);
+
+ delayedIsCreateTableDone(
+ table,
+ rpc,
+ getOpenTableCB(rpc, table),
+ getDelayedIsCreateTableDoneErrback(rpc));
+ }
+ return d;
+ }
+ };
+ }
+
+ /**
+ * Get the timeout used for operations on sessions and scanners.
+ * @return a timeout in milliseconds
+ */
+ public long getDefaultOperationTimeoutMs() {
+ return defaultOperationTimeoutMs;
+ }
+
+ /**
+ * Get the timeout used for admin operations.
+ * @return a timeout in milliseconds
+ */
+ public long getDefaultAdminOperationTimeoutMs() {
+ return defaultAdminOperationTimeoutMs;
+ }
+
+ /**
+ * Get the timeout used when waiting to read data from a socket. Will be triggered when nothing
+ * has been read on a socket connected to a tablet server for {@code timeout} milliseconds.
+ * @return a timeout in milliseconds
+ */
+ public long getDefaultSocketReadTimeoutMs() {
+ return defaultSocketReadTimeoutMs;
+ }
+
+ /**
+ * Check if statistics collection is enabled for this client.
+ * @return true if it is enabled, else false
+ */
+ public boolean isStatisticsEnabled() {
+ return !statisticsDisabled;
+ }
+
+ /**
+ * Get the statistics object of this client.
+ *
+ * @return this client's Statistics object
+ * @throws IllegalStateException thrown if statistics collection has been disabled
+ */
+ public Statistics getStatistics() {
+ if (statisticsDisabled) {
+ throw new IllegalStateException("This client's statistics is disabled");
+ }
+ return this.statistics;
+ }
+
+ RequestTracker getRequestTracker() {
+ return requestTracker;
+ }
+
+ /**
+ * Creates a new {@link AsyncKuduScanner.AsyncKuduScannerBuilder} for a particular table.
+ * @param table the name of the table you intend to scan.
+ * The string is assumed to use the platform's default charset.
+ * @return a new scanner builder for this table
+ */
+ public AsyncKuduScanner.AsyncKuduScannerBuilder newScannerBuilder(KuduTable table) {
+ checkIsClosed();
+ return new AsyncKuduScanner.AsyncKuduScannerBuilder(this, table);
+ }
+
+ /**
+ * Create a new session for interacting with the cluster.
+ * User is responsible for destroying the session object.
+ * This is a fully local operation (no RPCs or blocking).
+ * @return a new AsyncKuduSession
+ */
+ public AsyncKuduSession newSession() {
+ checkIsClosed();
+ AsyncKuduSession session = new AsyncKuduSession(this);
+ synchronized (sessions) {
+ sessions.add(session);
+ }
+ return session;
+ }
+
+ /**
+ * This method is for KuduSessions so that they can remove themselves as part of closing down.
+ * @param session Session to remove
+ */
+ void removeSession(AsyncKuduSession session) {
+ synchronized (sessions) {
+ boolean removed = sessions.remove(session);
+ assert removed == true;
+ }
+ }
+
+ /**
+ * Package-private access point for {@link AsyncKuduScanner}s to scan more rows.
+ * @param scanner The scanner to use.
+ * @return A deferred row.
+ */
+ Deferred<AsyncKuduScanner.Response> scanNextRows(final AsyncKuduScanner scanner) {
+ final RemoteTablet tablet = scanner.currentTablet();
+ final TabletClient client = clientFor(tablet);
+ final KuduRpc<AsyncKuduScanner.Response> next_request = scanner.getNextRowsRequest();
+ final Deferred<AsyncKuduScanner.Response> d = next_request.getDeferred();
+ // Important to increment the attempts before the next if statement since
+ // getSleepTimeForRpc() relies on it if the client is null or dead.
+ next_request.attempt++;
+ if (client == null || !client.isAlive()) {
+ // A null client means we either don't know about this tablet anymore (unlikely) or we
+ // couldn't find a leader (which could be triggered by a read timeout).
+ // We'll first delay the RPC in case things take some time to settle down, then retry.
+ delayedSendRpcToTablet(next_request, null);
+ return next_request.getDeferred();
+ }
+ client.sendRpc(next_request);
+ return d;
+ }
+
+ /**
+ * Package-private access point for {@link AsyncKuduScanner}s to close themselves.
+ * @param scanner the scanner to close
+ * @return a deferred object that indicates the completion of the request.
+ * The {@link AsyncKuduScanner.Response} can contain rows that were left to scan.
+ */
+ Deferred<AsyncKuduScanner.Response> closeScanner(final AsyncKuduScanner scanner) {
+ final RemoteTablet tablet = scanner.currentTablet();
+ // Getting a null tablet here without being in a closed state means we were in between tablets.
+ if (tablet == null) {
+ return Deferred.fromResult(null);
+ }
+
+ final TabletClient client = clientFor(tablet);
+ if (client == null || !client.isAlive()) {
+ // Oops, we couldn't find a tablet server that hosts this tablet. Our
+ // cache was probably invalidated while the client was scanning. So
+ // we can't close this scanner properly.
+ LOG.warn("Cannot close {} properly, no connection open for {}", scanner, tablet);
+ return Deferred.fromResult(null);
+ }
+ final KuduRpc<AsyncKuduScanner.Response> close_request = scanner.getCloseRequest();
+ final Deferred<AsyncKuduScanner.Response> d = close_request.getDeferred();
+ close_request.attempt++;
+ client.sendRpc(close_request);
+ return d;
+ }
+
+ /**
+ * Sends the provided {@link KuduRpc} to the tablet server hosting the leader
+ * of the tablet identified by the RPC's table and partition key.
+ *
+ * Note: despite the name, this method is also used for routing master
+ * requests to the leader master instance since it's also handled like a tablet.
+ *
+ * @param request the RPC to send
+ * @param <R> the expected return type of the RPC
+ * @return a {@code Deferred} which will contain the response
+ */
+ <R> Deferred<R> sendRpcToTablet(final KuduRpc<R> request) {
+ if (cannotRetryRequest(request)) {
+ return tooManyAttemptsOrTimeout(request, null);
+ }
+ request.attempt++;
+ final String tableId = request.getTable().getTableId();
+ byte[] partitionKey = request.partitionKey();
+ RemoteTablet tablet = getTablet(tableId, partitionKey);
+
+ if (tablet == null && partitionKey != null) {
+ // Check if the RPC is in a non-covered range.
+ Map.Entry<byte[], byte[]> nonCoveredRange = getNonCoveredRange(tableId, partitionKey);
+ if (nonCoveredRange != null) {
+ return Deferred.fromError(new NonCoveredRangeException(nonCoveredRange.getKey(),
+ nonCoveredRange.getValue()));
+ }
+ // Otherwise fall through to below where a GetTableLocations lookup will occur.
+ }
+
+ // Set the propagated timestamp so that the next time we send a message to
+ // the server the message includes the last propagated timestamp.
+ long lastPropagatedTs = getLastPropagatedTimestamp();
+ if (request.getExternalConsistencyMode() == CLIENT_PROPAGATED &&
+ lastPropagatedTs != NO_TIMESTAMP) {
+ request.setPropagatedTimestamp(lastPropagatedTs);
+ }
+
+ // If we found a tablet, we'll try to find the TS to talk to. If that TS was previously
+ // disconnected, say because we didn't query that tablet for some seconds, then we'll try to
+ // reconnect based on the old information. If that fails, we'll instead continue with the next
+ // block that queries the master.
+ if (tablet != null) {
+ TabletClient tabletClient = clientFor(tablet);
+ if (tabletClient != null) {
+ final Deferred<R> d = request.getDeferred();
+ if (tabletClient.isAlive()) {
+ request.setTablet(tablet);
+ tabletClient.sendRpc(request);
+ return d;
+ }
+ try {
+ tablet.reconnectTabletClient(tabletClient);
+ } catch (UnknownHostException e) {
+ LOG.error("Cached tablet server {}'s host cannot be resolved, will query the master",
+ tabletClient.getUuid(), e);
+ // Because of this exception, clientFor() below won't be able to find a newTabletClient
+ // and we'll delay the RPC.
+ }
+ TabletClient newTabletClient = clientFor(tablet);
+ assert (tabletClient != newTabletClient);
+
+ if (newTabletClient == null) {
+ // Wait a little bit before hitting the master.
+ delayedSendRpcToTablet(request, null);
+ return request.getDeferred();
+ }
+
+ if (!newTabletClient.isAlive()) {
+ LOG.debug("Tried reconnecting to tablet server {} but failed, " +
+ "will query the master", tabletClient.getUuid());
+ // Let fall through.
+ } else {
+ request.setTablet(tablet);
+ newTabletClient.sendRpc(request);
+ return d;
+ }
+ }
+ }
+
+ // We fall through to here in two cases:
+ //
+ // 1) This client has not yet discovered the tablet which is responsible for
+ // the RPC's table and partition key. This can happen when the client's
+ // tablet location cache is cold because the client is new, or the table
+ // is new.
+ //
+ // 2) The tablet is known, but we do not have an active client for the
+ // leader replica.
+ if (tablesNotServed.contains(tableId)) {
+ return delayedIsCreateTableDone(request.getTable(), request,
+ new RetryRpcCB<R, Master.IsCreateTableDoneResponsePB>(request),
+ getDelayedIsCreateTableDoneErrback(request));
+ }
+ Callback<Deferred<R>, Master.GetTableLocationsResponsePB> cb = new RetryRpcCB<>(request);
+ Callback<Deferred<R>, Exception> eb = new RetryRpcErrback<>(request);
+ Deferred<Master.GetTableLocationsResponsePB> returnedD =
+ locateTablet(request.getTable(), partitionKey);
+ return AsyncUtil.addCallbacksDeferring(returnedD, cb, eb);
+ }
+
+ /**
+ * Callback used to retry a RPC after another query finished, like looking up where that RPC
+ * should go.
+ * <p>
+ * Use {@code AsyncUtil.addCallbacksDeferring} to add this as the callback and
+ * {@link AsyncKuduClient.RetryRpcErrback} as the "errback" to the {@code Deferred}
+ * returned by {@link #locateTablet(KuduTable, byte[])}.
+ * @param <R> RPC's return type.
+ * @param <D> Previous query's return type, which we don't use, but need to specify in order to
+ * tie it all together.
+ */
+ final class RetryRpcCB<R, D> implements Callback<Deferred<R>, D> {
+ private final KuduRpc<R> request;
+ RetryRpcCB(KuduRpc<R> request) {
+ this.request = request;
+ }
+ public Deferred<R> call(final D arg) {
+ LOG.debug("Retrying sending RPC {} after lookup", request);
+ return sendRpcToTablet(request); // Retry the RPC.
+ }
+ public String toString() {
+ return "retry RPC";
+ }
+ }
+
+ /**
+ * "Errback" used to delayed-retry a RPC if it fails due to no leader master being found.
+ * Other exceptions are used to notify request RPC error, and passed through to be handled
+ * by the caller.
+ * <p>
+ * Use {@code AsyncUtil.addCallbacksDeferring} to add this as the "errback" and
+ * {@link RetryRpcCB} as the callback to the {@code Deferred} returned by
+ * {@link #locateTablet(KuduTable, byte[])}.
+ * @see #delayedSendRpcToTablet(KuduRpc, KuduException)
+ * @param <R> The type of the original RPC.
+ */
+ final class RetryRpcErrback<R> implements Callback<Deferred<R>, Exception> {
+ private final KuduRpc<R> request;
+
+ public RetryRpcErrback(KuduRpc<R> request) {
+ this.request = request;
+ }
+
+ @Override
+ public Deferred<R> call(Exception arg) {
+ if (arg instanceof NoLeaderMasterFoundException) {
+ // If we could not find the leader master, try looking up the leader master
+ // again.
+ // TODO: Handle the situation when multiple in-flight RPCs are queued waiting
+ // for the leader master to be determine (either after a failure or at initialization
+ // time). This could re-use some of the existing piping in place for non-master tablets.
+ Deferred<R> d = request.getDeferred();
+ delayedSendRpcToTablet(request, (NoLeaderMasterFoundException) arg);
+ return d;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Notify RPC %s after lookup exception", request), arg);
+ }
+ request.errback(arg);
+ return Deferred.fromError(arg);
+ }
+
+ @Override
+ public String toString() {
+ return "retry RPC after error";
+ }
+ }
+
+ /**
+ * This errback ensures that if the delayed call to IsCreateTableDone throws an Exception that
+ * it will be propagated back to the user.
+ * @param request Request to errback if there's a problem with the delayed call.
+ * @param <R> Request's return type.
+ * @return An errback.
+ */
+ <R> Callback<Exception, Exception> getDelayedIsCreateTableDoneErrback(final KuduRpc<R> request) {
+ return new Callback<Exception, Exception>() {
+ @Override
+ public Exception call(Exception e) throws Exception {
+ // TODO maybe we can retry it?
+ request.errback(e);
+ return e;
+ }
+ };
+ }
+
+ /**
+ * This method will call IsCreateTableDone on the master after sleeping for
+ * getSleepTimeForRpc() based on the provided KuduRpc's number of attempts. Once this is done,
+ * the provided callback will be called.
+ * @param table the table to lookup
+ * @param rpc the original KuduRpc that needs to access the table
+ * @param retryCB the callback to call on completion
+ * @param errback the errback to call if something goes wrong when calling IsCreateTableDone
+ * @return Deferred used to track the provided KuduRpc
+ */
+ <R> Deferred<R> delayedIsCreateTableDone(final KuduTable table, final KuduRpc<R> rpc,
+ final Callback<Deferred<R>,
+ Master.IsCreateTableDoneResponsePB> retryCB,
+ final Callback<Exception, Exception> errback) {
+
+ final class RetryTimer implements TimerTask {
+ public void run(final Timeout timeout) {
+ String tableId = table.getTableId();
+ final boolean has_permit = acquireMasterLookupPermit();
+ if (!has_permit) {
+ // If we failed to acquire a permit, it's worth checking if someone
+ // looked up the tablet we're interested in. Every once in a while
+ // this will save us a Master lookup.
+ if (!tablesNotServed.contains(tableId)) {
+ try {
+ retryCB.call(null);
+ return;
+ } catch (Exception e) {
+ // we're calling RetryRpcCB which doesn't throw exceptions, ignore
+ }
+ }
+ }
+ IsCreateTableDoneRequest rpc = new IsCreateTableDoneRequest(masterTable, tableId);
+ rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ final Deferred<Master.IsCreateTableDoneResponsePB> d =
+ sendRpcToTablet(rpc).addCallback(new IsCreateTableDoneCB(tableId));
+ if (has_permit) {
+ // The errback is needed here to release the lookup permit
+ d.addCallbacks(new ReleaseMasterLookupPermit<Master.IsCreateTableDoneResponsePB>(),
+ new ReleaseMasterLookupPermit<Exception>());
+ }
+ d.addCallbacks(retryCB, errback);
+ }
+ }
+ long sleepTime = getSleepTimeForRpc(rpc);
+ if (rpc.deadlineTracker.wouldSleepingTimeout(sleepTime)) {
+ return tooManyAttemptsOrTimeout(rpc, null);
+ }
+
+ newTimeout(new RetryTimer(), sleepTime);
+ return rpc.getDeferred();
+ }
+
+ private final class ReleaseMasterLookupPermit<T> implements Callback<T, T> {
+ public T call(final T arg) {
+ releaseMasterLookupPermit();
+ return arg;
+ }
+ public String toString() {
+ return "release master lookup permit";
+ }
+ }
+
+ /** Callback executed when IsCreateTableDone completes. */
+ private final class IsCreateTableDoneCB implements Callback<Master.IsCreateTableDoneResponsePB,
+ Master.IsCreateTableDoneResponsePB> {
+ final String tableName;
+ IsCreateTableDoneCB(String tableName) {
+ this.tableName = tableName;
+ }
+ public Master.IsCreateTableDoneResponsePB call(final Master.IsCreateTableDoneResponsePB response) {
+ if (response.getDone()) {
+ LOG.debug("Table {} was created", tableName);
+ tablesNotServed.remove(tableName);
+ } else {
+ LOG.debug("Table {} is still being created", tableName);
+ }
+ return response;
+ }
+ public String toString() {
+ return "ask the master if " + tableName + " was created";
+ }
+ }
+
+ boolean isTableNotServed(String tableId) {
+ return tablesNotServed.contains(tableId);
+ }
+
+
+ long getSleepTimeForRpc(KuduRpc<?> rpc) {
+ byte attemptCount = rpc.attempt;
+ assert (attemptCount > 0);
+ if (attemptCount == 0) {
+ LOG.warn("Possible bug: attempting to retry an RPC with no attempts. RPC: " + rpc,
+ new Exception("Exception created to collect stack trace"));
+ attemptCount = 1;
+ }
+ // Randomized exponential backoff, truncated at 4096ms.
+ long sleepTime = (long)(Math.pow(2.0, Math.min(attemptCount, 12))
+ * sleepRandomizer.nextDouble());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Going to sleep for " + sleepTime + " at retry " + rpc.attempt);
+ }
+ return sleepTime;
+ }
+
+ /**
+ * Modifying the list returned by this method won't change how AsyncKuduClient behaves,
+ * but calling certain methods on the returned TabletClients can. For example,
+ * it's possible to forcefully shutdown a connection to a tablet server by calling {@link
+ * TabletClient#shutdown()}.
+ * @return Copy of the current TabletClients list
+ */
+ @VisibleForTesting
+ List<TabletClient> getTabletClients() {
+ synchronized (ip2client) {
+ return new ArrayList<TabletClient>(ip2client.values());
+ }
+ }
+
+ /**
+ * This method first clears tabletsCache and then tablet2client without any regards for
+ * calls to {@link #discoverTablets}. Call only when AsyncKuduClient is in a steady state.
+ * @param tableId table for which we remove all the RemoteTablet entries
+ */
+ @VisibleForTesting
+ void emptyTabletsCacheForTable(String tableId) {
+ tabletsCache.remove(tableId);
+ Set<Map.Entry<Slice, RemoteTablet>> tablets = tablet2client.entrySet();
+ for (Map.Entry<Slice, RemoteTablet> entry : tablets) {
+ if (entry.getValue().getTableId().equals(tableId)) {
+ tablets.remove(entry);
+ }
+ }
+ }
+
+ TabletClient clientFor(RemoteTablet tablet) {
+ if (tablet == null) {
+ return null;
+ }
+
+ synchronized (tablet.tabletServers) {
+ if (tablet.tabletServers.isEmpty()) {
+ return null;
+ }
+ if (tablet.leaderIndex == RemoteTablet.NO_LEADER_INDEX) {
+ // TODO we don't know where the leader is, either because one wasn't provided or because
+ // we couldn't resolve its IP. We'll just send the client back so it retries and probably
+ // dies after too many attempts.
+ return null;
+ } else {
+ // TODO we currently always hit the leader, we probably don't need to except for writes
+ // and some reads.
+ return tablet.tabletServers.get(tablet.leaderIndex);
+ }
+ }
+ }
+
+ /**
+ * Checks whether or not an RPC can be retried once more
+ * @param rpc The RPC we're going to attempt to execute
+ * @return {@code true} if this RPC already had too many attempts,
+ * {@code false} otherwise (in which case it's OK to retry once more)
+ */
+ static boolean cannotRetryRequest(final KuduRpc<?> rpc) {
+ return rpc.deadlineTracker.timedOut() || rpc.attempt > MAX_RPC_ATTEMPTS;
+ }
+
+ /**
+ * Returns a {@link Deferred} containing an exception when an RPC couldn't
+ * succeed after too many attempts or if it already timed out.
+ * @param request The RPC that was retried too many times or timed out.
+ * @param cause What was cause of the last failed attempt, if known.
+ * You can pass {@code null} if the cause is unknown.
+ */
+ static <R> Deferred<R> tooManyAttemptsOrTimeout(final KuduRpc<R> request,
+ final KuduException cause) {
+ String message;
+ if (request.attempt > MAX_RPC_ATTEMPTS) {
+ message = "Too many attempts: ";
+ } else {
+ message = "RPC can not complete before timeout: ";
+ }
+ Status statusTimedOut = Status.TimedOut(message + request);
+ final Exception e = new NonRecoverableException(statusTimedOut, cause);
+ request.errback(e);
+ LOG.debug("Cannot continue with this RPC: {} because of: {}", request, message, e);
+ return Deferred.fromError(e);
+ }
+
+ /**
+ * Sends a getTableLocations RPC to the master to find the table's tablets.
+ * @param table table to lookup
+ * @param partitionKey can be null, if not we'll find the exact tablet that contains it
+ * @return Deferred to track the progress
+ */
+ private Deferred<Master.GetTableLocationsResponsePB> locateTablet(KuduTable table,
+ byte[] partitionKey) {
+ final boolean has_permit = acquireMasterLookupPermit();
+ String tableId = table.getTableId();
+ if (!has_permit) {
+ // If we failed to acquire a permit, it's worth checking if someone
+ // looked up the tablet we're interested in. Every once in a while
+ // this will save us a Master lookup.
+ RemoteTablet tablet = getTablet(tableId, partitionKey);
+ if (tablet != null && clientFor(tablet) != null) {
+ return Deferred.fromResult(null); // Looks like no lookup needed.
+ }
+ }
+ // Leave the end of the partition key range empty in order to pre-fetch tablet locations.
+ GetTableLocationsRequest rpc =
+ new GetTableLocationsRequest(masterTable, partitionKey, null, tableId);
+ rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ final Deferred<Master.GetTableLocationsResponsePB> d;
+
+ // If we know this is going to the master, check the master consensus
+ // configuration (as specified by 'masterAddresses' field) to determine and
+ // cache the current leader.
+ if (isMasterTable(tableId)) {
+ d = getMasterTableLocationsPB();
+ } else {
+ d = sendRpcToTablet(rpc);
+ }
+ d.addCallback(new MasterLookupCB(table, partitionKey));
+ if (has_permit) {
+ d.addBoth(new ReleaseMasterLookupPermit<Master.GetTableLocationsResponsePB>());
+ }
+ return d;
+ }
+
+ /**
+ * Update the master config: send RPCs to all config members, use the returned data to
+ * fill a {@link Master.GetTabletLocationsResponsePB} object.
+ * @return An initialized Deferred object to hold the response.
+ */
+ Deferred<Master.GetTableLocationsResponsePB> getMasterTableLocationsPB() {
+ final Deferred<Master.GetTableLocationsResponsePB> responseD = new Deferred<>();
+ final GetMasterRegistrationReceived received =
+ new GetMasterRegistrationReceived(masterAddresses, responseD);
+ for (HostAndPort hostAndPort : masterAddresses) {
+ Deferred<GetMasterRegistrationResponse> d;
+ // Note: we need to create a client for that host first, as there's a
+ // chicken and egg problem: since there is no source of truth beyond
+ // the master, the only way to get information about a master host is
+ // by making an RPC to that host.
+ TabletClient clientForHostAndPort = newMasterClient(hostAndPort);
+ if (clientForHostAndPort == null) {
+ String message = "Couldn't resolve this master's address " + hostAndPort.toString();
+ LOG.warn(message);
+ Status statusIOE = Status.IOError(message);
+ d = Deferred.fromError(new NonRecoverableException(statusIOE));
+ } else {
+ d = getMasterRegistration(clientForHostAndPort);
+ }
+ d.addCallbacks(received.callbackForNode(hostAndPort), received.errbackForNode(hostAndPort));
+ }
+ return responseD;
+ }
+
+
+ /**
+ * Get all or some tablets for a given table. This may query the master multiple times if there
+ * are a lot of tablets.
+ * This method blocks until it gets all the tablets.
+ * @param table the table to locate tablets from
+ * @param startPartitionKey where to start in the table, pass null to start at the beginning
+ * @param endPartitionKey where to stop in the table, pass null to get all the tablets until the
+ * end of the table
+ * @param deadline deadline in milliseconds for this method to finish
+ * @return a list of the tablets in the table, which can be queried for metadata about
+ * each tablet
+ * @throws Exception MasterErrorException if the table doesn't exist
+ */
+ List<LocatedTablet> syncLocateTable(KuduTable table,
+ byte[] startPartitionKey,
+ byte[] endPartitionKey,
+ long deadline) throws Exception {
+ return locateTable(table, startPartitionKey, endPartitionKey, deadline).join();
+ }
+
+ private Deferred<List<LocatedTablet>> loopLocateTable(final KuduTable table,
+ final byte[] startPartitionKey,
+ final byte[] endPartitionKey,
+ final List<LocatedTablet> ret,
+ final DeadlineTracker deadlineTracker) {
+ // We rely on the keys initially not being empty.
+ Preconditions.checkArgument(startPartitionKey == null || startPartitionKey.length > 0,
+ "use null for unbounded start partition key");
+ Preconditions.checkArgument(endPartitionKey == null || endPartitionKey.length > 0,
+ "use null for unbounded end partition key");
+
+ // The next partition key to look up. If null, then it represents
+ // the minimum partition key, If empty, it represents the maximum key.
+ byte[] partitionKey = startPartitionKey;
+ String tableId = table.getTableId();
+
+ // Continue while the partition key is the minimum, or it is not the maximum
+ // and it is less than the end partition key.
+ while (partitionKey == null ||
+ (partitionKey.length > 0 &&
+ (endPartitionKey == null || Bytes.memcmp(partitionKey, endPartitionKey) < 0))) {
+ byte[] key = partitionKey == null ? EMPTY_ARRAY : partitionKey;
+ RemoteTablet tablet = getTablet(tableId, key);
+ if (tablet != null) {
+ ret.add(new LocatedTablet(tablet));
+ partitionKey = tablet.getPartition().getPartitionKeyEnd();
+ continue;
+ }
+
+ Map.Entry<byte[], byte[]> nonCoveredRange = getNonCoveredRange(tableId, key);
+ if (nonCoveredRange != null) {
+ partitionKey = nonCoveredRange.getValue();
+ continue;
+ }
+
+ if (deadlineTracker.timedOut()) {
+ Status statusTimedOut = Status.TimedOut("Took too long getting the list of tablets, " +
+ deadlineTracker);
+ return Deferred.fromError(new NonRecoverableException(statusTimedOut));
+ }
+
+ // If the partition key location isn't cached, and the request hasn't timed out,
+ // then kick off a new tablet location lookup and try again when it completes.
+ // When lookup completes, the tablet (or non-covered range) for the next
+ // partition key will be located and added to the client's cache.
+ final byte[] lookupKey = partitionKey;
+ return locateTablet(table, key).addCallbackDeferring(
+ new Callback<Deferred<List<LocatedTablet>>, GetTableLocationsResponsePB>() {
+ @Override
+ public Deferred<List<LocatedTablet>> call(GetTableLocationsResponsePB resp) {
+ return loopLocateTable(table, lookupKey, endPartitionKey, ret, deadlineTracker);
+ }
+ @Override
+ public String toString() {
+ return "LoopLocateTableCB";
+ }
+ });
+ }
+
+ return Deferred.fromResult(ret);
+ }
+
+ /**
+ * Get all or some tablets for a given table. This may query the master multiple times if there
+ * are a lot of tablets.
+ * @param table the table to locate tablets from
+ * @param startPartitionKey where to start in the table, pass null to start at the beginning
+ * @param endPartitionKey where to stop in the table, pass null to get all the tablets until the
+ * end of the table
+ * @param deadline max time spent in milliseconds for the deferred result of this method to
+ * get called back, if deadline is reached, the deferred result will get erred back
+ * @return a deferred object that yields a list of the tablets in the table, which can be queried
+ * for metadata about each tablet
+ * @throws Exception MasterErrorException if the table doesn't exist
+ */
+ Deferred<List<LocatedTablet>> locateTable(final KuduTable table,
+ final byte[] startPartitionKey,
+ final byte[] endPartitionKey,
+ long deadline) {
+ final List<LocatedTablet> ret = Lists.newArrayList();
+ final DeadlineTracker deadlineTracker = new DeadlineTracker();
+ deadlineTracker.setDeadline(deadline);
+ return loopLocateTable(table, startPartitionKey, endPartitionKey, ret, deadlineTracker);
+ }
+
+ /**
+ * We're handling a tablet server that's telling us it doesn't have the tablet we're asking for.
+ * We're in the context of decode() meaning we need to either callback or retry later.
+ */
+ <R> void handleTabletNotFound(final KuduRpc<R> rpc, KuduException ex, TabletClient server) {
+ invalidateTabletCache(rpc.getTablet(), server);
+ handleRetryableError(rpc, ex);
+ }
+
+ /**
+ * A tablet server is letting us know that it isn't the specified tablet's leader in response
+ * a RPC, so we need to demote it and retry.
+ */
+ <R> void handleNotLeader(final KuduRpc<R> rpc, KuduException ex, TabletClient server) {
+ rpc.getTablet().demoteLeader(server);
+ handleRetryableError(rpc, ex);
+ }
+
+ <R> void handleRetryableError(final KuduRpc<R> rpc, KuduException ex) {
+ // TODO we don't always need to sleep, maybe another replica can serve this RPC.
+ delayedSendRpcToTablet(rpc, ex);
+ }
+
+ private <R> void delayedSendRpcToTablet(final KuduRpc<R> rpc, KuduException ex) {
+ // Here we simply retry the RPC later. We might be doing this along with a lot of other RPCs
+ // in parallel. Asynchbase does some hacking with a "probe" RPC while putting the other ones
+ // on hold but we won't be doing this for the moment. Regions in HBase can move a lot,
+ // we're not expecting this in Kudu.
+ final class RetryTimer implements TimerTask {
+ public void run(final Timeout timeout) {
+ sendRpcToTablet(rpc);
+ }
+ }
+ long sleepTime = getSleepTimeForRpc(rpc);
+ if (cannotRetryRequest(rpc) || rpc.deadlineTracker.wouldSleepingTimeout(sleepTime)) {
+ tooManyAttemptsOrTimeout(rpc, ex);
+ // Don't let it retry.
+ return;
+ }
+ newTimeout(new RetryTimer(), sleepTime);
+ }
+
+ /**
+ * Remove the tablet server from the RemoteTablet's locations. Right now nothing is removing
+ * the tablet itself from the caches.
+ */
+ private void invalidateTabletCache(RemoteTablet tablet, TabletClient server) {
+ LOG.info("Removing server " + server.getUuid() + " from this tablet's cache " +
+ tablet.getTabletIdAsString());
+ tablet.removeTabletClient(server);
+ }
+
+ /** Callback executed when a master lookup completes. */
+ private final class MasterLookupCB implements Callback<Object,
+ Master.GetTableLocationsResponsePB> {
+ final KuduTable table;
+ private final byte[] partitionKey;
+ MasterLookupCB(KuduTable table, byte[] partitionKey) {
+ this.table = table;
+ this.partitionKey = partitionKey;
+ }
+ public Object call(final GetTableLocationsResponsePB response) {
+ if (response.hasError()) {
+ if (response.getError().getCode() == Master.MasterErrorPB.Code.TABLET_NOT_RUNNING) {
+ // Keep a note that the table exists but at least one tablet is not yet running.
+ LOG.debug("Table {} has a non-running tablet", table.getName());
+ tablesNotServed.add(table.getTableId());
+ } else {
+ Status status = Status.fromMasterErrorPB(response.getError());
+ return new NonRecoverableException(status);
+ }
+ } else {
+ try {
+ discoverTablets(table, response.getTabletLocationsList());
+ } catch (NonRecoverableException e) {
+ return e;
+ }
+ if (partitionKey != null) {
+ discoverNonCoveredRangePartitions(table.getTableId(), partitionKey,
+ response.getTabletLocationsList());
+ }
+ }
+ return null;
+ }
+ public String toString() {
+ return "get tablet locations from the master for table " + table.getName();
+ }
+ }
+
+ boolean acquireMasterLookupPermit() {
+ try {
+ // With such a low timeout, the JVM may chose to spin-wait instead of
+ // de-scheduling the thread (and causing context switches and whatnot).
+ return masterLookups.tryAcquire(5, MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt(); // Make this someone else's problem.
+ return false;
+ }
+ }
+
+ /**
+ * Releases a master lookup permit that was acquired.
+ * @see #acquireMasterLookupPermit
+ */
+ void releaseMasterLookupPermit() {
+ masterLookups.release();
+ }
+
+ @VisibleForTesting
+ void discoverTablets(KuduTable table, List<Master.TabletLocationsPB> locations)
+ throws NonRecoverableException {
+ String tableId = table.getTableId();
+ String tableName = table.getName();
+
+ // Doing a get first instead of putIfAbsent to avoid creating unnecessary CSLMs because in
+ // the most common case the table should already be present
+ ConcurrentSkipListMap<byte[], RemoteTablet> tablets = tabletsCache.get(tableId);
+ if (tablets == null) {
+ tablets = new ConcurrentSkipListMap<>(Bytes.MEMCMP);
+ ConcurrentSkipListMap<byte[], RemoteTablet> oldTablets =
+ tabletsCache.putIfAbsent(tableId, tablets);
+ if (oldTablets != null) {
+ tablets = oldTablets;
+ }
+ }
+
+ for (Master.TabletLocationsPB tabletPb : locations) {
+ // Early creating the tablet so that it parses out the pb
+ RemoteTablet rt = createTabletFromPb(tableId, tabletPb);
+ Slice tabletId = rt.tabletId;
+
+ // If we already know about this one, just refresh the locations
+ RemoteTablet currentTablet = tablet2client.get(tabletId);
+ if (currentTablet != null) {
+ currentTablet.refreshTabletClients(tabletPb);
+ continue;
+ }
+
+ // Putting it here first doesn't make it visible because tabletsCache is always looked up
+ // first.
+ RemoteTablet oldRt = tablet2client.putIfAbsent(tabletId, rt);
+ if (oldRt != null) {
+ // someone beat us to it
+ continue;
+ }
+ LOG.info("Discovered tablet {} for table '{}' with partition {}",
+ tabletId.toString(Charset.defaultCharset()), tableName, rt.getPartition());
+ rt.refreshTabletClients(tabletPb);
+ // This is making this tablet available
+ // Even if two clients were racing in this method they are putting the same RemoteTablet
+ // with the same start key in the CSLM in the end
+ tablets.put(rt.getPartition().getPartitionKeyStart(), rt);
+ }
+ }
+
+ private void discoverNonCoveredRangePartitions(String tableId,
+ byte[] partitionKey,
+ List<Master.TabletLocationsPB> locations) {
+ NonCoveredRangeCache nonCoveredRanges = nonCoveredRangeCaches.get(tableId);
+ if (nonCoveredRanges == null) {
+ nonCoveredRanges = new NonCoveredRangeCache();
+ NonCoveredRangeCache oldCache = nonCoveredRangeCaches.putIfAbsent(tableId, nonCoveredRanges);
+ if (oldCache != null) {
+ nonCoveredRanges = oldCache;
+ }
+ }
+
+ // If there are no locations, then the table has no tablets. This is
+ // guaranteed because we never set an upper bound on the GetTableLocations
+ // request, and the master will always return the tablet *before* the start
+ // of the request, if the start key falls in a non-covered range (see the
+ // comment on GetTableLocationsResponsePB in master.proto).
+ if (locations.isEmpty()) {
+ nonCoveredRanges.addNonCoveredRange(EMPTY_ARRAY, EMPTY_ARRAY);
+ return;
+ }
+
+ // If the first tablet occurs after the requested partition key,
+ // then there is an initial non-covered range.
+ byte[] firstStartKey = locations.get(0).getPartition().getPartitionKeyStart().toByteArray();
+ if (Bytes.memcmp(partitionKey, firstStartKey) < 0) {
+ nonCoveredRanges.addNonCoveredRange(EMPTY_ARRAY, firstStartKey);
+ }
+
+ byte[] previousEndKey = null;
+ for (Master.TabletLocationsPB location : locations) {
+ byte[] startKey = location.getPartition().getPartitionKeyStart().toByteArray();
+
+ // Check if there is a non-covered range between this tablet and the previous.
+ if (previousEndKey != null && Bytes.memcmp(previousEndKey, startKey) < 0) {
+ nonCoveredRanges.addNonCoveredRange(previousEndKey, startKey);
+ }
+ previousEndKey = location.getPartition().getPartitionKeyEnd().toByteArray();
+ }
+
+ if (previousEndKey.length > 0 && Bytes.memcmp(previousEndKey, partitionKey) <= 0) {
+ // This happens if the partition key falls in a non-covered range that
+ // is unbounded (to the right).
+ nonCoveredRanges.addNonCoveredRange(previousEndKey, EMPTY_ARRAY);
+ }
+ }
+
+ RemoteTablet createTabletFromPb(String tableId, Master.TabletLocationsPB tabletPb) {
+ Partition partition = ProtobufHelper.pbToPartition(tabletPb.getPartition());
+ Slice tabletId = new Slice(tabletPb.getTabletId().toByteArray());
+ return new RemoteTablet(tableId, tabletId, partition);
+ }
+
+ /**
+ * Gives the tablet's ID for the table ID and partition key.
+ * In the future there will be multiple tablets and this method will find the right one.
+ * @param tableId table to find the tablet for
+ * @return a tablet ID as a slice or null if not found
+ */
+ RemoteTablet getTablet(String tableId, byte[] partitionKey) {
+ ConcurrentSkipListMap<byte[], RemoteTablet> tablets = tabletsCache.get(tableId);
+
+ if (tablets == null) {
+ return null;
+ }
+
+ // We currently only have one master tablet.
+ if (isMasterTable(tableId)) {
+ if (tablets.firstEntry() == null) {
+ return null;
+ }
+ return tablets.firstEntry().getValue();
+ }
+
+ Map.Entry<byte[], RemoteTablet> tabletPair = tablets.floorEntry(partitionKey);
+
+ if (tabletPair == null) {
+ return null;
+ }
+
+ Partition partition = tabletPair.getValue().getPartition();
+
+ // If the partition is not the end partition, but it doesn't include the key
+ // we are looking for, then we have not yet found the correct tablet.
+ if (!partition.isEndPartition()
+ && Bytes.memcmp(partitionKey, partition.getPartitionKeyEnd()) >= 0) {
+ return null;
+ }
+
+ return tabletPair.getValue();
+ }
+
+ /**
+ * Returns a deferred containing the located tablet which covers the partition key in the table.
+ * @param table the table
+ * @param partitionKey the partition key of the tablet to look up in the table
+ * @param deadline deadline in milliseconds for this lookup to finish
+ * @return a deferred containing the located tablet
+ */
+ Deferred<LocatedTablet> getTabletLocation(final KuduTable table,
+ final byte[] partitionKey,
+ long deadline) {
+ // Locate the tablets at the partition key by locating all tablets between
+ // the partition key (inclusive), and the incremented partition key (exclusive).
+
+ Deferred<List<LocatedTablet>> locatedTablets;
+ if (partitionKey.length == 0) {
+ locatedTablets = locateTable(table, null, new byte[] { 0x00 }, deadline);
+ } else {
+ locatedTablets = locateTable(table, partitionKey,
+ Arrays.copyOf(partitionKey, partitionKey.length + 1), deadline);
+ }
+
+ // Then pick out the single tablet result from the list.
+ return locatedTablets.addCallbackDeferring(
+ new Callback<Deferred<LocatedTablet>, List<LocatedTablet>>() {
+ @Override
+ public Deferred<LocatedTablet> call(List<LocatedTablet> tablets) {
+ Preconditions.checkArgument(tablets.size() <= 1,
+ "found more than one tablet for a single partition key");
+ if (tablets.size() == 0) {
+ Map.Entry<byte[], byte[]> nonCoveredRange =
+ nonCoveredRangeCaches.get(table.getTableId()).getNonCoveredRange(partitionKey);
+ return Deferred.fromError(new NonCoveredRangeException(nonCoveredRange.getKey(),
+ nonCoveredRange.getValue()));
+ }
+ return Deferred.fromResult(tablets.get(0));
+ }
+ });
+ }
+
+ /**
+ * Returns the non-covered range partition containing the {@code partitionKey} in
+ * the table, or null if there is no known non-covering range for the partition key.
+ * @param tableId of the table
+ * @param partitionKey to lookup
+ * @return the non-covering partition range, or {@code null}
+ */
+ Map.Entry<byte[], byte[]> getNonCoveredRange(String tableId, byte[] partitionKey) {
+ if (isMasterTable(tableId)) {
+ throw new IllegalArgumentException("No non-covering range partitions for the master");
+ }
+ NonCoveredRangeCache nonCoveredRangeCache = nonCoveredRangeCaches.get(tableId);
+ if (nonCoveredRangeCache == null) return null;
+
+ return nonCoveredRangeCache.getNonCoveredRange(partitionKey);
+ }
+
+ /**
+ * Retrieve the master registration (see {@link GetMasterRegistrationResponse}
+ * for a replica.
+ * @param masterClient An initialized client for the master replica.
+ * @return A Deferred object for the master replica's current registration.
+ */
+ Deferred<GetMasterRegistrationResponse> getMasterRegistration(TabletClient masterClient) {
+ GetMasterRegistrationRequest rpc = new GetMasterRegistrationRequest(masterTable);
+ rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
+ Deferred<GetMasterRegistrationResponse> d = rpc.getDeferred();
+ rpc.attempt++;
+ masterClient.sendRpc(rpc);
+ return d;
+ }
+
+ /**
+ * If a live client already exists for the specified master server, returns that client;
+ * otherwise, creates a new client for the specified master server.
+ * @param masterHostPort The RPC host and port for the master server.
+ * @return A live and initialized client for the specified master server.
+ */
+ TabletClient newMasterClient(HostAndPort masterHostPort) {
+ String ip = getIP(masterHostPort.getHostText());
+ if (ip == null) {
+ return null;
+ }
+ // We should pass a UUID here but we have a chicken and egg problem, we first need to
+ // communicate with the masters to find out about them, and that's what we're trying to do.
+ // The UUID is used for logging, so instead we're passing the "master table name" followed by
+ // host and port which is enough to identify the node we're connecting to.
+ return newClient(MASTER_TABLE_NAME_PLACEHOLDER + " - " + masterHostPort.toString(),
+ ip, masterHostPort.getPort());
+ }
+
+ TabletClient newClient(String uuid, final String host, final int port) {
+ final String hostport = host + ':' + port;
+ TabletClient client;
+ SocketChannel chan;
+ synchronized (ip2client) {
+ client = ip2client.get(hostport);
+ if (client != null && client.isAlive()) {
+ return client;
+ }
+ final TabletClientPipeline pipeline = new TabletClientPipeline();
+ client = pipeline.init(uuid, host, port);
+ chan = channelFactory.newChannel(pipeline);
+ ip2client.put(hostport, client); // This is guaranteed to return null.
+
+ // The client2tables map is assumed to contain `client` after it is published in ip2client.
+ this.client2tablets.put(client, new ArrayList<RemoteTablet>());
+ }
+ final SocketChannelConfig config = chan.getConfig();
+ config.setConnectTimeoutMillis(5000);
+ config.setTcpNoDelay(true);
+ // Unfortunately there is no way to override the keep-alive timeout in
+ // Java since the JRE doesn't expose any way to call setsockopt() with
+ // TCP_KEEPIDLE. And of course the default timeout is >2h. Sigh.
+ config.setKeepAlive(true);
+ chan.connect(new InetSocketAddress(host, port)); // Won't block.
+ return client;
+ }
+
+ /**
+ * Invokes {@link #shutdown()} and waits for the configured admin timeout. This method returns
+ * void, so consider invoking shutdown directly if there's a need to handle dangling RPCs.
+ * @throws Exception if an error happens while closing the connections
+ */
+ @Override
+ public void close() throws Exception {
+ shutdown().join(defaultAdminOperationTimeoutMs);
+ }
+
+ /**
+ * Performs a graceful shutdown of this instance.
+ * <p>
+ * <ul>
+ * <li>{@link AsyncKuduSession#flush Flushes} all buffered edits.</li>
+ * <li>Cancels all the other requests.</li>
+ * <li>Terminates all connections.</li>
+ * <li>Releases all other resources.</li>
+ * </ul>
+ * <strong>Not calling this method before losing the last reference to this
+ * instance may result in data loss and other unwanted side effects</strong>
+ * @return A {@link Deferred}, whose callback chain will be invoked once all
+ * of the above have been done. If this callback chain doesn't fail, then
+ * the clean shutdown will be successful, and all the data will be safe on
+ * the Kudu side. In case of a failure (the "errback" is invoked) you will have
+ * to open a new AsyncKuduClient if you want to retry those operations.
+ * The Deferred doesn't actually hold any content.
+ */
+ public Deferred<ArrayList<Void>> shutdown() {
+ checkIsClosed();
+ closed = true;
+ // This is part of step 3. We need to execute this in its own thread
+ // because Netty gets stuck in an infinite loop if you try to shut it
+ // down from within a thread of its own thread pool. They don't want
+ // to fix this so as a workaround we always shut Netty's thread pool
+ // down from another thread.
+ final class ShutdownThread extends Thread {
+ ShutdownThread() {
+ super("AsyncKuduClient@" + AsyncKuduClient.super.hashCode() + " shutdown");
+ }
+ public void run() {
+ // This terminates the Executor.
+ channelFactory.releaseExternalResources();
+ }
+ }
+
+ // 3. Release all other resources.
+ final class ReleaseResourcesCB implements Callback<ArrayList<Void>, ArrayList<Void>> {
+ public ArrayList<Void> call(final ArrayList<Void> arg) {
+ LOG.debug("Releasing all remaining resources");
+ timer.stop();
+ new ShutdownThread().start();
+ return arg;
+ }
+ public String toString() {
+ return "release resources callback";
+ }
+ }
+
+ // 2. Terminate all connections.
+ final class DisconnectCB implements Callback<Deferred<ArrayList<Void>>,
+ ArrayList<List<OperationResponse>>> {
+ public Deferred<ArrayList<Void>> call(ArrayList<List<OperationResponse>> ignoredResponses) {
+ return disconnectEverything().addCallback(new ReleaseResourcesCB());
+ }
+ public String toString() {
+ return "disconnect callback";
+ }
+ }
+
+ // 1. Flush everything.
+ // Notice that we do not handle the errback, if there's an exception it will come straight out.
+ return closeAllSessions().addCallbackDeferring(new DisconnectCB());
+ }
+
+ private void checkIsClosed() {
+ if (closed) {
+ throw new IllegalStateException("Cannot proceed, the client has already been closed");
+ }
+ }
+
+ private Deferred<ArrayList<List<OperationResponse>>> closeAllSessions() {
+ // We create a copy because AsyncKuduSession.close will call removeSession which would get us a
+ // concurrent modification during the iteration.
+ Set<AsyncKuduSession> copyOfSessions;
+ synchronized (sessions) {
+ copyOfSessions = new HashSet<AsyncKuduSession>(sessions);
+ }
+ if (sessions.isEmpty()) {
+ return Deferred.fromResult(null);
+ }
+ // Guaranteed that we'll have at least one session to close.
+ List<Deferred<List<OperationResponse>>> deferreds = new ArrayList<>(copyOfSessions.size());
+ for (AsyncKuduSession session : copyOfSessions ) {
+ deferreds.add(session.close());
+ }
+
+ return Deferred.group(deferreds);
+ }
+
+ /**
+ * Closes every socket, which will also cancel all the RPCs in flight.
+ */
+ private Deferred<ArrayList<Void>> disconnectEverything() {
+ ArrayList<Deferred<Void>> deferreds =
+ new ArrayList<Deferred<Void>>(2);
+ HashMap<String, TabletClient> ip2client_copy;
+ synchronized (ip2client) {
+ // Make a local copy so we can shutdown every Tablet Server clients
+ // without hold the lock while we iterate over the data structure.
+ ip2client_copy = new HashMap<String, TabletClient>(ip2client);
+ }
+
+ for (TabletClient ts : ip2client_copy.values()) {
+ deferreds.add(ts.shutdown());
+ }
+ final int size = deferreds.size();
+ return Deferred.group(deferreds).addCallback(
+ new Callback<ArrayList<Void>, ArrayList<Void>>() {
+ public ArrayList<Void> call(final ArrayList<Void> arg) {
+ // Normally, now that we've shutdown() every client, all our caches should
+ // be empty since each shutdown() generates a DISCONNECTED event, which
+ // causes TabletClientPipeline to call removeClientFromIpCache().
+ HashMap<String, TabletClient> logme = null;
+ synchronized (ip2client) {
+ if (!ip2client.isEmpty()) {
+ logme = new HashMap<String, TabletClient>(ip2client);
+ }
+ }
+ if (logme != null) {
+ // Putting this logging statement inside the synchronized block
+ // can lead to a deadlock, since HashMap.toString() is going to
+ // call TabletClient.toString() on each entry, and this locks the
+ // client briefly. Other parts of the code lock clients first and
+ // the ip2client HashMap second, so this can easily deadlock.
+ LOG.error("Some clients are left in the client cache and haven't"
+ + " been cleaned up: " + logme);
+ }
+ return arg;
+ }
+
+ public String toString() {
+ return "wait " + size + " TabletClient.shutdown()";
+ }
+ });
+ }
+
+ /**
+ * Blocking call.
+ * Performs a slow search of the IP used by the given client.
+ * <p>
+ * This is needed when we're trying to find the IP of the client before its
+ * channel has successfully connected, because Netty's API offers no way of
+ * retrieving the IP of the remote peer until we're connected to it.
+ * @param client The client we want the IP of.
+ * @return The IP of the client, or {@code null} if we couldn't find it.
+ */
+ private InetSocketAddress slowSearchClientIP(final TabletClient client) {
+ String hostport = null;
+ synchronized (ip2client) {
+ for (final Map.Entry<String, TabletClient> e : ip2client.entrySet()) {
+ if (e.getValue() == client) {
+ hostport = e.getKey();
+ break;
+ }
+ }
+ }
+
+ if (hostport == null) {
+ HashMap<String, TabletClient> copy;
+ synchronized (ip2client) {
+ copy = new HashMap<String, TabletClient>(ip2client);
+ }
+ LOG.error("WTF? Should never happen! Couldn't find " + client
+ + " in " + copy);
+ return null;
+ }
+ final int colon = hostport.indexOf(':', 1);
+ if (colon < 1) {
+ LOG.error("WTF? Should never happen! No `:' found in " + hostport);
+ return null;
+ }
+ final String host = getIP(hostport.substring(0, colon));
+ if (host == null) {
+ // getIP will print the reason why, there's nothing else we can do.
+ return null;
+ }
+
+ int port;
+ try {
+ port = parsePortNumber(hostport.substring(colon + 1,
+ hostport.length()));
+ } catch (NumberFormatException e) {
+ LOG.error("WTF? Should never happen! Bad port in " + hostport, e);
+ return null;
+ }
+ return new InetSocketAddress(host, port);
+ }
+
+ /**
+ * Removes the given client from the `ip2client` cache.
+ * @param client The client for which we must clear the ip cache
+ * @param remote The address of the remote peer, if known, or null
+ */
+ private void removeClientFromIpCache(final TabletClient client,
+ final SocketAddress remote) {
+
+ if (remote == null) {
+ return; // Can't continue without knowing the remote address.
+ }
+
+ String hostport;
+ if (remote instanceof InetSocketAddress) {
+ final InetSocketAddress sock = (InetSocketAddress) remote;
+ final InetAddress addr = sock.getAddress();
+ if (addr == null) {
+ LOG.error("WTF? Unresolved IP for " + remote
+ + ". This shouldn't happen.");
+ return;
+ } else {
+ hostport = addr.getHostAddress() + ':' + sock.getPort();
+ }
+ } else {
+ LOG.error("WTF? Found a non-InetSocketAddress remote: " + remote
+ + ". This shouldn't happen.");
+ return;
+ }
+
+ TabletClient old;
+ synchronized (ip2client) {
+ old = ip2client.remove(hostport);
+ }
+ LOG.debug("Removed from IP cache: {" + hostport + "} -> {" + client + "}");
+ if (old == null) {
+ // Currently we're seeing this message when masters are disconnected and the hostport we got
+ // above is different than the one the user passes (that we use to populate ip2client). At
+ // worst this doubles the entries for masters, which has an insignificant impact.
+ // TODO When fixed, make this a WARN again.
+ LOG.trace("When expiring " + client + " from the client cache (host:port="
+ + hostport + "), it was found that there was no entry"
+ + " corresponding to " + remote + ". This shouldn't happen.");
+ }
+ }
+
+ /**
+ * Call this method after encountering an error connecting to a tablet server so that we stop
+ * considering it a leader for the tablets it serves.
+ * @param client tablet server to use for demotion
+ */
+ void demoteAsLeaderForAllTablets(final TabletClient client) {
+ ArrayList<RemoteTablet> tablets = client2tablets.get(client);
+ if (tablets != null) {
+ // Make a copy so we don't need to synchronize on it while iterating.
+ RemoteTablet[] tablets_copy;
+ synchronized (tablets) {
+ tablets_copy = tablets.toArray(new RemoteTablet[tablets.size()]);
+ }
+ for (final RemoteTablet remoteTablet : tablets_copy) {
+ // It will be a no-op if it's not already a leader.
+ remoteTablet.demoteLeader(client);
+ }
+ }
+ }
+
+ private boolean isMasterTable(String tableId) {
+ // Checking that it's the same instance so there's absolutely no chance of confusing the master
+ // 'table' for a user one.
+ return MASTER_TABLE_NAME_PLACEHOLDER == tableId;
+ }
+
+ private final class TabletClientPipeline extends DefaultChannelPipeline {
+
+ private final Logger log = LoggerFactory.getLogger(TabletClientPipeline.class);
+ /**
+ * Have we already disconnected?.
+ * We use this to avoid doing the cleanup work for the same client more
+ * than once, even if we get multiple events indicating that the client
+ * is no longer connected to the TabletServer (e.g. DISCONNECTED, CLOSED).
+ * No synchronization needed as this is always accessed from only one
+ * thread at a time (equivalent to a non-shared state in a Netty handler).
+ */
+ private boolean disconnected = false;
+
+ TabletClient init(String uuid, String host, int port) {
+ final TabletClient client = new TabletClient(AsyncKuduClient.this, uuid, host, port);
+ if (defaultSocketReadTimeoutMs > 0) {
+ super.addLast("timeout-handler",
+ new ReadTimeoutHandler(timer,
+ defaultSocketReadTimeoutMs,
+ TimeUnit.MILLISECONDS));
+ }
+ super.addLast("kudu-handler", client);
+
+ return client;
+ }
+
+ @Override
+ public void sendDownstream(final ChannelEvent event) {
+ if (event instanceof ChannelStateEvent) {
+ handleDisconnect((ChannelStateEvent) event);
+ }
+ super.sendDownstream(event);
+ }
+
+ @Override
+ public void sendUpstream(final ChannelEvent event) {
+ if (event instanceof ChannelStateEvent) {
+ handleDisconnect((ChannelStateEvent) event);
+ }
+ super.sendUpstream(event);
+ }
+
+ private void handleDisconnect(final ChannelStateEvent state_event) {
+ if (disconnected) {
+ return;
+ }
+ switch (state_event.getState()) {
+ case OPEN:
+ if (state_event.getValue() == Boolean.FALSE) {
+ break; // CLOSED
+ }
+ return;
+ case CONNECTED:
+ if (state_event.getValue() == null) {
+ break; // DISCONNECTED
+ }
+ return;
+ default:
+ return; // Not an event we're interested in, ignore it.
+ }
+
+ disconnected = true; // So we don't clean up the same client twice.
+ try {
+ final TabletClient client = super.get(TabletClient.class);
+ SocketAddress remote = super.getChannel().getRemoteAddress();
+ // At this point Netty gives us no easy way to access the
+ // SocketAddress of the peer we tried to connect to. This
+ // kinda sucks but I couldn't find an easier way.
+ if (remote == null) {
+ remote = slowSearchClientIP(client);
+ }
+
+ synchronized (client) {
+ removeClientFromIpCache(client, remote);
+ }
+ } catch (Exception e) {
+ log.error("Uncaught exception when handling a disconnection of " + getChannel(), e);
+ }
+ }
+
+ }
+
+ /**
+ * Gets a hostname or an IP address and returns the textual representation
+ * of the IP address.
+ * <p>
+ * <strong>This method can block</strong> as there is no API for
+ * asynchronous DNS resolution in the JDK.
+ * @param host The hostname to resolve.
+ * @return The IP address associated with the given hostname,
+ * or {@code null} if the address couldn't be resolved.
+ */
+ private static String getIP(final String host) {
+ final long start = System.nanoTime();
+ try {
+ final String ip = InetAddress.getByName(host).getHostAddress();
+ final long latency = System.nanoTime() - start;
+ if (latency > 500000/*ns*/ && LOG.isDebugEnabled()) {
+ LOG.debug("Resolved IP of `" + host + "' to "
+ + ip + " in " + latency + "ns");
+ } else if (latency >= 3000000/*ns*/) {
+ LOG.warn("Slow DNS lookup! Resolved IP of `" + host + "' to "
+ + ip + " in " + latency + "ns");
+ }
+ return ip;
+ } catch (UnknownHostException e) {
+ LOG.error("Failed to resolve the IP of `" + host + "' in "
+ + (System.nanoTime() - start) + "ns");
+ return null;
+ }
+ }
+
+ /**
+ * Parses a TCP port number from a string.
+ * @param portnum The string to parse.
+ * @return A strictly positive, validated port number.
+ * @throws NumberFormatException if the string couldn't be parsed as an
+ * integer or if the value was outside of the range allowed for TCP ports.
+ */
+ private static int parsePortNumber(final String portnum)
+ throws NumberFormatException {
+ final int port = Integer.parseInt(portnum);
+ if (port <= 0 || port > 65535) {
+ throw new NumberFormatException(port == 0 ? "port is zero" :
+ (port < 0 ? "port is negative: "
+ : "port is too large: ") + port);
+ }
+ return port;
+ }
+
+ void newTimeout(final TimerTask task, final long timeout_ms) {
+ try {
+ timer.newTimeout(task, timeout_ms, MILLISECONDS);
+ } catch (IllegalStateException e) {
+ // This can happen if the timer fires just before shutdown()
+ // is called from another thread, and due to how threads get
+ // scheduled we tried to call newTimeout() after timer.stop().
+ LOG.warn("Failed to schedule timer."
+ + " Ignore this if we're shutting down.", e);
+ }
+ }
+
+ /**
+ * This class encapsulates the information regarding a tablet and its locations.
+ *
+ * Leader failover mechanism:
+ * When we get a complete peer list from the master, we place the leader in the first
+ * position of the tabletServers array. When we detect that it isn't the leader anymore (in
+ * TabletClient), we demote it and set the next TS in the array as the leader. When the RPC
+ * gets retried, it will use that TS since we always pick the leader.
+ *
+ * If that TS turns out to not be the leader, we will demote it and promote the next one, retry.
+ * When we hit the end of the list, we set the leaderIndex to NO_LEADER_INDEX which forces us
+ * to fetch the tablet locations from the master. We'll repeat this whole process until a RPC
+ * succeeds.
+ *
+ * Subtleties:
+ * We don't keep track of a TS after it disconnects (via removeTabletClient), so if we
+ * haven't contacted one for 10 seconds (socket timeout), it will be removed from the list of
+ * tabletServers. This means that if the leader fails, we only have one other TS to "promote"
+ * or maybe none at all. This is partly why we then set leaderIndex to NO_LEADER_INDEX.
+ *
+ * The effect of treating a TS as the new leader means that the Scanner will also try to hit it
+ * with requests. It's currently unclear if that's a good or a bad thing.
+ *
+ * Unlike the C++ client, we don't short-circuit the call to the master if it isn't available.
+ * This means that after trying all the peers to find the leader, we might get stuck waiting on
+ * a reachable master.
+ */
+ public class RemoteTablet implements Comparable<RemoteTablet> {
+
+ private static final int NO_LEADER_INDEX = -1;
+ private final String tableId;
+ private final Slice tabletId;
+ @GuardedBy("tabletServers")
+ private final ArrayList<TabletClient> tabletServers = new ArrayList<>();
+ private final AtomicReference<List<LocatedTablet.Replica>> replicas =
+ new AtomicReference(ImmutableList.of());
+ private final Partition partition;
+ private int leaderIndex = NO_LEADER_INDEX;
+
+ RemoteTablet(String tableId, Slice tabletId, Partition partition) {
+ this.tabletId = tabletId;
+ this.tableId = tableId;
+ this.partition = partition;
+ }
+
+ void refreshTabletClients(Master.TabletLocationsPB tabletLocations) throws NonRecoverableException {
+
+ synchronized (tabletServers) { // TODO not a fat lock with IP resolving in it
+ tabletServers.clear();
+ leaderIndex = NO_LEADER_INDEX;
+ List<UnknownHostException> lookupExceptions =
+ new ArrayList<>(tabletLocations.getReplicasCount());
+ for (Master.TabletLocationsPB.ReplicaPB replica : tabletLocations.getReplicasList()) {
+
+ List<Common.HostPortPB> addresses = replica.getTsInfo().getRpcAddressesList();
+ if (addresses.isEmpty()) {
+ LOG.warn("Tablet server for tablet " + getTabletIdAsString() + " doesn't have any " +
+ "address");
+ continue;
+ }
+ byte[] buf = Bytes.get(replica.getTsInfo().getPermanentUuid());
+ String uuid = Bytes.getString(buf);
+ // from meta_cache.cc
+ // TODO: if the TS advertises multiple host/ports, pick the right one
+ // based on some kind of policy. For now just use the first always.
+ try {
+ addTabletClient(uuid, addresses.get(0).getHost(), addresses.get(0).getPort(),
+ replica.getRole().equals(Metadata.RaftPeerPB.Role.LEADER));
+ } catch (UnknownHostException ex) {
+ lookupExceptions.add(ex);
+ }
+ }
+
+ if (leaderIndex == NO_LEADER_INDEX) {
+ LOG.warn("No leader provided for tablet {}", getTabletIdAsString());
+ }
+
+ // If we found a tablet that doesn't contain a single location that we can resolve, there's
+ // no point in retrying.
+ if (!lookupExceptions.isEmpty() &&
+ lookupExceptions.size() == tabletLocations.getReplicasCount()) {
+ Status statusIOE = Status.IOError("Couldn't find any valid locations, exceptions: " +
+ lookupExceptions);
+ throw new NonRecoverableException(statusIOE);
+ }
+
+ }
+
+ ImmutableList.Builder<LocatedTablet.Replica> replicasBuilder = new ImmutableList.Builder<>();
+ for (Master.TabletLocationsPB.ReplicaPB replica : tabletLocations.getReplicasList()) {
+ replicasBuilder.add(new LocatedTablet.Replica(replica));
+ }
+ replicas.set(replicasBuilder.build());
+ }
+
+ // Must be
<TRUNCATED>
[23/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
new file mode 100644
index 0000000..2b2cf64
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+import com.google.common.net.HostAndPort;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+import org.kududb.ColumnSchema;
+import org.kududb.Common;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+@InterfaceAudience.Private
+public class ProtobufHelper {
+
+ /**
+ * Utility method to convert a Schema to its wire format.
+ * @param schema Schema to convert
+ * @return a list of ColumnSchemaPB
+ */
+ public static List<Common.ColumnSchemaPB> schemaToListPb(Schema schema) {
+ ArrayList<Common.ColumnSchemaPB> columns =
+ new ArrayList<Common.ColumnSchemaPB>(schema.getColumnCount());
+ Common.ColumnSchemaPB.Builder schemaBuilder = Common.ColumnSchemaPB.newBuilder();
+ for (ColumnSchema col : schema.getColumns()) {
+ columns.add(columnToPb(schemaBuilder, col));
+ schemaBuilder.clear();
+ }
+ return columns;
+ }
+
+ public static Common.SchemaPB schemaToPb(Schema schema) {
+ Common.SchemaPB.Builder builder = Common.SchemaPB.newBuilder();
+ builder.addAllColumns(schemaToListPb(schema));
+ return builder.build();
+ }
+
+ public static Common.ColumnSchemaPB columnToPb(ColumnSchema column) {
+ return columnToPb(Common.ColumnSchemaPB.newBuilder(), column);
+ }
+
+ public static Common.ColumnSchemaPB
+ columnToPb(Common.ColumnSchemaPB.Builder schemaBuilder, ColumnSchema column) {
+ schemaBuilder
+ .setName(column.getName())
+ .setType(column.getType().getDataType())
+ .setIsKey(column.isKey())
+ .setIsNullable(column.isNullable())
+ .setCfileBlockSize(column.getDesiredBlockSize());
+ if (column.getEncoding() != null) {
+ schemaBuilder.setEncoding(column.getEncoding().getInternalPbType());
+ }
+ if (column.getCompressionAlgorithm() != null) {
+ schemaBuilder.setCompression(column.getCompressionAlgorithm().getInternalPbType());
+ }
+ if (column.getDefaultValue() != null) schemaBuilder.setReadDefaultValue
+ (ZeroCopyLiteralByteString.wrap(objectToWireFormat(column, column.getDefaultValue())));
+ return schemaBuilder.build();
+ }
+
+ public static ColumnSchema pbToColumnSchema(Common.ColumnSchemaPB pb) {
+ Type type = Type.getTypeForDataType(pb.getType());
+ Object defaultValue = pb.hasReadDefaultValue() ?
+ byteStringToObject(type, pb.getReadDefaultValue()) : null;
+ ColumnSchema.Encoding encoding = ColumnSchema.Encoding.valueOf(pb.getEncoding().name());
+ ColumnSchema.CompressionAlgorithm compressionAlgorithm =
+ ColumnSchema.CompressionAlgorithm.valueOf(pb.getCompression().name());
+ return new ColumnSchema.ColumnSchemaBuilder(pb.getName(), type)
+ .key(pb.getIsKey())
+ .nullable(pb.getIsNullable())
+ .defaultValue(defaultValue)
+ .encoding(encoding)
+ .compressionAlgorithm(compressionAlgorithm)
+ .build();
+ }
+
+ public static Schema pbToSchema(Common.SchemaPB schema) {
+ List<ColumnSchema> columns = new ArrayList<>(schema.getColumnsCount());
+ List<Integer> columnIds = new ArrayList<>(schema.getColumnsCount());
+ for (Common.ColumnSchemaPB columnPb : schema.getColumnsList()) {
+ columns.add(pbToColumnSchema(columnPb));
+ int id = columnPb.getId();
+ if (id < 0) {
+ throw new IllegalArgumentException("Illegal column ID: " + id);
+ }
+ columnIds.add(id);
+ }
+ return new Schema(columns, columnIds);
+ }
+
+ /**
+ * Factory method for creating a {@code PartitionSchema} from a protobuf message.
+ *
+ * @param pb the partition schema protobuf message
+ * @return a partition instance
+ */
+ static PartitionSchema pbToPartitionSchema(Common.PartitionSchemaPB pb, Schema schema) {
+ List<Integer> rangeColumns = pbToIds(pb.getRangeSchema().getColumnsList());
+ PartitionSchema.RangeSchema rangeSchema = new PartitionSchema.RangeSchema(rangeColumns);
+
+ ImmutableList.Builder<PartitionSchema.HashBucketSchema> hashSchemas = ImmutableList.builder();
+
+ for (Common.PartitionSchemaPB.HashBucketSchemaPB hashBucketSchemaPB
+ : pb.getHashBucketSchemasList()) {
+ List<Integer> hashColumnIds = pbToIds(hashBucketSchemaPB.getColumnsList());
+
+ PartitionSchema.HashBucketSchema hashSchema =
+ new PartitionSchema.HashBucketSchema(hashColumnIds,
+ hashBucketSchemaPB.getNumBuckets(),
+ hashBucketSchemaPB.getSeed());
+
+ hashSchemas.add(hashSchema);
+ }
+
+ return new PartitionSchema(rangeSchema, hashSchemas.build(), schema);
+ }
+
+ /**
+ * Constructs a new {@code Partition} instance from the a protobuf message.
+ * @param pb the protobuf message
+ * @return the {@code Partition} corresponding to the message
+ */
+ static Partition pbToPartition(Common.PartitionPB pb) {
+ return new Partition(pb.getPartitionKeyStart().toByteArray(),
+ pb.getPartitionKeyEnd().toByteArray(),
+ pb.getHashBucketsList());
+ }
+
+ /**
+ * Deserializes a list of column identifier protobufs into a list of column IDs. This method
+ * relies on the fact that the master will aways send a partition schema with column IDs, and not
+ * column names (column names are only used when the client is sending the partition schema to
+ * the master as part of the create table process).
+ *
+ * @param columnIdentifiers the column identifiers
+ * @return the column IDs
+ */
+ private static List<Integer> pbToIds(
+ List<Common.PartitionSchemaPB.ColumnIdentifierPB> columnIdentifiers) {
+ ImmutableList.Builder<Integer> columnIds = ImmutableList.builder();
+ for (Common.PartitionSchemaPB.ColumnIdentifierPB column : columnIdentifiers) {
+ switch (column.getIdentifierCase()) {
+ case ID:
+ columnIds.add(column.getId());
+ break;
+ case NAME:
+ throw new IllegalArgumentException(
+ String.format("Expected column ID from master: %s", column));
+ case IDENTIFIER_NOT_SET:
+ throw new IllegalArgumentException("Unknown column: " + column);
+ }
+ }
+ return columnIds.build();
+ }
+
+ private static byte[] objectToWireFormat(ColumnSchema col, Object value) {
+ switch (col.getType()) {
+ case BOOL:
+ return Bytes.fromBoolean((Boolean) value);
+ case INT8:
+ return new byte[] {(Byte) value};
+ case INT16:
+ return Bytes.fromShort((Short) value);
+ case INT32:
+ return Bytes.fromInt((Integer) value);
+ case INT64:
+ case TIMESTAMP:
+ return Bytes.fromLong((Long) value);
+ case STRING:
+ return ((String) value).getBytes(Charsets.UTF_8);
+ case BINARY:
+ return (byte[]) value;
+ case FLOAT:
+ return Bytes.fromFloat((Float) value);
+ case DOUBLE:
+ return Bytes.fromDouble((Double) value);
+ default:
+ throw new IllegalArgumentException("The column " + col.getName() + " is of type " + col
+ .getType() + " which is unknown");
+ }
+ }
+
+ private static Object byteStringToObject(Type type, ByteString value) {
+ byte[] buf = ZeroCopyLiteralByteString.zeroCopyGetBytes(value);
+ switch (type) {
+ case BOOL:
+ return Bytes.getBoolean(buf);
+ case INT8:
+ return Bytes.getByte(buf);
+ case INT16:
+ return Bytes.getShort(buf);
+ case INT32:
+ return Bytes.getInt(buf);
+ case INT64:
+ case TIMESTAMP:
+ return Bytes.getLong(buf);
+ case FLOAT:
+ return Bytes.getFloat(buf);
+ case DOUBLE:
+ return Bytes.getDouble(buf);
+ case STRING:
+ return new String(buf, Charsets.UTF_8);
+ case BINARY:
+ return buf;
+ default:
+ throw new IllegalArgumentException("This type is unknown: " + type);
+ }
+ }
+
+ /**
+ * Convert a {@link com.google.common.net.HostAndPort} to {@link org.kududb.Common.HostPortPB}
+ * protobuf message for serialization.
+ * @param hostAndPort The host and port object. Both host and port must be specified.
+ * @return An initialized HostPortPB object.
+ */
+ public static Common.HostPortPB hostAndPortToPB(HostAndPort hostAndPort) {
+ return Common.HostPortPB.newBuilder()
+ .setHost(hostAndPort.getHostText())
+ .setPort(hostAndPort.getPort())
+ .build();
+ }
+
+ /**
+ * Convert a {@link org.kududb.Common.HostPortPB} to {@link com.google.common.net.HostAndPort}.
+ * @param hostPortPB The fully initialized HostPortPB object. Must have both host and port
+ * specified.
+ * @return An initialized initialized HostAndPort object.
+ */
+ public static HostAndPort hostAndPortFromPB(Common.HostPortPB hostPortPB) {
+ return HostAndPort.fromParts(hostPortPB.getHost(), hostPortPB.getPort());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java
new file mode 100644
index 0000000..25c0fe0
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RecoverableException.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * An exception that's possible to retry.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+class RecoverableException extends KuduException {
+
+ /**
+ * Constructor.
+ * @param status status object containing the reason for the exception
+ * trace.
+ */
+ RecoverableException(Status status) {
+ super(status);
+ }
+
+ /**
+ * Constructor.
+ * @param status status object containing the reason for the exception
+ * @param cause The exception that caused this one to be thrown.
+ */
+ RecoverableException(Status status, Throwable cause) {
+ super(status, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java
new file mode 100644
index 0000000..229b64f
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RequestTracker.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+
+import java.util.Queue;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This is the same class as src/kudu/rpc/request_tracker.h.
+ */
+@InterfaceAudience.Private
+public class RequestTracker {
+ private final AtomicLong sequenceIdTracker = new AtomicLong();
+ private final Queue<Long> incompleteRpcs = new PriorityBlockingQueue<>();
+
+ static final long NO_SEQ_NO = -1;
+
+ private final String clientId;
+
+ /**
+ * Create a new request tracker for the given client id.
+ * @param clientId identifier for the client this tracker belongs to
+ */
+ public RequestTracker(String clientId) {
+ this.clientId = clientId;
+ }
+
+ /**
+ * Generates a new sequence number and tracks it.
+ * @return a new sequence number
+ */
+ public long newSeqNo() {
+ Long next = sequenceIdTracker.incrementAndGet();
+ incompleteRpcs.add(next);
+ return next;
+ }
+
+ /**
+ * Returns the oldest sequence number that wasn't marked as completed. If there is no incomplete
+ * RPC then {@link RequestTracker#NO_SEQ_NO} is returned.
+ * @return the first incomplete sequence number
+ */
+ public long firstIncomplete() {
+ Long peek = incompleteRpcs.peek();
+ return peek == null ? NO_SEQ_NO : peek;
+ }
+
+ /**
+ * Marks the given sequence id as complete. This operation is idempotent.
+ * @param sequenceId the sequence id to mark as complete
+ */
+ public void rpcCompleted(long sequenceId) {
+ incompleteRpcs.remove(sequenceId);
+ }
+
+ public String getClientId() {
+ return clientId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java
new file mode 100644
index 0000000..b4c8f36
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowError.java
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.WireProtocol;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.tserver.Tserver;
+
+/**
+ * Wrapper class for a single row error.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RowError {
+ private final Status status;
+ private final Operation operation;
+ private final String tsUUID;
+
+ /**
+ * Creates a new {@code RowError} with the provided status, operation, and tablet server UUID.
+ */
+ RowError(Status status, Operation operation, String tsUUID) {
+ this.status = status;
+ this.operation = operation;
+ this.tsUUID = tsUUID;
+ }
+
+ /**
+ * Creates a new {@code RowError} with the provided status, and operation.
+ *
+ * This constructor should be used when the operation fails before the tablet
+ * lookup is complete.
+ */
+ RowError(Status status, Operation operation) {
+ this(status, operation, null);
+ }
+
+ /**
+ * Get the status code and message of the row error.
+ */
+ public Status getErrorStatus() {
+ return status;
+ }
+
+ /**
+ * Get the string-representation of the error code that the tablet server returned.
+ * @return A short string representation of the error.
+ * @deprecated Please use getErrorStatus() instead. Will be removed in a future version.
+ */
+ public String getStatus() {
+ return status.getCodeName();
+ }
+
+ /**
+ * Get the error message the tablet server sent.
+ * @return The error message.
+ * @deprecated Please use getErrorStatus() instead. Will be removed in a future version.
+ */
+ public String getMessage() {
+ return status.getMessage();
+ }
+
+ /**
+ * Get the Operation that failed.
+ * @return The same Operation instance that failed
+ */
+ public Operation getOperation() {
+ return operation;
+ }
+
+ /**
+ * Get the identifier of the tablet server that sent the error.
+ * The UUID may be {@code null} if the failure occurred before sending the row
+ * to a tablet server (for instance, if the row falls in a non-covered range partition).
+ * @return A string containing a UUID
+ */
+ public String getTsUUID() {
+ return tsUUID;
+ }
+
+ @Override
+ public String toString() {
+ return "Row error for primary key=" + Bytes.pretty(operation.getRow().encodePrimaryKey()) +
+ ", tablet=" + operation.getTablet() +
+ ", server=" + tsUUID +
+ ", status=" + status.toString();
+ }
+
+ /**
+ * Converts a PerRowErrorPB into a RowError.
+ * @param errorPB a row error in its pb format
+ * @param operation the original operation
+ * @param tsUUID a string containing the originating TS's UUID
+ * @return a row error
+ */
+ static RowError fromRowErrorPb(Tserver.WriteResponsePB.PerRowErrorPB errorPB,
+ Operation operation, String tsUUID) {
+ WireProtocol.AppStatusPB statusPB = errorPB.getError();
+ return new RowError(Status.fromPB(statusPB), operation, tsUUID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java
new file mode 100644
index 0000000..17a4778
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowErrorsAndOverflowStatus.java
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Container class used as a response when retrieving pending row errors.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RowErrorsAndOverflowStatus {
+ private final RowError[] rowErrors;
+ private final boolean overflowed;
+
+ RowErrorsAndOverflowStatus(RowError[] rowErrors, boolean overflowed) {
+ this.rowErrors = rowErrors;
+ this.overflowed = overflowed;
+ }
+
+ /**
+ * Get the collected row errors.
+ * @return an array of row errors, may be empty
+ */
+ public RowError[] getRowErrors() {
+ return rowErrors;
+ }
+
+ /**
+ * Check if the error collector had an overflow and had to discard row errors.
+ * @return true if row errors were discarded, false otherwise
+ */
+ public boolean isOverflowed() {
+ return overflowed;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
new file mode 100644
index 0000000..7692c53
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResult.java
@@ -0,0 +1,570 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.util.Slice;
+
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.BitSet;
+import java.util.Date;
+import java.util.TimeZone;
+
+/**
+ * RowResult represents one row from a scanner. Do not reuse or store the objects.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RowResult {
+
+ private static final int INDEX_RESET_LOCATION = -1;
+ private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ {
+ DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT"));
+ }
+ private static final long MS_IN_S = 1000L;
+ private static final long US_IN_S = 1000L * 1000L;
+ private int index = INDEX_RESET_LOCATION;
+ private int offset;
+ private BitSet nullsBitSet;
+ private final int rowSize;
+ private final int[] columnOffsets;
+ private final Schema schema;
+ private final Slice rowData;
+ private final Slice indirectData;
+
+ /**
+ * Prepares the row representation using the provided data. Doesn't copy data
+ * out of the byte arrays. Package private.
+ * @param schema Schema used to build the rowData
+ * @param rowData The Slice of data returned by the tablet server
+ * @param indirectData The full indirect data that contains the strings
+ */
+ RowResult(Schema schema, Slice rowData, Slice indirectData) {
+ this.schema = schema;
+ this.rowData = rowData;
+ this.indirectData = indirectData;
+ int columnOffsetsSize = schema.getColumnCount();
+ if (schema.hasNullableColumns()) {
+ columnOffsetsSize++;
+ }
+ this.rowSize = this.schema.getRowSize();
+ columnOffsets = new int[columnOffsetsSize];
+ // Empty projection, usually used for quick row counting
+ if (columnOffsetsSize == 0) {
+ return;
+ }
+ int currentOffset = 0;
+ columnOffsets[0] = currentOffset;
+ // Pre-compute the columns offsets in rowData for easier lookups later
+ // If the schema has nullables, we also add the offset for the null bitmap at the end
+ for (int i = 1; i < columnOffsetsSize; i++) {
+ int previousSize = schema.getColumnByIndex(i - 1).getType().getSize();
+ columnOffsets[i] = previousSize + currentOffset;
+ currentOffset += previousSize;
+ }
+ }
+
+ /**
+ * Package-protected, only meant to be used by the RowResultIterator
+ */
+ void advancePointer() {
+ advancePointerTo(this.index + 1);
+ }
+
+ void resetPointer() {
+ advancePointerTo(INDEX_RESET_LOCATION);
+ }
+
+ void advancePointerTo(int rowIndex) {
+ this.index = rowIndex;
+ this.offset = this.rowSize * this.index;
+ if (schema.hasNullableColumns() && this.index != INDEX_RESET_LOCATION) {
+ this.nullsBitSet = Bytes.toBitSet(
+ this.rowData.getRawArray(),
+ this.rowData.getRawOffset()
+ + getCurrentRowDataOffsetForColumn(schema.getColumnCount()),
+ schema.getColumnCount());
+ }
+ }
+
+ int getCurrentRowDataOffsetForColumn(int columnIndex) {
+ return this.offset + this.columnOffsets[columnIndex];
+ }
+
+ /**
+ * Get the specified column's integer
+ * @param columnName name of the column to get data for
+ * @return An integer
+ * @throws IllegalArgumentException if the column is null
+ */
+ public int getInt(String columnName) {
+ return getInt(this.schema.getColumnIndex(columnName));
+ }
+
+ /**
+ * Get the specified column's integer
+ * @param columnIndex Column index in the schema
+ * @return An integer
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public int getInt(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.INT32);
+ return Bytes.getInt(this.rowData.getRawArray(),
+ this.rowData.getRawOffset() + getCurrentRowDataOffsetForColumn(columnIndex));
+ }
+
+ /**
+ * Get the specified column's short
+ * @param columnName name of the column to get data for
+ * @return A short
+ * @throws IllegalArgumentException if the column is null
+ */
+ public short getShort(String columnName) {
+ return getShort(this.schema.getColumnIndex(columnName));
+ }
+
+ /**
+ * Get the specified column's short
+ * @param columnIndex Column index in the schema
+ * @return A short
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public short getShort(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.INT16);
+ return Bytes.getShort(this.rowData.getRawArray(),
+ this.rowData.getRawOffset() + getCurrentRowDataOffsetForColumn(columnIndex));
+ }
+
+ /**
+ * Get the specified column's boolean
+ * @param columnName name of the column to get data for
+ * @return A boolean
+ * @throws IllegalArgumentException if the column is null
+ */
+ public boolean getBoolean(String columnName) {
+ return getBoolean(this.schema.getColumnIndex(columnName));
+ }
+
+ /**
+ * Get the specified column's boolean
+ * @param columnIndex Column index in the schema
+ * @return A boolean
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public boolean getBoolean(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.BOOL);
+ byte b = Bytes.getByte(this.rowData.getRawArray(),
+ this.rowData.getRawOffset()
+ + getCurrentRowDataOffsetForColumn(columnIndex));
+ return b == 1;
+ }
+
+ /**
+ * Get the specified column's byte
+ * @param columnName name of the column to get data for
+ * @return A byte
+ * @throws IllegalArgumentException if the column is null
+ */
+ public byte getByte(String columnName) {
+ return getByte(this.schema.getColumnIndex(columnName));
+
+ }
+
+ /**
+ * Get the specified column's byte
+ * @param columnIndex Column index in the schema
+ * @return A byte
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public byte getByte(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.INT8);
+ return Bytes.getByte(this.rowData.getRawArray(),
+ this.rowData.getRawOffset() + getCurrentRowDataOffsetForColumn(columnIndex));
+ }
+
+ /**
+ * Get the specified column's long
+ *
+ * If this is a TIMESTAMP column, the long value corresponds to a number of microseconds
+ * since midnight, January 1, 1970 UTC.
+ *
+ * @param columnName name of the column to get data for
+ * @return A positive long
+ * @throws IllegalArgumentException if the column is null\
+ */
+ public long getLong(String columnName) {
+ return getLong(this.schema.getColumnIndex(columnName));
+ }
+
+ /**
+ * Get the specified column's long
+ *
+ * If this is a TIMESTAMP column, the long value corresponds to a number of microseconds
+ * since midnight, January 1, 1970 UTC.
+ *
+ * @param columnIndex Column index in the schema
+ * @return A positive long
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public long getLong(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ // Can't check type because this could be a long, string, or Timestamp
+ return Bytes.getLong(this.rowData.getRawArray(),
+ this.rowData.getRawOffset()
+ + getCurrentRowDataOffsetForColumn(columnIndex));
+ }
+
+ /**
+ * Get the specified column's float
+ * @param columnName name of the column to get data for
+ * @return A float
+ */
+ public float getFloat(String columnName) {
+ return getFloat(this.schema.getColumnIndex(columnName));
+ }
+
+ /**
+ * Get the specified column's float
+ * @param columnIndex Column index in the schema
+ * @return A float
+ */
+ public float getFloat(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.FLOAT);
+ return Bytes.getFloat(this.rowData.getRawArray(),
+ this.rowData.getRawOffset()
+ + getCurrentRowDataOffsetForColumn(columnIndex));
+ }
+
+ /**
+ * Get the specified column's double
+ * @param columnName name of the column to get data for
+ * @return A double
+ */
+ public double getDouble(String columnName) {
+ return getDouble(this.schema.getColumnIndex(columnName));
+
+ }
+
+ /**
+ * Get the specified column's double
+ * @param columnIndex Column index in the schema
+ * @return A double
+ */
+ public double getDouble(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.DOUBLE);
+ return Bytes.getDouble(this.rowData.getRawArray(),
+ this.rowData.getRawOffset()
+ + getCurrentRowDataOffsetForColumn(columnIndex));
+ }
+
+ /**
+ * Get the schema used for this scanner's column projection.
+ * @return A column projection as a schema.
+ */
+ public Schema getColumnProjection() {
+ return this.schema;
+ }
+
+ /**
+ * Get the specified column's string.
+ * @param columnName name of the column to get data for
+ * @return A string
+ * @throws IllegalArgumentException if the column is null
+ */
+ public String getString(String columnName) {
+ return getString(this.schema.getColumnIndex(columnName));
+
+ }
+
+ /**
+ * Get the specified column's string.
+ * @param columnIndex Column index in the schema
+ * @return A string
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public String getString(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.STRING);
+ // C++ puts a Slice in rowData which is 16 bytes long for simplity, but we only support ints
+ long offset = getLong(columnIndex);
+ long length = rowData.getLong(getCurrentRowDataOffsetForColumn(columnIndex) + 8);
+ assert offset < Integer.MAX_VALUE;
+ assert length < Integer.MAX_VALUE;
+ return Bytes.getString(indirectData.getRawArray(),
+ indirectData.getRawOffset() + (int)offset,
+ (int)length);
+ }
+
+ /**
+ * Get a copy of the specified column's binary data.
+ * @param columnName name of the column to get data for
+ * @return a byte[] with the binary data.
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public byte[] getBinaryCopy(String columnName) {
+ return getBinaryCopy(this.schema.getColumnIndex(columnName));
+
+ }
+
+ /**
+ * Get a copy of the specified column's binary data.
+ * @param columnIndex Column index in the schema
+ * @return a byte[] with the binary data.
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public byte[] getBinaryCopy(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ // C++ puts a Slice in rowData which is 16 bytes long for simplicity,
+ // but we only support ints
+ long offset = getLong(columnIndex);
+ long length = rowData.getLong(getCurrentRowDataOffsetForColumn(columnIndex) + 8);
+ assert offset < Integer.MAX_VALUE;
+ assert length < Integer.MAX_VALUE;
+ byte[] ret = new byte[(int)length];
+ System.arraycopy(indirectData.getRawArray(), indirectData.getRawOffset() + (int) offset,
+ ret, 0, (int) length);
+ return ret;
+ }
+
+ /**
+ * Get the specified column's binary data.
+ *
+ * This doesn't copy the data and instead returns a ByteBuffer that wraps it.
+ *
+ * @param columnName name of the column to get data for
+ * @return a byte[] with the binary data.
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public ByteBuffer getBinary(String columnName) {
+ return getBinary(this.schema.getColumnIndex(columnName));
+ }
+
+ /**
+ * Get the specified column's binary data.
+ *
+ * This doesn't copy the data and instead returns a ByteBuffer that wraps it.
+ *
+ * @param columnIndex Column index in the schema
+ * @return a byte[] with the binary data.
+ * @throws IllegalArgumentException if the column is null
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public ByteBuffer getBinary(int columnIndex) {
+ checkValidColumn(columnIndex);
+ checkNull(columnIndex);
+ checkType(columnIndex, Type.BINARY);
+ // C++ puts a Slice in rowData which is 16 bytes long for simplicity,
+ // but we only support ints
+ long offset = getLong(columnIndex);
+ long length = rowData.getLong(getCurrentRowDataOffsetForColumn(columnIndex) + 8);
+ assert offset < Integer.MAX_VALUE;
+ assert length < Integer.MAX_VALUE;
+ return ByteBuffer.wrap(indirectData.getRawArray(), indirectData.getRawOffset() + (int) offset,
+ (int) length);
+ }
+
+ /**
+ * Get if the specified column is NULL
+ * @param columnName name of the column to get data for
+ * @return true if the column cell is null and the column is nullable,
+ * false otherwise
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public boolean isNull(String columnName) {
+ return isNull(this.schema.getColumnIndex(columnName));
+ }
+
+ /**
+ * Get if the specified column is NULL
+ * @param columnIndex Column index in the schema
+ * @return true if the column cell is null and the column is nullable,
+ * false otherwise
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public boolean isNull(int columnIndex) {
+ checkValidColumn(columnIndex);
+ if (nullsBitSet == null) {
+ return false;
+ }
+ return schema.getColumnByIndex(columnIndex).isNullable()
+ && nullsBitSet.get(columnIndex);
+ }
+
+ /**
+ * Get the type of a column in this result.
+ * @param columnName name of the column
+ * @return a type
+ */
+ public Type getColumnType(String columnName) {
+ return this.schema.getColumn(columnName).getType();
+ }
+
+ /**
+ * Get the type of a column in this result.
+ * @param columnIndex column index in the schema
+ * @return a type
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ public Type getColumnType(int columnIndex) {
+ return this.schema.getColumnByIndex(columnIndex).getType();
+ }
+
+ /**
+ * Get the schema associated with this result.
+ * @return a schema
+ */
+ public Schema getSchema() {
+ return schema;
+ }
+
+ /**
+ * @throws IndexOutOfBoundsException if the column doesn't exist
+ */
+ private void checkValidColumn(int columnIndex) {
+ if (columnIndex >= schema.getColumnCount()) {
+ throw new IndexOutOfBoundsException("Requested column is out of range, " +
+ columnIndex + " out of " + schema.getColumnCount());
+ }
+ }
+
+ /**
+ * @throws IllegalArgumentException if the column is null
+ */
+ private void checkNull(int columnIndex) {
+ if (!schema.hasNullableColumns()) {
+ return;
+ }
+ if (isNull(columnIndex)) {
+ ColumnSchema columnSchema = schema.getColumnByIndex(columnIndex);
+ throw new IllegalArgumentException("The requested column (name: " + columnSchema.getName() +
+ ", index: " + columnIndex + ") is null");
+ }
+ }
+
+ private void checkType(int columnIndex, Type expectedType) {
+ ColumnSchema columnSchema = schema.getColumnByIndex(columnIndex);
+ Type columnType = columnSchema.getType();
+ if (!columnType.equals(expectedType)) {
+ throw new IllegalArgumentException("Column (name: " + columnSchema.getName() +
+ ", index: " + columnIndex +") is of type " +
+ columnType.getName() + " but was requested as a type " + expectedType.getName());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RowResult index: " + this.index + ", size: " + this.rowSize + ", " +
+ "schema: " + this.schema;
+ }
+
+ /**
+ * Transforms a timestamp into a string, whose formatting and timezone is consistent
+ * across kudu.
+ * @param timestamp the timestamp, in microseconds
+ * @return a string, in the format: YYYY-MM-DD HH:MM:SS.ssssss GMT
+ */
+ static String timestampToString(long timestamp) {
+ long tsMillis = timestamp / MS_IN_S;
+ long tsMicros = timestamp % US_IN_S;
+ StringBuffer formattedTs = new StringBuffer();
+ formattedTs.append(DATE_FORMAT.format(new Date(tsMillis)));
+ formattedTs.append(String.format(".%06d GMT", tsMicros));
+ return formattedTs.toString();
+ }
+
+ /**
+ * Return the actual data from this row in a stringified key=value
+ * form.
+ */
+ public String rowToString() {
+ StringBuffer buf = new StringBuffer();
+ for (int i = 0; i < schema.getColumnCount(); i++) {
+ ColumnSchema col = schema.getColumnByIndex(i);
+ if (i != 0) {
+ buf.append(", ");
+ }
+ buf.append(col.getType().name());
+ buf.append(" ").append(col.getName()).append("=");
+ if (isNull(i)) {
+ buf.append("NULL");
+ } else {
+ switch (col.getType()) {
+ case INT8: buf.append(getByte(i)); break;
+ case INT16: buf.append(getShort(i));
+ break;
+ case INT32: buf.append(getInt(i)); break;
+ case INT64: buf.append(getLong(i)); break;
+ case TIMESTAMP: {
+ buf.append(timestampToString(getLong(i)));
+ } break;
+ case STRING: buf.append(getString(i)); break;
+ case BINARY: buf.append(Bytes.pretty(getBinaryCopy(i))); break;
+ case FLOAT: buf.append(getFloat(i)); break;
+ case DOUBLE: buf.append(getDouble(i)); break;
+ case BOOL: buf.append(getBoolean(i)); break;
+ default: buf.append("<unknown type!>"); break;
+ }
+ }
+ }
+ return buf.toString();
+ }
+
+ /**
+ * @return a string describing the location of this row result within
+ * the iterator as well as its data.
+ */
+ public String toStringLongFormat() {
+ StringBuffer buf = new StringBuffer(this.rowSize); // super rough estimation
+ buf.append(this.toString());
+ buf.append("{");
+ buf.append(rowToString());
+ buf.append("}");
+ return buf.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java
new file mode 100644
index 0000000..5705ea3
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/RowResultIterator.java
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import java.util.Iterator;
+import org.kududb.Schema;
+import org.kududb.WireProtocol;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.util.Slice;
+
+/**
+ * Class that contains the rows sent by a tablet server, exhausting this iterator only means
+ * that all the rows from the last server response were read.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RowResultIterator extends KuduRpcResponse implements Iterator<RowResult>,
+ Iterable<RowResult> {
+
+ private static final RowResultIterator EMPTY =
+ new RowResultIterator(0, null, null, 0, null, null);
+
+ private final Schema schema;
+ private final Slice bs;
+ private final Slice indirectBs;
+ private final int numRows;
+ private final RowResult rowResult;
+ private int currentRow = 0;
+
+ /**
+ * Package private constructor, only meant to be instantiated from AsyncKuduScanner.
+ * @param ellapsedMillis ime in milliseconds since RPC creation to now
+ * @param tsUUID UUID of the tablet server that handled our request
+ * @param schema schema used to parse the rows
+ * @param numRows how many rows are contained in the bs slice
+ * @param bs normal row data
+ * @param indirectBs indirect row data
+ */
+ private RowResultIterator(long ellapsedMillis, String tsUUID, Schema schema,
+ int numRows, Slice bs, Slice indirectBs) {
+ super(ellapsedMillis, tsUUID);
+ this.schema = schema;
+ this.bs = bs;
+ this.indirectBs = indirectBs;
+ this.numRows = numRows;
+
+ this.rowResult = numRows == 0 ? null : new RowResult(this.schema, this.bs, this.indirectBs);
+ }
+
+ static RowResultIterator makeRowResultIterator(long ellapsedMillis, String tsUUID,
+ Schema schema,
+ WireProtocol.RowwiseRowBlockPB data,
+ final CallResponse callResponse)
+ throws KuduException {
+ if (data == null || data.getNumRows() == 0) {
+ return new RowResultIterator(ellapsedMillis, tsUUID, schema, 0, null, null);
+ }
+
+ Slice bs = callResponse.getSidecar(data.getRowsSidecar());
+ Slice indirectBs = callResponse.getSidecar(data.getIndirectDataSidecar());
+ int numRows = data.getNumRows();
+
+ // Integrity check
+ int rowSize = schema.getRowSize();
+ int expectedSize = numRows * rowSize;
+ if (expectedSize != bs.length()) {
+ Status statusIllegalState = Status.IllegalState("RowResult block has " + bs.length() +
+ " bytes of data but expected " + expectedSize + " for " + numRows + " rows");
+ throw new NonRecoverableException(statusIllegalState);
+ }
+ return new RowResultIterator(ellapsedMillis, tsUUID, schema, numRows, bs, indirectBs);
+ }
+
+ /**
+ * @return an empty row result iterator
+ */
+ static RowResultIterator empty() {
+ return EMPTY;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.currentRow < numRows;
+ }
+
+ @Override
+ public RowResult next() {
+ // The rowResult keeps track of where it is internally
+ this.rowResult.advancePointer();
+ this.currentRow++;
+ return rowResult;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get the number of rows in this iterator. If all you want is to count
+ * rows, call this and skip the rest.
+ * @return number of rows in this iterator
+ */
+ public int getNumRows() {
+ return this.numRows;
+ }
+
+ @Override
+ public String toString() {
+ return "RowResultIterator for " + this.numRows + " rows";
+ }
+
+ @Override
+ public Iterator<RowResult> iterator() {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java b/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java
new file mode 100644
index 0000000..53e108a
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/SecureRpcHelper.java
@@ -0,0 +1,285 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the aabove copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.Channels;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.rpc.RpcHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.sasl.RealmCallback;
+import javax.security.sasl.RealmChoiceCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslClient;
+import javax.security.sasl.SaslException;
+
+@InterfaceAudience.Private
+public class SecureRpcHelper {
+
+ public static final Logger LOG = LoggerFactory.getLogger(TabletClient.class);
+
+ private final TabletClient client;
+ private SaslClient saslClient;
+ public static final String SASL_DEFAULT_REALM = "default";
+ public static final Map<String, String> SASL_PROPS = new TreeMap<>();
+ private static final int SASL_CALL_ID = -33;
+ private static final Set<RpcHeader.RpcFeatureFlag> SUPPORTED_RPC_FEATURES =
+ ImmutableSet.of(RpcHeader.RpcFeatureFlag.APPLICATION_FEATURE_FLAGS);
+ private volatile boolean negoUnderway = true;
+ private boolean useWrap = false; // no QOP at the moment
+ private Set<RpcHeader.RpcFeatureFlag> serverFeatures;
+
+ public static final String USER_AND_PASSWORD = "java_client";
+
+ public SecureRpcHelper(TabletClient client) {
+ this.client = client;
+ try {
+ saslClient = Sasl.createSaslClient(new String[]{"PLAIN"},
+ null,
+ null,
+ SASL_DEFAULT_REALM,
+ SASL_PROPS,
+ new SaslClientCallbackHandler(USER_AND_PASSWORD,
+ USER_AND_PASSWORD));
+ } catch (SaslException e) {
+ throw new RuntimeException("Could not create the SASL client", e);
+ }
+ }
+
+ public Set<RpcHeader.RpcFeatureFlag> getServerFeatures() {
+ Preconditions.checkState(!negoUnderway);
+ Preconditions.checkNotNull(serverFeatures);
+ return serverFeatures;
+ }
+
+ public void sendHello(Channel channel) {
+ sendNegotiateMessage(channel);
+ }
+
+ private void sendNegotiateMessage(Channel channel) {
+ RpcHeader.SaslMessagePB.Builder builder = RpcHeader.SaslMessagePB.newBuilder();
+
+ // Advertise our supported features
+ for (RpcHeader.RpcFeatureFlag flag : SUPPORTED_RPC_FEATURES) {
+ builder.addSupportedFeatures(flag);
+ }
+
+ builder.setState(RpcHeader.SaslMessagePB.SaslState.NEGOTIATE);
+ sendSaslMessage(channel, builder.build());
+ }
+
+ private void sendSaslMessage(Channel channel, RpcHeader.SaslMessagePB msg) {
+ RpcHeader.RequestHeader.Builder builder = RpcHeader.RequestHeader.newBuilder();
+ builder.setCallId(SASL_CALL_ID);
+ RpcHeader.RequestHeader header = builder.build();
+
+ ChannelBuffer buffer = KuduRpc.toChannelBuffer(header, msg);
+ Channels.write(channel, buffer);
+ }
+
+ public ChannelBuffer handleResponse(ChannelBuffer buf, Channel chan) throws SaslException {
+ if (!saslClient.isComplete() || negoUnderway) {
+ RpcHeader.SaslMessagePB response = parseSaslMsgResponse(buf);
+ switch (response.getState()) {
+ case NEGOTIATE:
+ handleNegotiateResponse(chan, response);
+ break;
+ case CHALLENGE:
+ handleChallengeResponse(chan, response);
+ break;
+ case SUCCESS:
+ handleSuccessResponse(chan, response);
+ break;
+ default:
+ System.out.println("Wrong sasl state");
+ }
+ return null;
+ }
+ return unwrap(buf);
+ }
+
+ /**
+ * When QOP of auth-int or auth-conf is selected
+ * This is used to unwrap the contents from the passed
+ * buffer payload.
+ */
+ public ChannelBuffer unwrap(ChannelBuffer payload) {
+ if(!useWrap) {
+ return payload;
+ }
+ int len = payload.readInt();
+ try {
+ payload =
+ ChannelBuffers.wrappedBuffer(saslClient.unwrap(payload.readBytes(len).array(), 0, len));
+ return payload;
+ } catch (SaslException e) {
+ throw new IllegalStateException("Failed to unwrap payload", e);
+ }
+ }
+
+ /**
+ * When QOP of auth-int or auth-conf is selected
+ * This is used to wrap the contents
+ * into the proper payload (ie encryption, signature, etc)
+ */
+ public ChannelBuffer wrap(ChannelBuffer content) {
+ if(!useWrap) {
+ return content;
+ }
+ try {
+ byte[] payload = new byte[content.writerIndex()];
+ content.readBytes(payload);
+ byte[] wrapped = saslClient.wrap(payload, 0, payload.length);
+ ChannelBuffer ret = ChannelBuffers.wrappedBuffer(new byte[4 + wrapped.length]);
+ ret.clear();
+ ret.writeInt(wrapped.length);
+ ret.writeBytes(wrapped);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Wrapped payload: "+Bytes.pretty(ret));
+ }
+ return ret;
+ } catch (SaslException e) {
+ throw new IllegalStateException("Failed to wrap payload", e);
+ }
+ }
+
+ private RpcHeader.SaslMessagePB parseSaslMsgResponse(ChannelBuffer buf) {
+ CallResponse response = new CallResponse(buf);
+ RpcHeader.ResponseHeader responseHeader = response.getHeader();
+ int id = responseHeader.getCallId();
+ if (id != SASL_CALL_ID) {
+ throw new IllegalStateException("Received a call that wasn't for SASL");
+ }
+
+ RpcHeader.SaslMessagePB.Builder saslBuilder = RpcHeader.SaslMessagePB.newBuilder();
+ KuduRpc.readProtobuf(response.getPBMessage(), saslBuilder);
+ return saslBuilder.build();
+ }
+
+
+ private void handleNegotiateResponse(Channel chan, RpcHeader.SaslMessagePB response) throws
+ SaslException {
+ RpcHeader.SaslMessagePB.SaslAuth negotiatedAuth = null;
+ for (RpcHeader.SaslMessagePB.SaslAuth auth : response.getAuthsList()) {
+ negotiatedAuth = auth;
+ }
+
+ ImmutableSet.Builder<RpcHeader.RpcFeatureFlag> features = ImmutableSet.builder();
+ for (RpcHeader.RpcFeatureFlag feature : response.getSupportedFeaturesList()) {
+ if (SUPPORTED_RPC_FEATURES.contains(feature)) {
+ features.add(feature);
+ }
+ }
+ serverFeatures = features.build();
+
+ byte[] saslToken = new byte[0];
+ if (saslClient.hasInitialResponse())
+ saslToken = saslClient.evaluateChallenge(saslToken);
+
+ RpcHeader.SaslMessagePB.Builder builder = RpcHeader.SaslMessagePB.newBuilder();
+ if (saslToken != null) {
+ builder.setToken(ZeroCopyLiteralByteString.wrap(saslToken));
+ }
+ builder.setState(RpcHeader.SaslMessagePB.SaslState.INITIATE);
+ builder.addAuths(negotiatedAuth);
+ sendSaslMessage(chan, builder.build());
+ }
+
+ private void handleChallengeResponse(Channel chan, RpcHeader.SaslMessagePB response) throws
+ SaslException {
+ ByteString bs = response.getToken();
+ byte[] saslToken = saslClient.evaluateChallenge(bs.toByteArray());
+ if (saslToken == null) {
+ throw new IllegalStateException("Not expecting an empty token");
+ }
+ RpcHeader.SaslMessagePB.Builder builder = RpcHeader.SaslMessagePB.newBuilder();
+ builder.setToken(ZeroCopyLiteralByteString.wrap(saslToken));
+ builder.setState(RpcHeader.SaslMessagePB.SaslState.RESPONSE);
+ sendSaslMessage(chan, builder.build());
+ }
+
+ private void handleSuccessResponse(Channel chan, RpcHeader.SaslMessagePB response) {
+ LOG.debug("nego finished");
+ negoUnderway = false;
+ client.sendContext(chan);
+ }
+
+ private static class SaslClientCallbackHandler implements CallbackHandler {
+ private final String userName;
+ private final char[] userPassword;
+
+ public SaslClientCallbackHandler(String user, String password) {
+ this.userName = user;
+ this.userPassword = password.toCharArray();
+ }
+
+ public void handle(Callback[] callbacks)
+ throws UnsupportedCallbackException {
+ NameCallback nc = null;
+ PasswordCallback pc = null;
+ RealmCallback rc = null;
+ for (Callback callback : callbacks) {
+ if (callback instanceof RealmChoiceCallback) {
+ continue;
+ } else if (callback instanceof NameCallback) {
+ nc = (NameCallback) callback;
+ } else if (callback instanceof PasswordCallback) {
+ pc = (PasswordCallback) callback;
+ } else if (callback instanceof RealmCallback) {
+ rc = (RealmCallback) callback;
+ } else {
+ throw new UnsupportedCallbackException(callback,
+ "Unrecognized SASL client callback");
+ }
+ }
+ if (nc != null) {
+ nc.setName(userName);
+ }
+ if (pc != null) {
+ pc.setPassword(userPassword);
+ }
+ if (rc != null) {
+ rc.setText(rc.getDefaultText());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java b/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java
new file mode 100644
index 0000000..94e0a66
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/SessionConfiguration.java
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Interface that defines the methods used to configure a session. It also exposes ways to
+ * query its state.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface SessionConfiguration {
+
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ enum FlushMode {
+ // Every write will be sent to the server in-band with the Apply()
+ // call. No batching will occur. This is the default flush mode. In this
+ // mode, the Flush() call never has any effect, since each Apply() call
+ // has already flushed the buffer.
+ AUTO_FLUSH_SYNC,
+
+ // Apply() calls will return immediately, but the writes will be sent in
+ // the background, potentially batched together with other writes from
+ // the same session. If there is not sufficient buffer space, then Apply()
+ // may block for buffer space to be available.
+ //
+ // Because writes are applied in the background, any errors will be stored
+ // in a session-local buffer. Call CountPendingErrors() or GetPendingErrors()
+ // to retrieve them.
+ //
+ // The Flush() call can be used to block until the buffer is empty.
+ AUTO_FLUSH_BACKGROUND,
+
+ // Apply() calls will return immediately, and the writes will not be
+ // sent until the user calls Flush(). If the buffer runs past the
+ // configured space limit, then Apply() will return an error.
+ MANUAL_FLUSH
+ }
+
+ /**
+ * Get the current flush mode.
+ * @return flush mode, AUTO_FLUSH_SYNC by default
+ */
+ FlushMode getFlushMode();
+
+ /**
+ * Set the new flush mode for this session.
+ * @param flushMode new flush mode, can be the same as the previous one.
+ * @throws IllegalArgumentException if the buffer isn't empty.
+ */
+ void setFlushMode(FlushMode flushMode);
+
+ /**
+ * Set the number of operations that can be buffered.
+ * @param size number of ops.
+ * @throws IllegalArgumentException if the buffer isn't empty.
+ */
+ void setMutationBufferSpace(int size);
+
+ /**
+ * Set the low watermark for this session. The default is set to half the mutation buffer space.
+ * For example, a buffer space of 1000 with a low watermark set to 50% (0.5) will start randomly
+ * sending PleaseRetryExceptions once there's an outstanding flush and the buffer is over 500.
+ * As the buffer gets fuller, it becomes likelier to hit the exception.
+ * @param mutationBufferLowWatermarkPercentage a new low watermark as a percentage,
+ * has to be between 0 and 1 (inclusive). A value of 1 disables
+ * the low watermark since it's the same as the high one
+ * @throws IllegalArgumentException if the buffer isn't empty or if the watermark isn't between
+ * 0 and 1
+ */
+ void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage);
+
+ /**
+ * Set the flush interval, which will be used for the next scheduling decision.
+ * @param interval interval in milliseconds.
+ */
+ void setFlushInterval(int interval);
+
+ /**
+ * Get the current timeout.
+ * @return operation timeout in milliseconds, 0 if none was configured.
+ */
+ long getTimeoutMillis();
+
+ /**
+ * Sets the timeout for the next applied operations.
+ * The default timeout is 0, which disables the timeout functionality.
+ * @param timeout Timeout in milliseconds.
+ */
+ void setTimeoutMillis(long timeout);
+
+ /**
+ * Returns true if this session has already been closed.
+ */
+ boolean isClosed();
+
+ /**
+ * Check if there are operations that haven't been completely applied.
+ * @return true if operations are pending, else false.
+ */
+ boolean hasPendingOperations();
+
+ /**
+ * Set the new external consistency mode for this session.
+ * @param consistencyMode new external consistency mode, can the same as the previous one.
+ * @throws IllegalArgumentException if the buffer isn't empty.
+ */
+ void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode);
+
+ /**
+ * Tells if the session is currently ignoring row errors when the whole list returned by a tablet
+ * server is of the AlreadyPresent type.
+ * @return true if the session is enforcing this, else false
+ */
+ boolean isIgnoreAllDuplicateRows();
+
+ /**
+ * Configures the option to ignore all the row errors if they are all of the AlreadyPresent type.
+ * This can be needed when facing KUDU-568. The effect of enabling this is that operation
+ * responses that match this pattern will be cleared of their row errors, meaning that we consider
+ * them successful.
+ * This is disabled by default.
+ * @param ignoreAllDuplicateRows true if this session should enforce this, else false
+ */
+ void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows);
+
+ /**
+ * Return the number of errors which are pending. Errors may accumulate when
+ * using the AUTO_FLUSH_BACKGROUND mode.
+ * @return a count of errors
+ */
+ int countPendingErrors();
+
+ /**
+ * Return any errors from previous calls. If there were more errors
+ * than could be held in the session's error storage, the overflow state is set to true.
+ * Resets the pending errors.
+ * @return an object that contains the errors and the overflow status
+ */
+ RowErrorsAndOverflowStatus getPendingErrors();
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java
new file mode 100644
index 0000000..ef45f9e
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Statistics.java
@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.Sets;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.util.Slice;
+import org.kududb.util.Slices;
+
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+
+/**
+ * A Statistics belongs to a specific AsyncKuduClient. It stores client-level
+ * statistics including number of operations, number of bytes written, number of
+ * rpcs. It is created along with the client's creation, and can be obtained through
+ * AsyncKuduClient or KuduClient's getStatistics method. Once obtained, an instance
+ * of this class can be used directly.
+ * <p>
+ * This class is thread-safe. The user can use it anywhere to get statistics of this
+ * client.
+ * <p>
+ * The method {@link #toString} can be useful to get a dump of all the metrics aggregated
+ * for all the tablets.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Statistics {
+ private final ConcurrentHashMap<Slice, Statistics.TabletStatistics> stsMap = new ConcurrentHashMap<>();
+
+ /**
+ * The statistic enum to pass when querying.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public enum Statistic {
+ /**
+ * How many bytes have been written by this client. If one rpc fails, this
+ * statistic won't be updated.
+ */
+ BYTES_WRITTEN(0),
+ /**
+ * How many operations have been sent to server and succeeded.
+ */
+ WRITE_OPS(1),
+ /**
+ * How many rpcs have been sent to server and succeeded. One rpc may contain
+ * multiple operations.
+ */
+ WRITE_RPCS(2),
+ /**
+ * How many operations have been sent to server but failed.
+ */
+ OPS_ERRORS(3),
+ /**
+ * How many rpcs have been sent to server but failed.
+ */
+ RPC_ERRORS(4);
+
+ Statistic(int idx) {
+ this.idx = idx;
+ }
+
+ /**
+ * Get index of this statistic.
+ * @return index
+ */
+ int getIndex() {
+ return this.idx;
+ }
+
+ private final int idx;
+ };
+
+ /**
+ * Get the statistic count of this tablet.
+ * If the specified tablet doesn't have statistics, 0 will be returned.
+ * @param tabletId the tablet's id
+ * @param statistic the statistic type to get
+ * @return the value of the statistic
+ */
+ public long getTabletStatistic(String tabletId, Statistic statistic) {
+ Slice tabletIdAsSlice = Slices.copiedBuffer(tabletId, Charset.defaultCharset());
+ TabletStatistics tabletStatistics = stsMap.get(tabletIdAsSlice);
+ if (tabletStatistics == null) {
+ return 0;
+ } else {
+ return tabletStatistics.getStatistic(statistic);
+ }
+ }
+
+ /**
+ * Get the statistic count of this table.
+ * @param tableName the table's name
+ * @param statistic the statistic type to get
+ * @return the value of the statistic
+ */
+ public long getTableStatistic(String tableName, Statistic statistic) {
+ long stsResult = 0;
+ for (TabletStatistics tabletStatistics : stsMap.values()) {
+ if (!tabletStatistics.tableName.equals(tableName)) {
+ continue;
+ }
+ stsResult += tabletStatistics.getStatistic(statistic);
+ }
+ return stsResult;
+ }
+
+ /**
+ * Get the statistic count of the whole client.
+ * @param statistic the statistic type to get
+ * @return the value of the statistic
+ */
+ public long getClientStatistic(Statistic statistic) {
+ long stsResult = 0;
+ for (TabletStatistics tabletStatistics : stsMap.values()) {
+ stsResult += tabletStatistics.getStatistic(statistic);
+ }
+ return stsResult;
+ }
+
+ /**
+ * Get the set of tablets which have been written into by this client,
+ * which have statistics information.
+ * @return set of tablet ids
+ */
+ public Set<String> getTabletSet() {
+ Set<String> tablets = Sets.newHashSet();
+ for (Slice tablet : stsMap.keySet()) {
+ tablets.add(tablet.toString(Charset.defaultCharset()));
+ }
+ return tablets;
+ }
+
+ /**
+ * Get the set of tables which have been written into by this client,
+ * which have statistics information.
+ * @return set of table names
+ */
+ public Set<String> getTableSet() {
+ Set<String> tables = Sets.newHashSet();
+ for (TabletStatistics tabletStat : stsMap.values()) {
+ tables.add(tabletStat.tableName);
+ }
+ return tables;
+ }
+
+ /**
+ * Get table name of the given tablet id.
+ * If the tablet has no statistics, null will be returned.
+ * @param tabletId the tablet's id
+ * @return table name
+ */
+ public String getTableName(String tabletId) {
+ Slice tabletIdAsSlice = Slices.copiedBuffer(tabletId, Charset.defaultCharset());
+ TabletStatistics tabletStatistics = stsMap.get(tabletIdAsSlice);
+ if (tabletStatistics == null) {
+ return null;
+ } else {
+ return tabletStatistics.tableName;
+ }
+ }
+
+ /**
+ * Get the TabletStatistics object for this specified tablet.
+ * @param tableName the table's name
+ * @param tabletId the tablet's id
+ * @return a TabletStatistics object
+ */
+ Statistics.TabletStatistics getTabletStatistics(String tableName, Slice tabletId) {
+ Statistics.TabletStatistics tabletStats = stsMap.get(tabletId);
+ if (tabletStats == null) {
+ Statistics.TabletStatistics newTabletStats = new Statistics.TabletStatistics(tableName,
+ tabletId.toString(Charset.defaultCharset()));
+ tabletStats = stsMap.putIfAbsent(tabletId, newTabletStats);
+ if (tabletStats == null) {
+ tabletStats = newTabletStats;
+ }
+ }
+ return tabletStats;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder buf = new StringBuilder();
+ buf.append("Current client statistics: ");
+ buf.append("bytes written:");
+ buf.append(getClientStatistic(Statistic.BYTES_WRITTEN));
+ buf.append(", write rpcs:");
+ buf.append(getClientStatistic(Statistic.WRITE_RPCS));
+ buf.append(", rpc errors:");
+ buf.append(getClientStatistic(Statistic.RPC_ERRORS));
+ buf.append(", write operations:");
+ buf.append(getClientStatistic(Statistic.WRITE_OPS));
+ buf.append(", operation errors:");
+ buf.append(getClientStatistic(Statistic.OPS_ERRORS));
+ return buf.toString();
+ }
+
+ static class TabletStatistics {
+ final private AtomicLongArray statistics;
+ final private String tableName;
+ final private String tabletId;
+
+ TabletStatistics(String tableName, String tabletId) {
+ this.tableName = tableName;
+ this.tabletId = tabletId;
+ this.statistics = new AtomicLongArray(Statistic.values().length);
+ }
+
+ void incrementStatistic(Statistic statistic, long count) {
+ this.statistics.addAndGet(statistic.getIndex(), count);
+ }
+
+ long getStatistic(Statistic statistic) {
+ return this.statistics.get(statistic.getIndex());
+ }
+
+ public String toString() {
+ final StringBuilder buf = new StringBuilder();
+ buf.append("Table: ");
+ buf.append(tableName);
+ buf.append(", tablet:");
+ buf.append(tabletId);
+ buf.append(", bytes written:");
+ buf.append(getStatistic(Statistic.BYTES_WRITTEN));
+ buf.append(", write rpcs:");
+ buf.append(getStatistic(Statistic.WRITE_RPCS));
+ buf.append(", rpc errors:");
+ buf.append(getStatistic(Statistic.RPC_ERRORS));
+ buf.append(", write operations:");
+ buf.append(getStatistic(Statistic.WRITE_OPS));
+ buf.append(", operation errors:");
+ buf.append(getStatistic(Statistic.OPS_ERRORS));
+ return buf.toString();
+ }
+ }
+}
[20/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
deleted file mode 100644
index c1ccb3f..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
+++ /dev/null
@@ -1,2437 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-package org.kududb.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.net.HostAndPort;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.Message;
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.channel.socket.nio.NioWorkerPool;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
-import org.kududb.master.Master.GetTableLocationsResponsePB;
-import org.kududb.util.AsyncUtil;
-import org.kududb.util.NetUtil;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
-import org.jboss.netty.channel.ChannelEvent;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.DefaultChannelPipeline;
-import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
-import org.jboss.netty.channel.socket.SocketChannel;
-import org.jboss.netty.channel.socket.SocketChannelConfig;
-import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
-import org.jboss.netty.handler.timeout.ReadTimeoutHandler;
-import org.jboss.netty.util.HashedWheelTimer;
-import org.jboss.netty.util.Timeout;
-import org.jboss.netty.util.TimerTask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.concurrent.GuardedBy;
-
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
-
-/**
- * A fully asynchronous and thread-safe client for Kudu.
- * <p>
- * This client should be
- * instantiated only once. You can use it with any number of tables at the
- * same time. The only case where you should have multiple instances is when
- * you want to use multiple different clusters at the same time.
- * <p>
- * If you play by the rules, this client is completely
- * thread-safe. Read the documentation carefully to know what the requirements
- * are for this guarantee to apply.
- * <p>
- * This client is fully non-blocking, any blocking operation will return a
- * {@link Deferred} instance to which you can attach a {@link Callback} chain
- * that will execute when the asynchronous operation completes.
- *
- * <h1>Note regarding {@code KuduRpc} instances passed to this class</h1>
- * Every {@link KuduRpc} passed to a method of this class should not be
- * changed or re-used until the {@code Deferred} returned by that method
- * calls you back. <strong>Changing or re-using any {@link KuduRpc} for
- * an RPC in flight will lead to <em>unpredictable</em> results and voids
- * your warranty</strong>.
- *
- * <h1>{@code throws} clauses</h1>
- * None of the asynchronous methods in this API are expected to throw an
- * exception. But the {@link Deferred} object they return to you can carry an
- * exception that you should handle (using "errbacks", see the javadoc of
- * {@link Deferred}). In order to be able to do proper asynchronous error
- * handling, you need to know what types of exceptions you're expected to face
- * in your errbacks. In order to document that, the methods of this API use
- * javadoc's {@code @throws} to spell out the exception types you should
- * handle in your errback. Asynchronous exceptions will be indicated as such
- * in the javadoc with "(deferred)".
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class AsyncKuduClient implements AutoCloseable {
-
- public static final Logger LOG = LoggerFactory.getLogger(AsyncKuduClient.class);
- public static final int SLEEP_TIME = 500;
- public static final byte[] EMPTY_ARRAY = new byte[0];
- public static final long NO_TIMESTAMP = -1;
- public static final long DEFAULT_OPERATION_TIMEOUT_MS = 30000;
- public static final long DEFAULT_SOCKET_READ_TIMEOUT_MS = 10000;
- private static final long MAX_RPC_ATTEMPTS = 100;
-
- private final ClientSocketChannelFactory channelFactory;
-
- /**
- * This map and the next 2 maps contain the same data, but indexed
- * differently. There is no consistency guarantee across the maps.
- * They are not updated all at the same time atomically. This map
- * is always the first to be updated, because that's the map from
- * which all the lookups are done in the fast-path of the requests
- * that need to locate a tablet. The second map to be updated is
- * tablet2client, because it comes second in the fast-path
- * of every requests that need to locate a tablet. The third map
- * is only used to handle TabletServer disconnections gracefully.
- *
- * This map is keyed by table ID.
- */
- private final ConcurrentHashMap<String, ConcurrentSkipListMap<byte[],
- RemoteTablet>> tabletsCache = new ConcurrentHashMap<>();
-
- /**
- * Maps a tablet ID to the RemoteTablet that knows where all the replicas are served.
- */
- private final ConcurrentHashMap<Slice, RemoteTablet> tablet2client = new ConcurrentHashMap<>();
-
- /**
- * Maps a client connected to a TabletServer to the list of tablets we know
- * it's serving so far.
- */
- private final ConcurrentHashMap<TabletClient, ArrayList<RemoteTablet>> client2tablets =
- new ConcurrentHashMap<>();
-
- /**
- * Map of table ID to non-covered range cache.
- *
- * TODO: Currently once a non-covered range is added to the cache, it is never
- * removed. Once adding range partitions becomes possible entries will need to
- * be expired.
- */
- private final ConcurrentMap<String, NonCoveredRangeCache> nonCoveredRangeCaches =
- new ConcurrentHashMap<>();
-
- /**
- * Cache that maps a TabletServer address ("ip:port") to the clients
- * connected to it.
- * <p>
- * Access to this map must be synchronized by locking its monitor.
- * Lock ordering: when locking both this map and a TabletClient, the
- * TabletClient must always be locked first to avoid deadlocks. Logging
- * the contents of this map (or calling toString) requires copying it first.
- * <p>
- * This isn't a {@link ConcurrentHashMap} because we don't use it frequently
- * (just when connecting to / disconnecting from TabletClients) and when we
- * add something to it, we want to do an atomic get-and-put, but
- * {@code putIfAbsent} isn't a good fit for us since it requires to create
- * an object that may be "wasted" in case another thread wins the insertion
- * race, and we don't want to create unnecessary connections.
- * <p>
- * Upon disconnection, clients are automatically removed from this map.
- * We don't use a {@code ChannelGroup} because a {@code ChannelGroup} does
- * the clean-up on the {@code channelClosed} event, which is actually the
- * 3rd and last event to be fired when a channel gets disconnected. The
- * first one to get fired is, {@code channelDisconnected}. This matters to
- * us because we want to purge disconnected clients from the cache as
- * quickly as possible after the disconnection, to avoid handing out clients
- * that are going to cause unnecessary errors.
- * @see TabletClientPipeline#handleDisconnect
- */
- private final HashMap<String, TabletClient> ip2client =
- new HashMap<String, TabletClient>();
-
- @GuardedBy("sessions")
- private final Set<AsyncKuduSession> sessions = new HashSet<AsyncKuduSession>();
-
- // Since the masters also go through TabletClient, we need to treat them as if they were a normal
- // table. We'll use the following fake table name to identify places where we need special
- // handling.
- static final String MASTER_TABLE_NAME_PLACEHOLDER = "Kudu Master";
- final KuduTable masterTable;
- private final List<HostAndPort> masterAddresses;
-
- private final HashedWheelTimer timer;
-
- /**
- * Timestamp required for HybridTime external consistency through timestamp
- * propagation.
- * @see src/kudu/common/common.proto
- */
- private long lastPropagatedTimestamp = NO_TIMESTAMP;
-
- // A table is considered not served when we get an empty list of locations but know
- // that a tablet exists. This is currently only used for new tables. The objects stored are
- // table IDs.
- private final Set<String> tablesNotServed = Collections.newSetFromMap(new
- ConcurrentHashMap<String, Boolean>());
-
- /**
- * Semaphore used to rate-limit master lookups
- * Once we have more than this number of concurrent master lookups, we'll
- * start to throttle ourselves slightly.
- * @see #acquireMasterLookupPermit
- */
- private final Semaphore masterLookups = new Semaphore(50);
-
- private final Random sleepRandomizer = new Random();
-
- private final long defaultOperationTimeoutMs;
-
- private final long defaultAdminOperationTimeoutMs;
-
- private final long defaultSocketReadTimeoutMs;
-
- private final Statistics statistics;
-
- private final boolean statisticsDisabled;
-
- private final RequestTracker requestTracker;
-
- private volatile boolean closed;
-
- private AsyncKuduClient(AsyncKuduClientBuilder b) {
- this.channelFactory = b.createChannelFactory();
- this.masterAddresses = b.masterAddresses;
- this.masterTable = new KuduTable(this, MASTER_TABLE_NAME_PLACEHOLDER,
- MASTER_TABLE_NAME_PLACEHOLDER, null, null);
- this.defaultOperationTimeoutMs = b.defaultOperationTimeoutMs;
- this.defaultAdminOperationTimeoutMs = b.defaultAdminOperationTimeoutMs;
- this.defaultSocketReadTimeoutMs = b.defaultSocketReadTimeoutMs;
- this.statisticsDisabled = b.statisticsDisabled;
- statistics = statisticsDisabled ? null : new Statistics();
- this.timer = b.timer;
- String clientId = UUID.randomUUID().toString().replace("-", "");
- this.requestTracker = new RequestTracker(clientId);
- }
-
- /**
- * Updates the last timestamp received from a server. Used for CLIENT_PROPAGATED
- * external consistency. This is only publicly visible so that it can be set
- * on tests, users should generally disregard this method.
- *
- * @param lastPropagatedTimestamp the last timestamp received from a server
- */
- @VisibleForTesting
- public synchronized void updateLastPropagatedTimestamp(long lastPropagatedTimestamp) {
- if (this.lastPropagatedTimestamp == -1 ||
- this.lastPropagatedTimestamp < lastPropagatedTimestamp) {
- this.lastPropagatedTimestamp = lastPropagatedTimestamp;
- }
- }
-
- @VisibleForTesting
- public synchronized long getLastPropagatedTimestamp() {
- return lastPropagatedTimestamp;
- }
-
- /**
- * Returns a synchronous {@link KuduClient} which wraps this asynchronous client.
- * Calling {@link KuduClient#close} on the returned client will close this client.
- * If this asynchronous client should outlive the returned synchronous client,
- * then do not close the synchronous client.
- * @return a new synchronous {@code KuduClient}
- */
- public KuduClient syncClient() {
- return new KuduClient(this);
- }
-
- /**
- * Create a table on the cluster with the specified name, schema, and table configurations.
- * @param name the table's name
- * @param schema the table's schema
- * @param builder a builder containing the table's configurations
- * @return a deferred object to track the progress of the createTable command that gives
- * an object to communicate with the created table
- */
- public Deferred<KuduTable> createTable(final String name, Schema schema,
- CreateTableOptions builder) {
- checkIsClosed();
- if (builder == null) {
- throw new IllegalArgumentException("CreateTableOptions may not be null");
- }
- if (!builder.getBuilder().getPartitionSchema().hasRangeSchema() &&
- builder.getBuilder().getPartitionSchema().getHashBucketSchemasCount() == 0) {
- throw new IllegalArgumentException("Table partitioning must be specified using " +
- "setRangePartitionColumns or addHashPartitions");
-
- }
- CreateTableRequest create = new CreateTableRequest(this.masterTable, name, schema, builder);
- create.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- return sendRpcToTablet(create).addCallbackDeferring(
- new Callback<Deferred<KuduTable>, CreateTableResponse>() {
- @Override
- public Deferred<KuduTable> call(CreateTableResponse createTableResponse) throws Exception {
- return openTable(name);
- }
- });
- }
-
- /**
- * Delete a table on the cluster with the specified name.
- * @param name the table's name
- * @return a deferred object to track the progress of the deleteTable command
- */
- public Deferred<DeleteTableResponse> deleteTable(String name) {
- checkIsClosed();
- DeleteTableRequest delete = new DeleteTableRequest(this.masterTable, name);
- delete.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- return sendRpcToTablet(delete);
- }
-
- /**
- * Alter a table on the cluster as specified by the builder.
- *
- * When the returned deferred completes it only indicates that the master accepted the alter
- * command, use {@link AsyncKuduClient#isAlterTableDone(String)} to know when the alter finishes.
- * @param name the table's name, if this is a table rename then the old table name must be passed
- * @param ato the alter table builder
- * @return a deferred object to track the progress of the alter command
- */
- public Deferred<AlterTableResponse> alterTable(String name, AlterTableOptions ato) {
- checkIsClosed();
- AlterTableRequest alter = new AlterTableRequest(this.masterTable, name, ato);
- alter.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- return sendRpcToTablet(alter);
- }
-
- /**
- * Helper method that checks and waits until the completion of an alter command.
- * It will block until the alter command is done or the deadline is reached.
- * @param name the table's name, if the table was renamed then that name must be checked against
- * @return a deferred object to track the progress of the isAlterTableDone command
- */
- public Deferred<IsAlterTableDoneResponse> isAlterTableDone(String name) {
- checkIsClosed();
- IsAlterTableDoneRequest request = new IsAlterTableDoneRequest(this.masterTable, name);
- request.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- return sendRpcToTablet(request);
- }
-
- /**
- * Get the list of running tablet servers.
- * @return a deferred object that yields a list of tablet servers
- */
- public Deferred<ListTabletServersResponse> listTabletServers() {
- checkIsClosed();
- ListTabletServersRequest rpc = new ListTabletServersRequest(this.masterTable);
- rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- return sendRpcToTablet(rpc);
- }
-
- Deferred<GetTableSchemaResponse> getTableSchema(String name) {
- GetTableSchemaRequest rpc = new GetTableSchemaRequest(this.masterTable, name);
- rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- return sendRpcToTablet(rpc);
- }
-
- /**
- * Get the list of all the tables.
- * @return a deferred object that yields a list of all the tables
- */
- public Deferred<ListTablesResponse> getTablesList() {
- return getTablesList(null);
- }
-
- /**
- * Get a list of table names. Passing a null filter returns all the tables. When a filter is
- * specified, it only returns tables that satisfy a substring match.
- * @param nameFilter an optional table name filter
- * @return a deferred that yields the list of table names
- */
- public Deferred<ListTablesResponse> getTablesList(String nameFilter) {
- ListTablesRequest rpc = new ListTablesRequest(this.masterTable, nameFilter);
- rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- return sendRpcToTablet(rpc);
- }
-
- /**
- * Test if a table exists.
- * @param name a non-null table name
- * @return true if the table exists, else false
- */
- public Deferred<Boolean> tableExists(final String name) {
- if (name == null) {
- throw new IllegalArgumentException("The table name cannot be null");
- }
- return getTablesList().addCallbackDeferring(new Callback<Deferred<Boolean>,
- ListTablesResponse>() {
- @Override
- public Deferred<Boolean> call(ListTablesResponse listTablesResponse) throws Exception {
- for (String tableName : listTablesResponse.getTablesList()) {
- if (name.equals(tableName)) {
- return Deferred.fromResult(true);
- }
- }
- return Deferred.fromResult(false);
- }
- });
- }
-
- /**
- * Open the table with the given name. If the table was just created, the Deferred will only get
- * called back when all the tablets have been successfully created.
- * @param name table to open
- * @return a KuduTable if the table exists, else a MasterErrorException
- */
- public Deferred<KuduTable> openTable(final String name) {
- checkIsClosed();
-
- // We create an RPC that we're never going to send, and will instead use it to keep track of
- // timeouts and use its Deferred.
- final KuduRpc<KuduTable> fakeRpc = new KuduRpc<KuduTable>(null) {
- @Override
- ChannelBuffer serialize(Message header) { return null; }
-
- @Override
- String serviceName() { return null; }
-
- @Override
- String method() {
- return "IsCreateTableDone";
- }
-
- @Override
- Pair<KuduTable, Object> deserialize(CallResponse callResponse, String tsUUID)
- throws Exception { return null; }
- };
- fakeRpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
-
- return getTableSchema(name).addCallbackDeferring(new Callback<Deferred<KuduTable>,
- GetTableSchemaResponse>() {
- @Override
- public Deferred<KuduTable> call(GetTableSchemaResponse response) throws Exception {
- KuduTable table = new KuduTable(AsyncKuduClient.this,
- name,
- response.getTableId(),
- response.getSchema(),
- response.getPartitionSchema());
- // We grab the Deferred first because calling callback on the RPC will reset it and we'd
- // return a different, non-triggered Deferred.
- Deferred<KuduTable> d = fakeRpc.getDeferred();
- if (response.isCreateTableDone()) {
- LOG.debug("Opened table {}", name);
- fakeRpc.callback(table);
- } else {
- LOG.debug("Delaying opening table {}, its tablets aren't fully created", name);
- fakeRpc.attempt++;
- delayedIsCreateTableDone(
- table,
- fakeRpc,
- getOpenTableCB(fakeRpc, table),
- getDelayedIsCreateTableDoneErrback(fakeRpc));
- }
- return d;
- }
- });
- }
-
- /**
- * This callback will be repeatadly used when opening a table until it is done being created.
- */
- Callback<Deferred<KuduTable>, Master.IsCreateTableDoneResponsePB> getOpenTableCB(
- final KuduRpc<KuduTable> rpc, final KuduTable table) {
- return new Callback<Deferred<KuduTable>, Master.IsCreateTableDoneResponsePB>() {
- @Override
- public Deferred<KuduTable> call(
- Master.IsCreateTableDoneResponsePB isCreateTableDoneResponsePB) throws Exception {
- String tableName = table.getName();
- Deferred<KuduTable> d = rpc.getDeferred();
- if (isCreateTableDoneResponsePB.getDone()) {
- LOG.debug("Table {}'s tablets are now created", tableName);
- rpc.callback(table);
- } else {
- rpc.attempt++;
- LOG.debug("Table {}'s tablets are still not created, further delaying opening it",
- tableName);
-
- delayedIsCreateTableDone(
- table,
- rpc,
- getOpenTableCB(rpc, table),
- getDelayedIsCreateTableDoneErrback(rpc));
- }
- return d;
- }
- };
- }
-
- /**
- * Get the timeout used for operations on sessions and scanners.
- * @return a timeout in milliseconds
- */
- public long getDefaultOperationTimeoutMs() {
- return defaultOperationTimeoutMs;
- }
-
- /**
- * Get the timeout used for admin operations.
- * @return a timeout in milliseconds
- */
- public long getDefaultAdminOperationTimeoutMs() {
- return defaultAdminOperationTimeoutMs;
- }
-
- /**
- * Get the timeout used when waiting to read data from a socket. Will be triggered when nothing
- * has been read on a socket connected to a tablet server for {@code timeout} milliseconds.
- * @return a timeout in milliseconds
- */
- public long getDefaultSocketReadTimeoutMs() {
- return defaultSocketReadTimeoutMs;
- }
-
- /**
- * Check if statistics collection is enabled for this client.
- * @return true if it is enabled, else false
- */
- public boolean isStatisticsEnabled() {
- return !statisticsDisabled;
- }
-
- /**
- * Get the statistics object of this client.
- *
- * @return this client's Statistics object
- * @throws IllegalStateException thrown if statistics collection has been disabled
- */
- public Statistics getStatistics() {
- if (statisticsDisabled) {
- throw new IllegalStateException("This client's statistics is disabled");
- }
- return this.statistics;
- }
-
- RequestTracker getRequestTracker() {
- return requestTracker;
- }
-
- /**
- * Creates a new {@link AsyncKuduScanner.AsyncKuduScannerBuilder} for a particular table.
- * @param table the name of the table you intend to scan.
- * The string is assumed to use the platform's default charset.
- * @return a new scanner builder for this table
- */
- public AsyncKuduScanner.AsyncKuduScannerBuilder newScannerBuilder(KuduTable table) {
- checkIsClosed();
- return new AsyncKuduScanner.AsyncKuduScannerBuilder(this, table);
- }
-
- /**
- * Create a new session for interacting with the cluster.
- * User is responsible for destroying the session object.
- * This is a fully local operation (no RPCs or blocking).
- * @return a new AsyncKuduSession
- */
- public AsyncKuduSession newSession() {
- checkIsClosed();
- AsyncKuduSession session = new AsyncKuduSession(this);
- synchronized (sessions) {
- sessions.add(session);
- }
- return session;
- }
-
- /**
- * This method is for KuduSessions so that they can remove themselves as part of closing down.
- * @param session Session to remove
- */
- void removeSession(AsyncKuduSession session) {
- synchronized (sessions) {
- boolean removed = sessions.remove(session);
- assert removed == true;
- }
- }
-
- /**
- * Package-private access point for {@link AsyncKuduScanner}s to scan more rows.
- * @param scanner The scanner to use.
- * @return A deferred row.
- */
- Deferred<AsyncKuduScanner.Response> scanNextRows(final AsyncKuduScanner scanner) {
- final RemoteTablet tablet = scanner.currentTablet();
- final TabletClient client = clientFor(tablet);
- final KuduRpc<AsyncKuduScanner.Response> next_request = scanner.getNextRowsRequest();
- final Deferred<AsyncKuduScanner.Response> d = next_request.getDeferred();
- // Important to increment the attempts before the next if statement since
- // getSleepTimeForRpc() relies on it if the client is null or dead.
- next_request.attempt++;
- if (client == null || !client.isAlive()) {
- // A null client means we either don't know about this tablet anymore (unlikely) or we
- // couldn't find a leader (which could be triggered by a read timeout).
- // We'll first delay the RPC in case things take some time to settle down, then retry.
- delayedSendRpcToTablet(next_request, null);
- return next_request.getDeferred();
- }
- client.sendRpc(next_request);
- return d;
- }
-
- /**
- * Package-private access point for {@link AsyncKuduScanner}s to close themselves.
- * @param scanner the scanner to close
- * @return a deferred object that indicates the completion of the request.
- * The {@link AsyncKuduScanner.Response} can contain rows that were left to scan.
- */
- Deferred<AsyncKuduScanner.Response> closeScanner(final AsyncKuduScanner scanner) {
- final RemoteTablet tablet = scanner.currentTablet();
- // Getting a null tablet here without being in a closed state means we were in between tablets.
- if (tablet == null) {
- return Deferred.fromResult(null);
- }
-
- final TabletClient client = clientFor(tablet);
- if (client == null || !client.isAlive()) {
- // Oops, we couldn't find a tablet server that hosts this tablet. Our
- // cache was probably invalidated while the client was scanning. So
- // we can't close this scanner properly.
- LOG.warn("Cannot close {} properly, no connection open for {}", scanner, tablet);
- return Deferred.fromResult(null);
- }
- final KuduRpc<AsyncKuduScanner.Response> close_request = scanner.getCloseRequest();
- final Deferred<AsyncKuduScanner.Response> d = close_request.getDeferred();
- close_request.attempt++;
- client.sendRpc(close_request);
- return d;
- }
-
- /**
- * Sends the provided {@link KuduRpc} to the tablet server hosting the leader
- * of the tablet identified by the RPC's table and partition key.
- *
- * Note: despite the name, this method is also used for routing master
- * requests to the leader master instance since it's also handled like a tablet.
- *
- * @param request the RPC to send
- * @param <R> the expected return type of the RPC
- * @return a {@code Deferred} which will contain the response
- */
- <R> Deferred<R> sendRpcToTablet(final KuduRpc<R> request) {
- if (cannotRetryRequest(request)) {
- return tooManyAttemptsOrTimeout(request, null);
- }
- request.attempt++;
- final String tableId = request.getTable().getTableId();
- byte[] partitionKey = request.partitionKey();
- RemoteTablet tablet = getTablet(tableId, partitionKey);
-
- if (tablet == null && partitionKey != null) {
- // Check if the RPC is in a non-covered range.
- Map.Entry<byte[], byte[]> nonCoveredRange = getNonCoveredRange(tableId, partitionKey);
- if (nonCoveredRange != null) {
- return Deferred.fromError(new NonCoveredRangeException(nonCoveredRange.getKey(),
- nonCoveredRange.getValue()));
- }
- // Otherwise fall through to below where a GetTableLocations lookup will occur.
- }
-
- // Set the propagated timestamp so that the next time we send a message to
- // the server the message includes the last propagated timestamp.
- long lastPropagatedTs = getLastPropagatedTimestamp();
- if (request.getExternalConsistencyMode() == CLIENT_PROPAGATED &&
- lastPropagatedTs != NO_TIMESTAMP) {
- request.setPropagatedTimestamp(lastPropagatedTs);
- }
-
- // If we found a tablet, we'll try to find the TS to talk to. If that TS was previously
- // disconnected, say because we didn't query that tablet for some seconds, then we'll try to
- // reconnect based on the old information. If that fails, we'll instead continue with the next
- // block that queries the master.
- if (tablet != null) {
- TabletClient tabletClient = clientFor(tablet);
- if (tabletClient != null) {
- final Deferred<R> d = request.getDeferred();
- if (tabletClient.isAlive()) {
- request.setTablet(tablet);
- tabletClient.sendRpc(request);
- return d;
- }
- try {
- tablet.reconnectTabletClient(tabletClient);
- } catch (UnknownHostException e) {
- LOG.error("Cached tablet server {}'s host cannot be resolved, will query the master",
- tabletClient.getUuid(), e);
- // Because of this exception, clientFor() below won't be able to find a newTabletClient
- // and we'll delay the RPC.
- }
- TabletClient newTabletClient = clientFor(tablet);
- assert (tabletClient != newTabletClient);
-
- if (newTabletClient == null) {
- // Wait a little bit before hitting the master.
- delayedSendRpcToTablet(request, null);
- return request.getDeferred();
- }
-
- if (!newTabletClient.isAlive()) {
- LOG.debug("Tried reconnecting to tablet server {} but failed, " +
- "will query the master", tabletClient.getUuid());
- // Let fall through.
- } else {
- request.setTablet(tablet);
- newTabletClient.sendRpc(request);
- return d;
- }
- }
- }
-
- // We fall through to here in two cases:
- //
- // 1) This client has not yet discovered the tablet which is responsible for
- // the RPC's table and partition key. This can happen when the client's
- // tablet location cache is cold because the client is new, or the table
- // is new.
- //
- // 2) The tablet is known, but we do not have an active client for the
- // leader replica.
- if (tablesNotServed.contains(tableId)) {
- return delayedIsCreateTableDone(request.getTable(), request,
- new RetryRpcCB<R, Master.IsCreateTableDoneResponsePB>(request),
- getDelayedIsCreateTableDoneErrback(request));
- }
- Callback<Deferred<R>, Master.GetTableLocationsResponsePB> cb = new RetryRpcCB<>(request);
- Callback<Deferred<R>, Exception> eb = new RetryRpcErrback<>(request);
- Deferred<Master.GetTableLocationsResponsePB> returnedD =
- locateTablet(request.getTable(), partitionKey);
- return AsyncUtil.addCallbacksDeferring(returnedD, cb, eb);
- }
-
- /**
- * Callback used to retry a RPC after another query finished, like looking up where that RPC
- * should go.
- * <p>
- * Use {@code AsyncUtil.addCallbacksDeferring} to add this as the callback and
- * {@link AsyncKuduClient.RetryRpcErrback} as the "errback" to the {@code Deferred}
- * returned by {@link #locateTablet(KuduTable, byte[])}.
- * @param <R> RPC's return type.
- * @param <D> Previous query's return type, which we don't use, but need to specify in order to
- * tie it all together.
- */
- final class RetryRpcCB<R, D> implements Callback<Deferred<R>, D> {
- private final KuduRpc<R> request;
- RetryRpcCB(KuduRpc<R> request) {
- this.request = request;
- }
- public Deferred<R> call(final D arg) {
- LOG.debug("Retrying sending RPC {} after lookup", request);
- return sendRpcToTablet(request); // Retry the RPC.
- }
- public String toString() {
- return "retry RPC";
- }
- }
-
- /**
- * "Errback" used to delayed-retry a RPC if it fails due to no leader master being found.
- * Other exceptions are used to notify request RPC error, and passed through to be handled
- * by the caller.
- * <p>
- * Use {@code AsyncUtil.addCallbacksDeferring} to add this as the "errback" and
- * {@link RetryRpcCB} as the callback to the {@code Deferred} returned by
- * {@link #locateTablet(KuduTable, byte[])}.
- * @see #delayedSendRpcToTablet(KuduRpc, KuduException)
- * @param <R> The type of the original RPC.
- */
- final class RetryRpcErrback<R> implements Callback<Deferred<R>, Exception> {
- private final KuduRpc<R> request;
-
- public RetryRpcErrback(KuduRpc<R> request) {
- this.request = request;
- }
-
- @Override
- public Deferred<R> call(Exception arg) {
- if (arg instanceof NoLeaderMasterFoundException) {
- // If we could not find the leader master, try looking up the leader master
- // again.
- // TODO: Handle the situation when multiple in-flight RPCs are queued waiting
- // for the leader master to be determine (either after a failure or at initialization
- // time). This could re-use some of the existing piping in place for non-master tablets.
- Deferred<R> d = request.getDeferred();
- delayedSendRpcToTablet(request, (NoLeaderMasterFoundException) arg);
- return d;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("Notify RPC %s after lookup exception", request), arg);
- }
- request.errback(arg);
- return Deferred.fromError(arg);
- }
-
- @Override
- public String toString() {
- return "retry RPC after error";
- }
- }
-
- /**
- * This errback ensures that if the delayed call to IsCreateTableDone throws an Exception that
- * it will be propagated back to the user.
- * @param request Request to errback if there's a problem with the delayed call.
- * @param <R> Request's return type.
- * @return An errback.
- */
- <R> Callback<Exception, Exception> getDelayedIsCreateTableDoneErrback(final KuduRpc<R> request) {
- return new Callback<Exception, Exception>() {
- @Override
- public Exception call(Exception e) throws Exception {
- // TODO maybe we can retry it?
- request.errback(e);
- return e;
- }
- };
- }
-
- /**
- * This method will call IsCreateTableDone on the master after sleeping for
- * getSleepTimeForRpc() based on the provided KuduRpc's number of attempts. Once this is done,
- * the provided callback will be called.
- * @param table the table to lookup
- * @param rpc the original KuduRpc that needs to access the table
- * @param retryCB the callback to call on completion
- * @param errback the errback to call if something goes wrong when calling IsCreateTableDone
- * @return Deferred used to track the provided KuduRpc
- */
- <R> Deferred<R> delayedIsCreateTableDone(final KuduTable table, final KuduRpc<R> rpc,
- final Callback<Deferred<R>,
- Master.IsCreateTableDoneResponsePB> retryCB,
- final Callback<Exception, Exception> errback) {
-
- final class RetryTimer implements TimerTask {
- public void run(final Timeout timeout) {
- String tableId = table.getTableId();
- final boolean has_permit = acquireMasterLookupPermit();
- if (!has_permit) {
- // If we failed to acquire a permit, it's worth checking if someone
- // looked up the tablet we're interested in. Every once in a while
- // this will save us a Master lookup.
- if (!tablesNotServed.contains(tableId)) {
- try {
- retryCB.call(null);
- return;
- } catch (Exception e) {
- // we're calling RetryRpcCB which doesn't throw exceptions, ignore
- }
- }
- }
- IsCreateTableDoneRequest rpc = new IsCreateTableDoneRequest(masterTable, tableId);
- rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- final Deferred<Master.IsCreateTableDoneResponsePB> d =
- sendRpcToTablet(rpc).addCallback(new IsCreateTableDoneCB(tableId));
- if (has_permit) {
- // The errback is needed here to release the lookup permit
- d.addCallbacks(new ReleaseMasterLookupPermit<Master.IsCreateTableDoneResponsePB>(),
- new ReleaseMasterLookupPermit<Exception>());
- }
- d.addCallbacks(retryCB, errback);
- }
- }
- long sleepTime = getSleepTimeForRpc(rpc);
- if (rpc.deadlineTracker.wouldSleepingTimeout(sleepTime)) {
- return tooManyAttemptsOrTimeout(rpc, null);
- }
-
- newTimeout(new RetryTimer(), sleepTime);
- return rpc.getDeferred();
- }
-
- private final class ReleaseMasterLookupPermit<T> implements Callback<T, T> {
- public T call(final T arg) {
- releaseMasterLookupPermit();
- return arg;
- }
- public String toString() {
- return "release master lookup permit";
- }
- }
-
- /** Callback executed when IsCreateTableDone completes. */
- private final class IsCreateTableDoneCB implements Callback<Master.IsCreateTableDoneResponsePB,
- Master.IsCreateTableDoneResponsePB> {
- final String tableName;
- IsCreateTableDoneCB(String tableName) {
- this.tableName = tableName;
- }
- public Master.IsCreateTableDoneResponsePB call(final Master.IsCreateTableDoneResponsePB response) {
- if (response.getDone()) {
- LOG.debug("Table {} was created", tableName);
- tablesNotServed.remove(tableName);
- } else {
- LOG.debug("Table {} is still being created", tableName);
- }
- return response;
- }
- public String toString() {
- return "ask the master if " + tableName + " was created";
- }
- }
-
- boolean isTableNotServed(String tableId) {
- return tablesNotServed.contains(tableId);
- }
-
-
- long getSleepTimeForRpc(KuduRpc<?> rpc) {
- byte attemptCount = rpc.attempt;
- assert (attemptCount > 0);
- if (attemptCount == 0) {
- LOG.warn("Possible bug: attempting to retry an RPC with no attempts. RPC: " + rpc,
- new Exception("Exception created to collect stack trace"));
- attemptCount = 1;
- }
- // Randomized exponential backoff, truncated at 4096ms.
- long sleepTime = (long)(Math.pow(2.0, Math.min(attemptCount, 12))
- * sleepRandomizer.nextDouble());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Going to sleep for " + sleepTime + " at retry " + rpc.attempt);
- }
- return sleepTime;
- }
-
- /**
- * Modifying the list returned by this method won't change how AsyncKuduClient behaves,
- * but calling certain methods on the returned TabletClients can. For example,
- * it's possible to forcefully shutdown a connection to a tablet server by calling {@link
- * TabletClient#shutdown()}.
- * @return Copy of the current TabletClients list
- */
- @VisibleForTesting
- List<TabletClient> getTabletClients() {
- synchronized (ip2client) {
- return new ArrayList<TabletClient>(ip2client.values());
- }
- }
-
- /**
- * This method first clears tabletsCache and then tablet2client without any regards for
- * calls to {@link #discoverTablets}. Call only when AsyncKuduClient is in a steady state.
- * @param tableId table for which we remove all the RemoteTablet entries
- */
- @VisibleForTesting
- void emptyTabletsCacheForTable(String tableId) {
- tabletsCache.remove(tableId);
- Set<Map.Entry<Slice, RemoteTablet>> tablets = tablet2client.entrySet();
- for (Map.Entry<Slice, RemoteTablet> entry : tablets) {
- if (entry.getValue().getTableId().equals(tableId)) {
- tablets.remove(entry);
- }
- }
- }
-
- TabletClient clientFor(RemoteTablet tablet) {
- if (tablet == null) {
- return null;
- }
-
- synchronized (tablet.tabletServers) {
- if (tablet.tabletServers.isEmpty()) {
- return null;
- }
- if (tablet.leaderIndex == RemoteTablet.NO_LEADER_INDEX) {
- // TODO we don't know where the leader is, either because one wasn't provided or because
- // we couldn't resolve its IP. We'll just send the client back so it retries and probably
- // dies after too many attempts.
- return null;
- } else {
- // TODO we currently always hit the leader, we probably don't need to except for writes
- // and some reads.
- return tablet.tabletServers.get(tablet.leaderIndex);
- }
- }
- }
-
- /**
- * Checks whether or not an RPC can be retried once more
- * @param rpc The RPC we're going to attempt to execute
- * @return {@code true} if this RPC already had too many attempts,
- * {@code false} otherwise (in which case it's OK to retry once more)
- */
- static boolean cannotRetryRequest(final KuduRpc<?> rpc) {
- return rpc.deadlineTracker.timedOut() || rpc.attempt > MAX_RPC_ATTEMPTS;
- }
-
- /**
- * Returns a {@link Deferred} containing an exception when an RPC couldn't
- * succeed after too many attempts or if it already timed out.
- * @param request The RPC that was retried too many times or timed out.
- * @param cause What was cause of the last failed attempt, if known.
- * You can pass {@code null} if the cause is unknown.
- */
- static <R> Deferred<R> tooManyAttemptsOrTimeout(final KuduRpc<R> request,
- final KuduException cause) {
- String message;
- if (request.attempt > MAX_RPC_ATTEMPTS) {
- message = "Too many attempts: ";
- } else {
- message = "RPC can not complete before timeout: ";
- }
- Status statusTimedOut = Status.TimedOut(message + request);
- final Exception e = new NonRecoverableException(statusTimedOut, cause);
- request.errback(e);
- LOG.debug("Cannot continue with this RPC: {} because of: {}", request, message, e);
- return Deferred.fromError(e);
- }
-
- /**
- * Sends a getTableLocations RPC to the master to find the table's tablets.
- * @param table table to lookup
- * @param partitionKey can be null, if not we'll find the exact tablet that contains it
- * @return Deferred to track the progress
- */
- private Deferred<Master.GetTableLocationsResponsePB> locateTablet(KuduTable table,
- byte[] partitionKey) {
- final boolean has_permit = acquireMasterLookupPermit();
- String tableId = table.getTableId();
- if (!has_permit) {
- // If we failed to acquire a permit, it's worth checking if someone
- // looked up the tablet we're interested in. Every once in a while
- // this will save us a Master lookup.
- RemoteTablet tablet = getTablet(tableId, partitionKey);
- if (tablet != null && clientFor(tablet) != null) {
- return Deferred.fromResult(null); // Looks like no lookup needed.
- }
- }
- // Leave the end of the partition key range empty in order to pre-fetch tablet locations.
- GetTableLocationsRequest rpc =
- new GetTableLocationsRequest(masterTable, partitionKey, null, tableId);
- rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- final Deferred<Master.GetTableLocationsResponsePB> d;
-
- // If we know this is going to the master, check the master consensus
- // configuration (as specified by 'masterAddresses' field) to determine and
- // cache the current leader.
- if (isMasterTable(tableId)) {
- d = getMasterTableLocationsPB();
- } else {
- d = sendRpcToTablet(rpc);
- }
- d.addCallback(new MasterLookupCB(table, partitionKey));
- if (has_permit) {
- d.addBoth(new ReleaseMasterLookupPermit<Master.GetTableLocationsResponsePB>());
- }
- return d;
- }
-
- /**
- * Update the master config: send RPCs to all config members, use the returned data to
- * fill a {@link Master.GetTabletLocationsResponsePB} object.
- * @return An initialized Deferred object to hold the response.
- */
- Deferred<Master.GetTableLocationsResponsePB> getMasterTableLocationsPB() {
- final Deferred<Master.GetTableLocationsResponsePB> responseD = new Deferred<>();
- final GetMasterRegistrationReceived received =
- new GetMasterRegistrationReceived(masterAddresses, responseD);
- for (HostAndPort hostAndPort : masterAddresses) {
- Deferred<GetMasterRegistrationResponse> d;
- // Note: we need to create a client for that host first, as there's a
- // chicken and egg problem: since there is no source of truth beyond
- // the master, the only way to get information about a master host is
- // by making an RPC to that host.
- TabletClient clientForHostAndPort = newMasterClient(hostAndPort);
- if (clientForHostAndPort == null) {
- String message = "Couldn't resolve this master's address " + hostAndPort.toString();
- LOG.warn(message);
- Status statusIOE = Status.IOError(message);
- d = Deferred.fromError(new NonRecoverableException(statusIOE));
- } else {
- d = getMasterRegistration(clientForHostAndPort);
- }
- d.addCallbacks(received.callbackForNode(hostAndPort), received.errbackForNode(hostAndPort));
- }
- return responseD;
- }
-
-
- /**
- * Get all or some tablets for a given table. This may query the master multiple times if there
- * are a lot of tablets.
- * This method blocks until it gets all the tablets.
- * @param table the table to locate tablets from
- * @param startPartitionKey where to start in the table, pass null to start at the beginning
- * @param endPartitionKey where to stop in the table, pass null to get all the tablets until the
- * end of the table
- * @param deadline deadline in milliseconds for this method to finish
- * @return a list of the tablets in the table, which can be queried for metadata about
- * each tablet
- * @throws Exception MasterErrorException if the table doesn't exist
- */
- List<LocatedTablet> syncLocateTable(KuduTable table,
- byte[] startPartitionKey,
- byte[] endPartitionKey,
- long deadline) throws Exception {
- return locateTable(table, startPartitionKey, endPartitionKey, deadline).join();
- }
-
- private Deferred<List<LocatedTablet>> loopLocateTable(final KuduTable table,
- final byte[] startPartitionKey,
- final byte[] endPartitionKey,
- final List<LocatedTablet> ret,
- final DeadlineTracker deadlineTracker) {
- // We rely on the keys initially not being empty.
- Preconditions.checkArgument(startPartitionKey == null || startPartitionKey.length > 0,
- "use null for unbounded start partition key");
- Preconditions.checkArgument(endPartitionKey == null || endPartitionKey.length > 0,
- "use null for unbounded end partition key");
-
- // The next partition key to look up. If null, then it represents
- // the minimum partition key, If empty, it represents the maximum key.
- byte[] partitionKey = startPartitionKey;
- String tableId = table.getTableId();
-
- // Continue while the partition key is the minimum, or it is not the maximum
- // and it is less than the end partition key.
- while (partitionKey == null ||
- (partitionKey.length > 0 &&
- (endPartitionKey == null || Bytes.memcmp(partitionKey, endPartitionKey) < 0))) {
- byte[] key = partitionKey == null ? EMPTY_ARRAY : partitionKey;
- RemoteTablet tablet = getTablet(tableId, key);
- if (tablet != null) {
- ret.add(new LocatedTablet(tablet));
- partitionKey = tablet.getPartition().getPartitionKeyEnd();
- continue;
- }
-
- Map.Entry<byte[], byte[]> nonCoveredRange = getNonCoveredRange(tableId, key);
- if (nonCoveredRange != null) {
- partitionKey = nonCoveredRange.getValue();
- continue;
- }
-
- if (deadlineTracker.timedOut()) {
- Status statusTimedOut = Status.TimedOut("Took too long getting the list of tablets, " +
- deadlineTracker);
- return Deferred.fromError(new NonRecoverableException(statusTimedOut));
- }
-
- // If the partition key location isn't cached, and the request hasn't timed out,
- // then kick off a new tablet location lookup and try again when it completes.
- // When lookup completes, the tablet (or non-covered range) for the next
- // partition key will be located and added to the client's cache.
- final byte[] lookupKey = partitionKey;
- return locateTablet(table, key).addCallbackDeferring(
- new Callback<Deferred<List<LocatedTablet>>, GetTableLocationsResponsePB>() {
- @Override
- public Deferred<List<LocatedTablet>> call(GetTableLocationsResponsePB resp) {
- return loopLocateTable(table, lookupKey, endPartitionKey, ret, deadlineTracker);
- }
- @Override
- public String toString() {
- return "LoopLocateTableCB";
- }
- });
- }
-
- return Deferred.fromResult(ret);
- }
-
- /**
- * Get all or some tablets for a given table. This may query the master multiple times if there
- * are a lot of tablets.
- * @param table the table to locate tablets from
- * @param startPartitionKey where to start in the table, pass null to start at the beginning
- * @param endPartitionKey where to stop in the table, pass null to get all the tablets until the
- * end of the table
- * @param deadline max time spent in milliseconds for the deferred result of this method to
- * get called back, if deadline is reached, the deferred result will get erred back
- * @return a deferred object that yields a list of the tablets in the table, which can be queried
- * for metadata about each tablet
- * @throws Exception MasterErrorException if the table doesn't exist
- */
- Deferred<List<LocatedTablet>> locateTable(final KuduTable table,
- final byte[] startPartitionKey,
- final byte[] endPartitionKey,
- long deadline) {
- final List<LocatedTablet> ret = Lists.newArrayList();
- final DeadlineTracker deadlineTracker = new DeadlineTracker();
- deadlineTracker.setDeadline(deadline);
- return loopLocateTable(table, startPartitionKey, endPartitionKey, ret, deadlineTracker);
- }
-
- /**
- * We're handling a tablet server that's telling us it doesn't have the tablet we're asking for.
- * We're in the context of decode() meaning we need to either callback or retry later.
- */
- <R> void handleTabletNotFound(final KuduRpc<R> rpc, KuduException ex, TabletClient server) {
- invalidateTabletCache(rpc.getTablet(), server);
- handleRetryableError(rpc, ex);
- }
-
- /**
- * A tablet server is letting us know that it isn't the specified tablet's leader in response
- * a RPC, so we need to demote it and retry.
- */
- <R> void handleNotLeader(final KuduRpc<R> rpc, KuduException ex, TabletClient server) {
- rpc.getTablet().demoteLeader(server);
- handleRetryableError(rpc, ex);
- }
-
- <R> void handleRetryableError(final KuduRpc<R> rpc, KuduException ex) {
- // TODO we don't always need to sleep, maybe another replica can serve this RPC.
- delayedSendRpcToTablet(rpc, ex);
- }
-
- private <R> void delayedSendRpcToTablet(final KuduRpc<R> rpc, KuduException ex) {
- // Here we simply retry the RPC later. We might be doing this along with a lot of other RPCs
- // in parallel. Asynchbase does some hacking with a "probe" RPC while putting the other ones
- // on hold but we won't be doing this for the moment. Regions in HBase can move a lot,
- // we're not expecting this in Kudu.
- final class RetryTimer implements TimerTask {
- public void run(final Timeout timeout) {
- sendRpcToTablet(rpc);
- }
- }
- long sleepTime = getSleepTimeForRpc(rpc);
- if (cannotRetryRequest(rpc) || rpc.deadlineTracker.wouldSleepingTimeout(sleepTime)) {
- tooManyAttemptsOrTimeout(rpc, ex);
- // Don't let it retry.
- return;
- }
- newTimeout(new RetryTimer(), sleepTime);
- }
-
- /**
- * Remove the tablet server from the RemoteTablet's locations. Right now nothing is removing
- * the tablet itself from the caches.
- */
- private void invalidateTabletCache(RemoteTablet tablet, TabletClient server) {
- LOG.info("Removing server " + server.getUuid() + " from this tablet's cache " +
- tablet.getTabletIdAsString());
- tablet.removeTabletClient(server);
- }
-
- /** Callback executed when a master lookup completes. */
- private final class MasterLookupCB implements Callback<Object,
- Master.GetTableLocationsResponsePB> {
- final KuduTable table;
- private final byte[] partitionKey;
- MasterLookupCB(KuduTable table, byte[] partitionKey) {
- this.table = table;
- this.partitionKey = partitionKey;
- }
- public Object call(final GetTableLocationsResponsePB response) {
- if (response.hasError()) {
- if (response.getError().getCode() == Master.MasterErrorPB.Code.TABLET_NOT_RUNNING) {
- // Keep a note that the table exists but at least one tablet is not yet running.
- LOG.debug("Table {} has a non-running tablet", table.getName());
- tablesNotServed.add(table.getTableId());
- } else {
- Status status = Status.fromMasterErrorPB(response.getError());
- return new NonRecoverableException(status);
- }
- } else {
- try {
- discoverTablets(table, response.getTabletLocationsList());
- } catch (NonRecoverableException e) {
- return e;
- }
- if (partitionKey != null) {
- discoverNonCoveredRangePartitions(table.getTableId(), partitionKey,
- response.getTabletLocationsList());
- }
- }
- return null;
- }
- public String toString() {
- return "get tablet locations from the master for table " + table.getName();
- }
- }
-
- boolean acquireMasterLookupPermit() {
- try {
- // With such a low timeout, the JVM may chose to spin-wait instead of
- // de-scheduling the thread (and causing context switches and whatnot).
- return masterLookups.tryAcquire(5, MILLISECONDS);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt(); // Make this someone else's problem.
- return false;
- }
- }
-
- /**
- * Releases a master lookup permit that was acquired.
- * @see #acquireMasterLookupPermit
- */
- void releaseMasterLookupPermit() {
- masterLookups.release();
- }
-
- @VisibleForTesting
- void discoverTablets(KuduTable table, List<Master.TabletLocationsPB> locations)
- throws NonRecoverableException {
- String tableId = table.getTableId();
- String tableName = table.getName();
-
- // Doing a get first instead of putIfAbsent to avoid creating unnecessary CSLMs because in
- // the most common case the table should already be present
- ConcurrentSkipListMap<byte[], RemoteTablet> tablets = tabletsCache.get(tableId);
- if (tablets == null) {
- tablets = new ConcurrentSkipListMap<>(Bytes.MEMCMP);
- ConcurrentSkipListMap<byte[], RemoteTablet> oldTablets =
- tabletsCache.putIfAbsent(tableId, tablets);
- if (oldTablets != null) {
- tablets = oldTablets;
- }
- }
-
- for (Master.TabletLocationsPB tabletPb : locations) {
- // Early creating the tablet so that it parses out the pb
- RemoteTablet rt = createTabletFromPb(tableId, tabletPb);
- Slice tabletId = rt.tabletId;
-
- // If we already know about this one, just refresh the locations
- RemoteTablet currentTablet = tablet2client.get(tabletId);
- if (currentTablet != null) {
- currentTablet.refreshTabletClients(tabletPb);
- continue;
- }
-
- // Putting it here first doesn't make it visible because tabletsCache is always looked up
- // first.
- RemoteTablet oldRt = tablet2client.putIfAbsent(tabletId, rt);
- if (oldRt != null) {
- // someone beat us to it
- continue;
- }
- LOG.info("Discovered tablet {} for table '{}' with partition {}",
- tabletId.toString(Charset.defaultCharset()), tableName, rt.getPartition());
- rt.refreshTabletClients(tabletPb);
- // This is making this tablet available
- // Even if two clients were racing in this method they are putting the same RemoteTablet
- // with the same start key in the CSLM in the end
- tablets.put(rt.getPartition().getPartitionKeyStart(), rt);
- }
- }
-
- private void discoverNonCoveredRangePartitions(String tableId,
- byte[] partitionKey,
- List<Master.TabletLocationsPB> locations) {
- NonCoveredRangeCache nonCoveredRanges = nonCoveredRangeCaches.get(tableId);
- if (nonCoveredRanges == null) {
- nonCoveredRanges = new NonCoveredRangeCache();
- NonCoveredRangeCache oldCache = nonCoveredRangeCaches.putIfAbsent(tableId, nonCoveredRanges);
- if (oldCache != null) {
- nonCoveredRanges = oldCache;
- }
- }
-
- // If there are no locations, then the table has no tablets. This is
- // guaranteed because we never set an upper bound on the GetTableLocations
- // request, and the master will always return the tablet *before* the start
- // of the request, if the start key falls in a non-covered range (see the
- // comment on GetTableLocationsResponsePB in master.proto).
- if (locations.isEmpty()) {
- nonCoveredRanges.addNonCoveredRange(EMPTY_ARRAY, EMPTY_ARRAY);
- return;
- }
-
- // If the first tablet occurs after the requested partition key,
- // then there is an initial non-covered range.
- byte[] firstStartKey = locations.get(0).getPartition().getPartitionKeyStart().toByteArray();
- if (Bytes.memcmp(partitionKey, firstStartKey) < 0) {
- nonCoveredRanges.addNonCoveredRange(EMPTY_ARRAY, firstStartKey);
- }
-
- byte[] previousEndKey = null;
- for (Master.TabletLocationsPB location : locations) {
- byte[] startKey = location.getPartition().getPartitionKeyStart().toByteArray();
-
- // Check if there is a non-covered range between this tablet and the previous.
- if (previousEndKey != null && Bytes.memcmp(previousEndKey, startKey) < 0) {
- nonCoveredRanges.addNonCoveredRange(previousEndKey, startKey);
- }
- previousEndKey = location.getPartition().getPartitionKeyEnd().toByteArray();
- }
-
- if (previousEndKey.length > 0 && Bytes.memcmp(previousEndKey, partitionKey) <= 0) {
- // This happens if the partition key falls in a non-covered range that
- // is unbounded (to the right).
- nonCoveredRanges.addNonCoveredRange(previousEndKey, EMPTY_ARRAY);
- }
- }
-
- RemoteTablet createTabletFromPb(String tableId, Master.TabletLocationsPB tabletPb) {
- Partition partition = ProtobufHelper.pbToPartition(tabletPb.getPartition());
- Slice tabletId = new Slice(tabletPb.getTabletId().toByteArray());
- return new RemoteTablet(tableId, tabletId, partition);
- }
-
- /**
- * Gives the tablet's ID for the table ID and partition key.
- * In the future there will be multiple tablets and this method will find the right one.
- * @param tableId table to find the tablet for
- * @return a tablet ID as a slice or null if not found
- */
- RemoteTablet getTablet(String tableId, byte[] partitionKey) {
- ConcurrentSkipListMap<byte[], RemoteTablet> tablets = tabletsCache.get(tableId);
-
- if (tablets == null) {
- return null;
- }
-
- // We currently only have one master tablet.
- if (isMasterTable(tableId)) {
- if (tablets.firstEntry() == null) {
- return null;
- }
- return tablets.firstEntry().getValue();
- }
-
- Map.Entry<byte[], RemoteTablet> tabletPair = tablets.floorEntry(partitionKey);
-
- if (tabletPair == null) {
- return null;
- }
-
- Partition partition = tabletPair.getValue().getPartition();
-
- // If the partition is not the end partition, but it doesn't include the key
- // we are looking for, then we have not yet found the correct tablet.
- if (!partition.isEndPartition()
- && Bytes.memcmp(partitionKey, partition.getPartitionKeyEnd()) >= 0) {
- return null;
- }
-
- return tabletPair.getValue();
- }
-
- /**
- * Returns a deferred containing the located tablet which covers the partition key in the table.
- * @param table the table
- * @param partitionKey the partition key of the tablet to look up in the table
- * @param deadline deadline in milliseconds for this lookup to finish
- * @return a deferred containing the located tablet
- */
- Deferred<LocatedTablet> getTabletLocation(final KuduTable table,
- final byte[] partitionKey,
- long deadline) {
- // Locate the tablets at the partition key by locating all tablets between
- // the partition key (inclusive), and the incremented partition key (exclusive).
-
- Deferred<List<LocatedTablet>> locatedTablets;
- if (partitionKey.length == 0) {
- locatedTablets = locateTable(table, null, new byte[] { 0x00 }, deadline);
- } else {
- locatedTablets = locateTable(table, partitionKey,
- Arrays.copyOf(partitionKey, partitionKey.length + 1), deadline);
- }
-
- // Then pick out the single tablet result from the list.
- return locatedTablets.addCallbackDeferring(
- new Callback<Deferred<LocatedTablet>, List<LocatedTablet>>() {
- @Override
- public Deferred<LocatedTablet> call(List<LocatedTablet> tablets) {
- Preconditions.checkArgument(tablets.size() <= 1,
- "found more than one tablet for a single partition key");
- if (tablets.size() == 0) {
- Map.Entry<byte[], byte[]> nonCoveredRange =
- nonCoveredRangeCaches.get(table.getTableId()).getNonCoveredRange(partitionKey);
- return Deferred.fromError(new NonCoveredRangeException(nonCoveredRange.getKey(),
- nonCoveredRange.getValue()));
- }
- return Deferred.fromResult(tablets.get(0));
- }
- });
- }
-
- /**
- * Returns the non-covered range partition containing the {@code partitionKey} in
- * the table, or null if there is no known non-covering range for the partition key.
- * @param tableId of the table
- * @param partitionKey to lookup
- * @return the non-covering partition range, or {@code null}
- */
- Map.Entry<byte[], byte[]> getNonCoveredRange(String tableId, byte[] partitionKey) {
- if (isMasterTable(tableId)) {
- throw new IllegalArgumentException("No non-covering range partitions for the master");
- }
- NonCoveredRangeCache nonCoveredRangeCache = nonCoveredRangeCaches.get(tableId);
- if (nonCoveredRangeCache == null) return null;
-
- return nonCoveredRangeCache.getNonCoveredRange(partitionKey);
- }
-
- /**
- * Retrieve the master registration (see {@link GetMasterRegistrationResponse}
- * for a replica.
- * @param masterClient An initialized client for the master replica.
- * @return A Deferred object for the master replica's current registration.
- */
- Deferred<GetMasterRegistrationResponse> getMasterRegistration(TabletClient masterClient) {
- GetMasterRegistrationRequest rpc = new GetMasterRegistrationRequest(masterTable);
- rpc.setTimeoutMillis(defaultAdminOperationTimeoutMs);
- Deferred<GetMasterRegistrationResponse> d = rpc.getDeferred();
- rpc.attempt++;
- masterClient.sendRpc(rpc);
- return d;
- }
-
- /**
- * If a live client already exists for the specified master server, returns that client;
- * otherwise, creates a new client for the specified master server.
- * @param masterHostPort The RPC host and port for the master server.
- * @return A live and initialized client for the specified master server.
- */
- TabletClient newMasterClient(HostAndPort masterHostPort) {
- String ip = getIP(masterHostPort.getHostText());
- if (ip == null) {
- return null;
- }
- // We should pass a UUID here but we have a chicken and egg problem, we first need to
- // communicate with the masters to find out about them, and that's what we're trying to do.
- // The UUID is used for logging, so instead we're passing the "master table name" followed by
- // host and port which is enough to identify the node we're connecting to.
- return newClient(MASTER_TABLE_NAME_PLACEHOLDER + " - " + masterHostPort.toString(),
- ip, masterHostPort.getPort());
- }
-
- TabletClient newClient(String uuid, final String host, final int port) {
- final String hostport = host + ':' + port;
- TabletClient client;
- SocketChannel chan;
- synchronized (ip2client) {
- client = ip2client.get(hostport);
- if (client != null && client.isAlive()) {
- return client;
- }
- final TabletClientPipeline pipeline = new TabletClientPipeline();
- client = pipeline.init(uuid, host, port);
- chan = channelFactory.newChannel(pipeline);
- ip2client.put(hostport, client); // This is guaranteed to return null.
-
- // The client2tables map is assumed to contain `client` after it is published in ip2client.
- this.client2tablets.put(client, new ArrayList<RemoteTablet>());
- }
- final SocketChannelConfig config = chan.getConfig();
- config.setConnectTimeoutMillis(5000);
- config.setTcpNoDelay(true);
- // Unfortunately there is no way to override the keep-alive timeout in
- // Java since the JRE doesn't expose any way to call setsockopt() with
- // TCP_KEEPIDLE. And of course the default timeout is >2h. Sigh.
- config.setKeepAlive(true);
- chan.connect(new InetSocketAddress(host, port)); // Won't block.
- return client;
- }
-
- /**
- * Invokes {@link #shutdown()} and waits for the configured admin timeout. This method returns
- * void, so consider invoking shutdown directly if there's a need to handle dangling RPCs.
- * @throws Exception if an error happens while closing the connections
- */
- @Override
- public void close() throws Exception {
- shutdown().join(defaultAdminOperationTimeoutMs);
- }
-
- /**
- * Performs a graceful shutdown of this instance.
- * <p>
- * <ul>
- * <li>{@link AsyncKuduSession#flush Flushes} all buffered edits.</li>
- * <li>Cancels all the other requests.</li>
- * <li>Terminates all connections.</li>
- * <li>Releases all other resources.</li>
- * </ul>
- * <strong>Not calling this method before losing the last reference to this
- * instance may result in data loss and other unwanted side effects</strong>
- * @return A {@link Deferred}, whose callback chain will be invoked once all
- * of the above have been done. If this callback chain doesn't fail, then
- * the clean shutdown will be successful, and all the data will be safe on
- * the Kudu side. In case of a failure (the "errback" is invoked) you will have
- * to open a new AsyncKuduClient if you want to retry those operations.
- * The Deferred doesn't actually hold any content.
- */
- public Deferred<ArrayList<Void>> shutdown() {
- checkIsClosed();
- closed = true;
- // This is part of step 3. We need to execute this in its own thread
- // because Netty gets stuck in an infinite loop if you try to shut it
- // down from within a thread of its own thread pool. They don't want
- // to fix this so as a workaround we always shut Netty's thread pool
- // down from another thread.
- final class ShutdownThread extends Thread {
- ShutdownThread() {
- super("AsyncKuduClient@" + AsyncKuduClient.super.hashCode() + " shutdown");
- }
- public void run() {
- // This terminates the Executor.
- channelFactory.releaseExternalResources();
- }
- }
-
- // 3. Release all other resources.
- final class ReleaseResourcesCB implements Callback<ArrayList<Void>, ArrayList<Void>> {
- public ArrayList<Void> call(final ArrayList<Void> arg) {
- LOG.debug("Releasing all remaining resources");
- timer.stop();
- new ShutdownThread().start();
- return arg;
- }
- public String toString() {
- return "release resources callback";
- }
- }
-
- // 2. Terminate all connections.
- final class DisconnectCB implements Callback<Deferred<ArrayList<Void>>,
- ArrayList<List<OperationResponse>>> {
- public Deferred<ArrayList<Void>> call(ArrayList<List<OperationResponse>> ignoredResponses) {
- return disconnectEverything().addCallback(new ReleaseResourcesCB());
- }
- public String toString() {
- return "disconnect callback";
- }
- }
-
- // 1. Flush everything.
- // Notice that we do not handle the errback, if there's an exception it will come straight out.
- return closeAllSessions().addCallbackDeferring(new DisconnectCB());
- }
-
- private void checkIsClosed() {
- if (closed) {
- throw new IllegalStateException("Cannot proceed, the client has already been closed");
- }
- }
-
- private Deferred<ArrayList<List<OperationResponse>>> closeAllSessions() {
- // We create a copy because AsyncKuduSession.close will call removeSession which would get us a
- // concurrent modification during the iteration.
- Set<AsyncKuduSession> copyOfSessions;
- synchronized (sessions) {
- copyOfSessions = new HashSet<AsyncKuduSession>(sessions);
- }
- if (sessions.isEmpty()) {
- return Deferred.fromResult(null);
- }
- // Guaranteed that we'll have at least one session to close.
- List<Deferred<List<OperationResponse>>> deferreds = new ArrayList<>(copyOfSessions.size());
- for (AsyncKuduSession session : copyOfSessions ) {
- deferreds.add(session.close());
- }
-
- return Deferred.group(deferreds);
- }
-
- /**
- * Closes every socket, which will also cancel all the RPCs in flight.
- */
- private Deferred<ArrayList<Void>> disconnectEverything() {
- ArrayList<Deferred<Void>> deferreds =
- new ArrayList<Deferred<Void>>(2);
- HashMap<String, TabletClient> ip2client_copy;
- synchronized (ip2client) {
- // Make a local copy so we can shutdown every Tablet Server clients
- // without hold the lock while we iterate over the data structure.
- ip2client_copy = new HashMap<String, TabletClient>(ip2client);
- }
-
- for (TabletClient ts : ip2client_copy.values()) {
- deferreds.add(ts.shutdown());
- }
- final int size = deferreds.size();
- return Deferred.group(deferreds).addCallback(
- new Callback<ArrayList<Void>, ArrayList<Void>>() {
- public ArrayList<Void> call(final ArrayList<Void> arg) {
- // Normally, now that we've shutdown() every client, all our caches should
- // be empty since each shutdown() generates a DISCONNECTED event, which
- // causes TabletClientPipeline to call removeClientFromIpCache().
- HashMap<String, TabletClient> logme = null;
- synchronized (ip2client) {
- if (!ip2client.isEmpty()) {
- logme = new HashMap<String, TabletClient>(ip2client);
- }
- }
- if (logme != null) {
- // Putting this logging statement inside the synchronized block
- // can lead to a deadlock, since HashMap.toString() is going to
- // call TabletClient.toString() on each entry, and this locks the
- // client briefly. Other parts of the code lock clients first and
- // the ip2client HashMap second, so this can easily deadlock.
- LOG.error("Some clients are left in the client cache and haven't"
- + " been cleaned up: " + logme);
- }
- return arg;
- }
-
- public String toString() {
- return "wait " + size + " TabletClient.shutdown()";
- }
- });
- }
-
- /**
- * Blocking call.
- * Performs a slow search of the IP used by the given client.
- * <p>
- * This is needed when we're trying to find the IP of the client before its
- * channel has successfully connected, because Netty's API offers no way of
- * retrieving the IP of the remote peer until we're connected to it.
- * @param client The client we want the IP of.
- * @return The IP of the client, or {@code null} if we couldn't find it.
- */
- private InetSocketAddress slowSearchClientIP(final TabletClient client) {
- String hostport = null;
- synchronized (ip2client) {
- for (final Map.Entry<String, TabletClient> e : ip2client.entrySet()) {
- if (e.getValue() == client) {
- hostport = e.getKey();
- break;
- }
- }
- }
-
- if (hostport == null) {
- HashMap<String, TabletClient> copy;
- synchronized (ip2client) {
- copy = new HashMap<String, TabletClient>(ip2client);
- }
- LOG.error("WTF? Should never happen! Couldn't find " + client
- + " in " + copy);
- return null;
- }
- final int colon = hostport.indexOf(':', 1);
- if (colon < 1) {
- LOG.error("WTF? Should never happen! No `:' found in " + hostport);
- return null;
- }
- final String host = getIP(hostport.substring(0, colon));
- if (host == null) {
- // getIP will print the reason why, there's nothing else we can do.
- return null;
- }
-
- int port;
- try {
- port = parsePortNumber(hostport.substring(colon + 1,
- hostport.length()));
- } catch (NumberFormatException e) {
- LOG.error("WTF? Should never happen! Bad port in " + hostport, e);
- return null;
- }
- return new InetSocketAddress(host, port);
- }
-
- /**
- * Removes the given client from the `ip2client` cache.
- * @param client The client for which we must clear the ip cache
- * @param remote The address of the remote peer, if known, or null
- */
- private void removeClientFromIpCache(final TabletClient client,
- final SocketAddress remote) {
-
- if (remote == null) {
- return; // Can't continue without knowing the remote address.
- }
-
- String hostport;
- if (remote instanceof InetSocketAddress) {
- final InetSocketAddress sock = (InetSocketAddress) remote;
- final InetAddress addr = sock.getAddress();
- if (addr == null) {
- LOG.error("WTF? Unresolved IP for " + remote
- + ". This shouldn't happen.");
- return;
- } else {
- hostport = addr.getHostAddress() + ':' + sock.getPort();
- }
- } else {
- LOG.error("WTF? Found a non-InetSocketAddress remote: " + remote
- + ". This shouldn't happen.");
- return;
- }
-
- TabletClient old;
- synchronized (ip2client) {
- old = ip2client.remove(hostport);
- }
- LOG.debug("Removed from IP cache: {" + hostport + "} -> {" + client + "}");
- if (old == null) {
- // Currently we're seeing this message when masters are disconnected and the hostport we got
- // above is different than the one the user passes (that we use to populate ip2client). At
- // worst this doubles the entries for masters, which has an insignificant impact.
- // TODO When fixed, make this a WARN again.
- LOG.trace("When expiring " + client + " from the client cache (host:port="
- + hostport + "), it was found that there was no entry"
- + " corresponding to " + remote + ". This shouldn't happen.");
- }
- }
-
- /**
- * Call this method after encountering an error connecting to a tablet server so that we stop
- * considering it a leader for the tablets it serves.
- * @param client tablet server to use for demotion
- */
- void demoteAsLeaderForAllTablets(final TabletClient client) {
- ArrayList<RemoteTablet> tablets = client2tablets.get(client);
- if (tablets != null) {
- // Make a copy so we don't need to synchronize on it while iterating.
- RemoteTablet[] tablets_copy;
- synchronized (tablets) {
- tablets_copy = tablets.toArray(new RemoteTablet[tablets.size()]);
- }
- for (final RemoteTablet remoteTablet : tablets_copy) {
- // It will be a no-op if it's not already a leader.
- remoteTablet.demoteLeader(client);
- }
- }
- }
-
- private boolean isMasterTable(String tableId) {
- // Checking that it's the same instance so there's absolutely no chance of confusing the master
- // 'table' for a user one.
- return MASTER_TABLE_NAME_PLACEHOLDER == tableId;
- }
-
- private final class TabletClientPipeline extends DefaultChannelPipeline {
-
- private final Logger log = LoggerFactory.getLogger(TabletClientPipeline.class);
- /**
- * Have we already disconnected?.
- * We use this to avoid doing the cleanup work for the same client more
- * than once, even if we get multiple events indicating that the client
- * is no longer connected to the TabletServer (e.g. DISCONNECTED, CLOSED).
- * No synchronization needed as this is always accessed from only one
- * thread at a time (equivalent to a non-shared state in a Netty handler).
- */
- private boolean disconnected = false;
-
- TabletClient init(String uuid, String host, int port) {
- final TabletClient client = new TabletClient(AsyncKuduClient.this, uuid, host, port);
- if (defaultSocketReadTimeoutMs > 0) {
- super.addLast("timeout-handler",
- new ReadTimeoutHandler(timer,
- defaultSocketReadTimeoutMs,
- TimeUnit.MILLISECONDS));
- }
- super.addLast("kudu-handler", client);
-
- return client;
- }
-
- @Override
- public void sendDownstream(final ChannelEvent event) {
- if (event instanceof ChannelStateEvent) {
- handleDisconnect((ChannelStateEvent) event);
- }
- super.sendDownstream(event);
- }
-
- @Override
- public void sendUpstream(final ChannelEvent event) {
- if (event instanceof ChannelStateEvent) {
- handleDisconnect((ChannelStateEvent) event);
- }
- super.sendUpstream(event);
- }
-
- private void handleDisconnect(final ChannelStateEvent state_event) {
- if (disconnected) {
- return;
- }
- switch (state_event.getState()) {
- case OPEN:
- if (state_event.getValue() == Boolean.FALSE) {
- break; // CLOSED
- }
- return;
- case CONNECTED:
- if (state_event.getValue() == null) {
- break; // DISCONNECTED
- }
- return;
- default:
- return; // Not an event we're interested in, ignore it.
- }
-
- disconnected = true; // So we don't clean up the same client twice.
- try {
- final TabletClient client = super.get(TabletClient.class);
- SocketAddress remote = super.getChannel().getRemoteAddress();
- // At this point Netty gives us no easy way to access the
- // SocketAddress of the peer we tried to connect to. This
- // kinda sucks but I couldn't find an easier way.
- if (remote == null) {
- remote = slowSearchClientIP(client);
- }
-
- synchronized (client) {
- removeClientFromIpCache(client, remote);
- }
- } catch (Exception e) {
- log.error("Uncaught exception when handling a disconnection of " + getChannel(), e);
- }
- }
-
- }
-
- /**
- * Gets a hostname or an IP address and returns the textual representation
- * of the IP address.
- * <p>
- * <strong>This method can block</strong> as there is no API for
- * asynchronous DNS resolution in the JDK.
- * @param host The hostname to resolve.
- * @return The IP address associated with the given hostname,
- * or {@code null} if the address couldn't be resolved.
- */
- private static String getIP(final String host) {
- final long start = System.nanoTime();
- try {
- final String ip = InetAddress.getByName(host).getHostAddress();
- final long latency = System.nanoTime() - start;
- if (latency > 500000/*ns*/ && LOG.isDebugEnabled()) {
- LOG.debug("Resolved IP of `" + host + "' to "
- + ip + " in " + latency + "ns");
- } else if (latency >= 3000000/*ns*/) {
- LOG.warn("Slow DNS lookup! Resolved IP of `" + host + "' to "
- + ip + " in " + latency + "ns");
- }
- return ip;
- } catch (UnknownHostException e) {
- LOG.error("Failed to resolve the IP of `" + host + "' in "
- + (System.nanoTime() - start) + "ns");
- return null;
- }
- }
-
- /**
- * Parses a TCP port number from a string.
- * @param portnum The string to parse.
- * @return A strictly positive, validated port number.
- * @throws NumberFormatException if the string couldn't be parsed as an
- * integer or if the value was outside of the range allowed for TCP ports.
- */
- private static int parsePortNumber(final String portnum)
- throws NumberFormatException {
- final int port = Integer.parseInt(portnum);
- if (port <= 0 || port > 65535) {
- throw new NumberFormatException(port == 0 ? "port is zero" :
- (port < 0 ? "port is negative: "
- : "port is too large: ") + port);
- }
- return port;
- }
-
- void newTimeout(final TimerTask task, final long timeout_ms) {
- try {
- timer.newTimeout(task, timeout_ms, MILLISECONDS);
- } catch (IllegalStateException e) {
- // This can happen if the timer fires just before shutdown()
- // is called from another thread, and due to how threads get
- // scheduled we tried to call newTimeout() after timer.stop().
- LOG.warn("Failed to schedule timer."
- + " Ignore this if we're shutting down.", e);
- }
- }
-
- /**
- * This class encapsulates the information regarding a tablet and its locations.
- *
- * Leader failover mechanism:
- * When we get a complete peer list from the master, we place the leader in the first
- * position of the tabletServers array. When we detect that it isn't the leader anymore (in
- * TabletClient), we demote it and set the next TS in the array as the leader. When the RPC
- * gets retried, it will use that TS since we always pick the leader.
- *
- * If that TS turns out to not be the leader, we will demote it and promote the next one, retry.
- * When we hit the end of the list, we set the leaderIndex to NO_LEADER_INDEX which forces us
- * to fetch the tablet locations from the master. We'll repeat this whole process until a RPC
- * succeeds.
- *
- * Subtleties:
- * We don't keep track of a TS after it disconnects (via removeTabletClient), so if we
- * haven't contacted one for 10 seconds (socket timeout), it will be removed from the list of
- * tabletServers. This means that if the leader fails, we only have one other TS to "promote"
- * or maybe none at all. This is partly why we then set leaderIndex to NO_LEADER_INDEX.
- *
- * The effect of treating a TS as the new leader means that the Scanner will also try to hit it
- * with requests. It's currently unclear if that's a good or a bad thing.
- *
- * Unlike the C++ client, we don't short-circuit the call to the master if it isn't available.
- * This means that after trying all the peers to find the leader, we might get stuck waiting on
- * a reachable master.
- */
- public class RemoteTablet implements Comparable<RemoteTablet> {
-
- private static final int NO_LEADER_INDEX = -1;
- private final String tableId;
- private final Slice tabletId;
- @GuardedBy("tabletServers")
- private final ArrayList<TabletClient> tabletServers = new ArrayList<>();
- private final AtomicReference<List<LocatedTablet.Replica>> replicas =
- new AtomicReference(ImmutableList.of());
- private final Partition partition;
- private int leaderIndex = NO_LEADER_INDEX;
-
- RemoteTablet(String tableId, Slice tabletId, Partition partition) {
- this.tabletId = tabletId;
- this.tableId = tableId;
- this.partition = partition;
- }
-
- void refreshTabletClients(Master.TabletLocationsPB tabletLocations) throws NonRecoverableException {
-
- synchronized (tabletServers) { // TODO not a fat lock with IP resolving in it
- tabletServers.clear();
- leaderIndex = NO_LEADER_INDEX;
- List<UnknownHostException> lookupExceptions =
- new ArrayList<>(tabletLocations.getReplicasCount());
- for (Master.TabletLocationsPB.ReplicaPB replica : tabletLocations.getReplicasList()) {
-
- List<Common.HostPortPB> addresses = replica.getTsInfo().getRpcAddressesList();
- if (addresses.isEmpty()) {
- LOG.warn("Tablet server for tablet " + getTabletIdAsString() + " doesn't have any " +
- "address");
- continue;
- }
- byte[] buf = Bytes.get(replica.getTsInfo().getPermanentUuid());
- String uuid = Bytes.getString(buf);
- // from meta_cache.cc
- // TODO: if the TS advertises multiple host/ports, pick the right one
- // based on some kind of policy. For now just use the first always.
- try {
- addTabletClient(uuid, addresses.get(0).getHost(), addresses.get(0).getPort(),
- replica.getRole().equals(Metadata.RaftPeerPB.Role.LEADER));
- } catch (UnknownHostException ex) {
- lookupExceptions.add(ex);
- }
- }
-
- if (leaderIndex == NO_LEADER_INDEX) {
- LOG.warn("No leader provided for tablet {}", getTabletIdAsString());
- }
-
- // If we found a tablet that doesn't contain a single location that we can resolve, there's
- // no point in retrying.
- if (!lookupExceptions.isEmpty() &&
- lookupExceptions.size() == tabletLocations.getReplicasCount()) {
- Status statusIOE = Status.IOError("Couldn't find any valid locations, exceptions: " +
- lookupExceptions);
- throw new NonRecoverableException(statusIOE);
- }
-
- }
-
- ImmutableList.Builder<LocatedTablet.Replica> replicasBuilder = new ImmutableList.Builder<>();
- for (Master.TabletLocationsPB.ReplicaPB replica : tabletLocations.getReplicasList()) {
- replicasBuilder.add(new LocatedTablet.Replica(replica));
- }
- replicas.set(replicasBuilder.build());
- }
-
- // Must be called with tabl
<TRUNCATED>
[27/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java
new file mode 100644
index 0000000..0e68e93
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java
@@ -0,0 +1,1094 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import com.google.common.io.BaseEncoding;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.util.Slice;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.util.CharsetUtil;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Comparator;
+
+/**
+ * Helper functions to manipulate byte arrays.
+ */
+@InterfaceAudience.Private
+public final class Bytes {
+
+ // Two's complement reference: 2^n .
+ // In this case, 2^64 (so as to emulate a unsigned long)
+ // from http://stackoverflow.com/questions/10886962/interpret-a-negative-number-as-unsigned-with-
+ // biginteger-java
+ private static final BigInteger TWO_COMPL_REF = BigInteger.ONE.shiftLeft(64);
+
+ private Bytes() { // Can't instantiate.
+ }
+
+ // -------------------------------- //
+ // Byte array conversion utilities. //
+ // -------------------------------- //
+
+ /**
+ * Reads a boolean from the beginning of the given array.
+ * @param b The array to read from.
+ * @return A boolean
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static boolean getBoolean(final byte[] b) {
+ byte v = getByte(b, 0);
+ return v == 1;
+ }
+
+ /**
+ * Reads a boolean from an offset in the given array.
+ * @param b The array to read from.
+ * @param offset The offset into the array.
+ * @return A boolean
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static boolean getBoolean(final byte[] b, final int offset) {
+ byte v = getByte(b, offset);
+ return v == 1;
+ }
+
+ /**
+ * Reads a byte from the beginning of the given array.
+ * @param b The array to read from.
+ * @return A byte
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static byte getByte(final byte[] b) {
+ return getByte(b, 0);
+ }
+
+ /**
+ * Reads a byte from an offset in the given array.
+ * @param b The array to read from.
+ * @return A byte
+ * @return
+ */
+ public static byte getByte(final byte[] b, final int offset) {
+ return b[offset];
+ }
+
+ /**
+ * Reads an unsigned byte from the beginning of the given array.
+ * @param b The array to read from.
+ * @return A positive byte
+ */
+ public static short getUnsignedByte(final byte[] b) {
+ return getUnsignedByte(b, 0);
+ }
+
+ /**
+ * Reads an unsigned byte from an offset in the given array.
+ * @param b The array to read from.
+ * @return A positive byte
+ */
+ public static short getUnsignedByte(final byte[] b, final int offset) {
+ return (short) (b[offset] & 0x00FF);
+ }
+
+ /**
+ * Writes an unsigned byte at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n An unsigned byte.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedByte(final byte[] b, final short n) {
+ setUnsignedByte(b, n, 0);
+ }
+
+ /**
+ * Writes an unsigned byte at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n An unsigned byte.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedByte(final byte[] b, final short n,
+ final int offset) {
+ b[offset] = (byte) n;
+ }
+
+ /**
+ * Creates a new byte array containing an unsigned byte.
+ * @param n An unsigned byte.
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromUnsignedByte(final short n) {
+ final byte[] b = new byte[1];
+ setUnsignedByte(b, n);
+ return b;
+ }
+
+ /**
+ * Reads a little-endian 2-byte short from the beginning of the given array.
+ * @param b The array to read from.
+ * @return A short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static short getShort(final byte[] b) {
+ return getShort(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 2-byte short from an offset in the given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return A short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static short getShort(final byte[] b, final int offset) {
+ return (short) (b[offset] & 0xFF | b[offset + 1] << 8 );
+ }
+
+ /**
+ * Reads a little-endian 2-byte unsigned short from the beginning of the
+ * given array.
+ * @param b The array to read from.
+ * @return A positive short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static int getUnsignedShort(final byte[] b) {
+ return getUnsignedShort(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 2-byte unsigned short from an offset in the
+ * given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return A positive short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static int getUnsignedShort(final byte[] b, final int offset) {
+ return getShort(b, offset) & 0x0000FFFF;
+ }
+
+ /**
+ * Writes a little-endian 2-byte short at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n A short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setShort(final byte[] b, final short n) {
+ setShort(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 2-byte short at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n A short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setShort(final byte[] b, final short n,
+ final int offset) {
+ b[offset + 0] = (byte) (n >>> 0);
+ b[offset + 1] = (byte) (n >>> 8);
+ }
+
+ /**
+ * Writes a little-endian 2-byte unsigned short at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n An unsigned short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedShort(final byte[] b, final int n) {
+ setUnsignedShort(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 2-byte unsigned short at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n An unsigned short integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedShort(final byte[] b, final int n,
+ final int offset) {
+ b[offset + 0] = (byte) (n >>> 0);
+ b[offset + 1] = (byte) (n >>> 8);
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 2-byte short integer.
+ * @param n A short integer.
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromShort(final short n) {
+ final byte[] b = new byte[2];
+ setShort(b, n);
+ return b;
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 2-byte unsigned short integer.
+ * @param n An unsigned short integer.
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromUnsignedShort(final int n) {
+ final byte[] b = new byte[2];
+ setUnsignedShort(b, n);
+ return b;
+ }
+
+ /**
+ * Reads a little-endian 4-byte integer from the beginning of the given array.
+ * @param b The array to read from.
+ * @return An integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static int getInt(final byte[] b) {
+ return getInt(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 4-byte integer from an offset in the given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return An integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static int getInt(final byte[] b, final int offset) {
+ return (b[offset + 0] & 0xFF) << 0
+ | (b[offset + 1] & 0xFF) << 8
+ | (b[offset + 2] & 0xFF) << 16
+ | (b[offset + 3] & 0xFF) << 24;
+ }
+
+ /**
+ * Reads a little-endian 4-byte unsigned integer from the beginning of the
+ * given array.
+ * @param b The array to read from.
+ * @return A positive integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static long getUnsignedInt(final byte[] b) {
+ return getUnsignedInt(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 4-byte unsigned integer from an offset in the
+ * given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return A positive integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static long getUnsignedInt(final byte[] b, final int offset) {
+ return getInt(b, offset) & 0x00000000FFFFFFFFL;
+ }
+
+ /**
+ * Writes a little-endian 4-byte int at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n An integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setInt(final byte[] b, final int n) {
+ setInt(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 4-byte int at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n An integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setInt(final byte[] b, final int n, final int offset) {
+ b[offset + 0] = (byte) (n >>> 0);
+ b[offset + 1] = (byte) (n >>> 8);
+ b[offset + 2] = (byte) (n >>> 16);
+ b[offset + 3] = (byte) (n >>> 24);
+ }
+
+ /**
+ * Writes a little-endian 4-byte unsigned int at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n An unsigned integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedInt(final byte[] b, final long n) {
+ setUnsignedInt(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 4-byte unsigned int at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n An unsigned integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedInt(final byte[] b, final long n, final int offset) {
+ b[offset + 0] = (byte) (n >>> 0);
+ b[offset + 1] = (byte) (n >>> 8);
+ b[offset + 2] = (byte) (n >>> 16);
+ b[offset + 3] = (byte) (n >>> 24);
+ }
+
+ public static void putVarInt32(final ByteBuffer b, final int v) {
+ int B = 128;
+ if (v < (1<<7)) {
+ b.put((byte)v);
+ } else if (v < (1<<14)) {
+ b.put((byte)(v | B));
+ b.put((byte)((v>>7) | B));
+ } else if (v < (1<<21)) {
+ b.put((byte)(v | B));
+ b.put((byte)((v>>7) | B));
+ b.put((byte)(v>>14));
+ } else if (v < (1<<28)) {
+ b.put((byte)(v | B));
+ b.put((byte)((v>>7) | B));
+ b.put((byte)((v>>14) | B));
+ b.put((byte)(v>>21));
+ } else {
+ b.put((byte)(v | B));
+ b.put((byte)((v>>7) | B));
+ b.put((byte)((v>>14) | B));
+ b.put((byte)((v>>21) | B));
+ b.put((byte)(v>>28));
+ }
+ }
+
+ /**
+ * Reads a 32-bit variable-length integer value as used in Protocol Buffers.
+ * @param buf The buffer to read from.
+ * @return The integer read.
+ */
+ static int readVarInt32(final ChannelBuffer buf) {
+ int result = buf.readByte();
+ if (result >= 0) {
+ return result;
+ }
+ result &= 0x7F;
+ result |= buf.readByte() << 7;
+ if (result >= 0) {
+ return result;
+ }
+ result &= 0x3FFF;
+ result |= buf.readByte() << 14;
+ if (result >= 0) {
+ return result;
+ }
+ result &= 0x1FFFFF;
+ result |= buf.readByte() << 21;
+ if (result >= 0) {
+ return result;
+ }
+ result &= 0x0FFFFFFF;
+ final byte b = buf.readByte();
+ result |= b << 28;
+ if (b >= 0) {
+ return result;
+ }
+ throw new IllegalArgumentException("Not a 32 bit varint: " + result
+ + " (5th byte: " + b + ")");
+ }
+
+ public static byte[] fromBoolean(final boolean n) {
+ final byte[] b = new byte[1];
+ b[0] = (byte) (n ? 1 : 0);
+ return b;
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 4-byte integer.
+ * @param n An integer.
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromInt(final int n) {
+ final byte[] b = new byte[4];
+ setInt(b, n);
+ return b;
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 4-byte unsigned integer.
+ * @param n An unsigned integer.
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromUnsignedInt(final long n) {
+ final byte[] b = new byte[4];
+ setUnsignedInt(b, n);
+ return b;
+ }
+
+ /**
+ * Reads a little-endian 8-byte unsigned long from the beginning of the given array.
+ * @param b The array to read from.
+ * @return A long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static BigInteger getUnsignedLong(final byte[] b) {
+ return getUnsignedLong(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 8-byte unsigned long from an offset in the given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return A long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static BigInteger getUnsignedLong(final byte[] b, final int offset) {
+ long l = getLong(b, offset);
+ BigInteger bi = new BigInteger(l+"");
+ if (bi.compareTo(BigInteger.ZERO) < 0) {
+ bi = bi.add(TWO_COMPL_REF);
+ }
+ return bi;
+ }
+
+ /**
+ * Reads a little-endian 8-byte long from the beginning of the given array.
+ * @param b The array to read from.
+ * @return A long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static long getLong(final byte[] b) {
+ return getLong(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 8-byte long from an offset in the given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return A long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static long getLong(final byte[] b, final int offset) {
+ return (b[offset + 0] & 0xFFL) << 0
+ | (b[offset + 1] & 0xFFL) << 8
+ | (b[offset + 2] & 0xFFL) << 16
+ | (b[offset + 3] & 0xFFL) << 24
+ | (b[offset + 4] & 0xFFL) << 32
+ | (b[offset + 5] & 0xFFL) << 40
+ | (b[offset + 6] & 0xFFL) << 48
+ | (b[offset + 7] & 0xFFL) << 56;
+ }
+
+ /**
+ * Writes a little-endian 8-byte long at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n A long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setLong(final byte[] b, final long n) {
+ setLong(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 8-byte long at an offset in the given array.
+ * @param b The array to write to.
+ * @param n A long integer.
+ * @param offset The offset in the array to start writing at.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setLong(final byte[] b, final long n, final int offset) {
+ b[offset + 0] = (byte) (n >>> 0);
+ b[offset + 1] = (byte) (n >>> 8);
+ b[offset + 2] = (byte) (n >>> 16);
+ b[offset + 3] = (byte) (n >>> 24);
+ b[offset + 4] = (byte) (n >>> 32);
+ b[offset + 5] = (byte) (n >>> 40);
+ b[offset + 6] = (byte) (n >>> 48);
+ b[offset + 7] = (byte) (n >>> 56);
+ }
+
+ /**
+ * Writes a little-endian 8-byte unsigned long at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n An unsigned long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedLong(final byte[] b, final BigInteger n) {
+ setUnsignedLong(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 8-byte unsigned long at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n An unsigned long integer.
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setUnsignedLong(final byte[] b, final BigInteger n, final int offset) {
+ setLong(b, n.longValue(), offset);
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 8-byte long integer.
+ * @param n A long integer.
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromLong(final long n) {
+ final byte[] b = new byte[8];
+ setLong(b, n);
+ return b;
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 8-byte unsigned long integer.
+ * @param n An unsigned long integer.
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromUnsignedLong(final BigInteger n) {
+ final byte[] b = new byte[8];
+ setUnsignedLong(b, n);
+ return b;
+ }
+
+ /**
+ * Reads a little-endian 4-byte float from the beginning of the given array.
+ * @param b The array to read from.
+ * @return a float
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static float getFloat(final byte[] b) {
+ return getFloat(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 4-byte float from an offset in the given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return a float
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static float getFloat(final byte[] b, final int offset) {
+ return Float.intBitsToFloat(getInt(b, offset));
+ }
+
+ /**
+ * Writes a little-endian 4-byte float at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n a float
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setFloat(final byte[] b, final float n) {
+ setFloat(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 4-byte float at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n a float
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setFloat(final byte[] b, final float n, final int offset) {
+ setInt(b, Float.floatToIntBits(n), offset);
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 4-byte float.
+ * @param n A float
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromFloat(float n) {
+ byte[] b = new byte[4];
+ setFloat(b, n);
+ return b;
+ }
+
+ /**
+ * Reads a little-endian 8-byte double from the beginning of the given array.
+ * @param b The array to read from.
+ * @return a double
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static double getDouble(final byte[] b) {
+ return getDouble(b, 0);
+ }
+
+ /**
+ * Reads a little-endian 8-byte double from an offset in the given array.
+ * @param b The array to read from.
+ * @param offset The offset in the array to start reading from.
+ * @return a double
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static double getDouble(final byte[] b, final int offset) {
+ return Double.longBitsToDouble(getLong(b, offset));
+ }
+
+ /**
+ * Writes a little-endian 8-byte double at the beginning of the given array.
+ * @param b The array to write to.
+ * @param n a double
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setDouble(final byte[] b, final double n) {
+ setDouble(b, n, 0);
+ }
+
+ /**
+ * Writes a little-endian 8-byte double at an offset in the given array.
+ * @param b The array to write to.
+ * @param offset The offset in the array to start writing at.
+ * @param n a double
+ * @throws IndexOutOfBoundsException if the byte array is too small.
+ */
+ public static void setDouble(final byte[] b, final double n, final int offset) {
+ setLong(b, Double.doubleToLongBits(n), offset);
+ }
+
+ /**
+ * Creates a new byte array containing a little-endian 8-byte double.
+ * @param n A double
+ * @return A new byte array containing the given value.
+ */
+ public static byte[] fromDouble(double n) {
+ byte[] b = new byte[8];
+ setDouble(b, n);
+ return b;
+ }
+
+ /**
+ * Extracts the byte array from the given {@link ByteString} without copy.
+ * @param buf A buffer from which to extract the array. This buffer must be
+ * actually an instance of a {@code LiteralByteString}.
+ * @since 1.5
+ */
+ public static byte[] get(final ByteString buf) {
+ return ZeroCopyLiteralByteString.zeroCopyGetBytes(buf);
+ }
+
+ /** Transforms a string into an UTF-8 encoded byte array. */
+ public static byte[] UTF8(final String s) {
+ return s.getBytes(CharsetUtil.UTF_8);
+ }
+
+ /** Transforms a string into an ISO-8859-1 encoded byte array. */
+ public static byte[] ISO88591(final String s) {
+ return s.getBytes(CharsetUtil.ISO_8859_1);
+ }
+
+ // ---------------------------- //
+ // Pretty-printing byte arrays. //
+ // ---------------------------- //
+
+ private static final char[] HEX = {
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+ 'A', 'B', 'C', 'D', 'E', 'F'
+ };
+
+ /**
+ * Pretty-prints a byte array into a human-readable output buffer.
+ * @param outbuf The buffer where to write the output.
+ * @param array The (possibly {@code null}) array to pretty-print.
+ */
+ public static void pretty(final StringBuilder outbuf, final byte[] array) {
+ if (array == null) {
+ outbuf.append("null");
+ return;
+ }
+ int ascii = 0;
+ final int start_length = outbuf.length();
+ final int n = array.length;
+ outbuf.ensureCapacity(start_length + 1 + n + 1);
+ outbuf.append('"');
+ for (int i = 0; i < n; i++) {
+ final byte b = array[i];
+ if (' ' <= b && b <= '~') {
+ ascii++;
+ outbuf.append((char) b);
+ } else if (b == '\n') {
+ outbuf.append('\\').append('n');
+ } else if (b == '\t') {
+ outbuf.append('\\').append('t');
+ } else {
+ outbuf.append("\\x")
+ .append(HEX[(b >>> 4) & 0x0F])
+ .append(HEX[b & 0x0F]);
+ }
+ }
+ if (ascii < n / 2) {
+ outbuf.setLength(start_length);
+ outbuf.append(Arrays.toString(array));
+ } else {
+ outbuf.append('"');
+ }
+ }
+
+ /**
+ * Pretty-prints an array of byte arrays into a human-readable output buffer.
+ * @param outbuf The buffer where to write the output.
+ * @param arrays The (possibly {@code null}) array of arrays to pretty-print.
+ * @since 1.3
+ */
+ public static void pretty(final StringBuilder outbuf, final byte[][] arrays) {
+ if (arrays == null) {
+ outbuf.append("null");
+ return;
+ } else { // Do some right-sizing.
+ int size = 2;
+ for (int i = 0; i < arrays.length; i++) {
+ size += 2 + 2 + arrays[i].length;
+ }
+ outbuf.ensureCapacity(outbuf.length() + size);
+ }
+ outbuf.append('[');
+ for (int i = 0; i < arrays.length; i++) {
+ Bytes.pretty(outbuf, arrays[i]);
+ outbuf.append(", ");
+ }
+ outbuf.setLength(outbuf.length() - 2); // Remove the last ", "
+ outbuf.append(']');
+ }
+
+ /**
+ * Pretty-prints a byte array into a human-readable string.
+ * @param array The (possibly {@code null}) array to pretty-print.
+ * @return The array in a pretty-printed string.
+ */
+ public static String pretty(final byte[] array) {
+ if (array == null) {
+ return "null";
+ }
+ final StringBuilder buf = new StringBuilder(1 + array.length + 1);
+ pretty(buf, array);
+ return buf.toString();
+ }
+
+ /**
+ * Convert a byte array to a hex encoded string.
+ * @param bytes the bytes to encode
+ * @return the hex encoded bytes
+ */
+ public static String hex(byte[] bytes) {
+ StringBuilder sb = new StringBuilder(2 + bytes.length * 2);
+ sb.append('0');
+ sb.append('x');
+ sb.append(BaseEncoding.base16().encode(bytes));
+ return sb.toString();
+ }
+
+ // Ugly stuff
+ // ----------
+ // Background: when using ReplayingDecoder (which makes it easy to deal with
+ // unframed RPC responses), the ChannelBuffer we manipulate is in fact a
+ // ReplayingDecoderBuffer, a package-private class that Netty uses. This
+ // class, for some reason, throws UnsupportedOperationException on its
+ // array() method. This method is unfortunately the only way to easily dump
+ // the contents of a ChannelBuffer, which is useful for debugging or logging
+ // unexpected buffers. An issue (NETTY-346) has been filed to get access to
+ // the buffer, but the resolution was useless: instead of making the array()
+ // method work, a new internalBuffer() method was added on ReplayingDecoder,
+ // which would require that we keep a reference on the ReplayingDecoder all
+ // along in order to properly convert the buffer to a string.
+ // So we instead use ugly reflection to gain access to the underlying buffer
+ // while taking into account that the implementation of Netty has changed
+ // over time, so depending which version of Netty we're working with, we do
+ // a different hack. Yes this is horrible, but it's for the greater good as
+ // this is what allows us to debug unexpected buffers when deserializing RPCs
+ // and what's more important than being able to debug unexpected stuff?
+ private static final Class<?> ReplayingDecoderBuffer;
+ private static final Field RDB_buffer; // For Netty 3.5.0 and before.
+ private static final Method RDB_buf; // For Netty 3.5.1 and above.
+ static {
+ try {
+ ReplayingDecoderBuffer = Class.forName("org.jboss.netty.handler.codec."
+ + "replay.ReplayingDecoderBuffer");
+ Field field = null;
+ try {
+ field = ReplayingDecoderBuffer.getDeclaredField("buffer");
+ field.setAccessible(true);
+ } catch (NoSuchFieldException e) {
+ // Ignore. Field has been removed in Netty 3.5.1.
+ }
+ RDB_buffer = field;
+ if (field != null) { // Netty 3.5.0 or before.
+ RDB_buf = null;
+ } else {
+ RDB_buf = ReplayingDecoderBuffer.getDeclaredMethod("buf");
+ RDB_buf.setAccessible(true);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("static initializer failed", e);
+ }
+ }
+
+ /**
+ * Pretty-prints all the bytes of a buffer into a human-readable string.
+ * @param buf The (possibly {@code null}) buffer to pretty-print.
+ * @return The buffer in a pretty-printed string.
+ */
+ public static String pretty(final ChannelBuffer buf) {
+ if (buf == null) {
+ return "null";
+ }
+ byte[] array;
+ try {
+ if (buf.getClass() != ReplayingDecoderBuffer) {
+ array = buf.array();
+ } else if (RDB_buf != null) { // Netty 3.5.1 and above.
+ array = ((ChannelBuffer) RDB_buf.invoke(buf)).array();
+ } else { // Netty 3.5.0 and before.
+ final ChannelBuffer wrapped_buf = (ChannelBuffer) RDB_buffer.get(buf);
+ array = wrapped_buf.array();
+ }
+ } catch (UnsupportedOperationException e) {
+ return "(failed to extract content of buffer of type "
+ + buf.getClass().getName() + ')';
+ } catch (IllegalAccessException e) {
+ throw new AssertionError("Should not happen: " + e);
+ } catch (InvocationTargetException e) {
+ throw new AssertionError("Should not happen: " + e);
+ }
+ return pretty(array);
+ }
+
+ // ---------------------- //
+ // Comparing byte arrays. //
+ // ---------------------- //
+ // Don't ask me why this isn't in java.util.Arrays.
+
+ /**
+ * A singleton {@link Comparator} for non-{@code null} byte arrays.
+ * @see #memcmp
+ */
+ public static final MemCmp MEMCMP = new MemCmp();
+
+ /** {@link Comparator} for non-{@code null} byte arrays. */
+ private static final class MemCmp implements Comparator<byte[]> {
+
+ private MemCmp() { // Can't instantiate outside of this class.
+ }
+
+ @Override
+ public int compare(final byte[] a, final byte[] b) {
+ return memcmp(a, b);
+ }
+
+ }
+
+ /**
+ * {@code memcmp} in Java, hooray.
+ * @param a First non-{@code null} byte array to compare.
+ * @param b Second non-{@code null} byte array to compare.
+ * @return 0 if the two arrays are identical, otherwise the difference
+ * between the first two different bytes, otherwise the different between
+ * their lengths.
+ */
+ public static int memcmp(final byte[] a, final byte[] b) {
+ final int length = Math.min(a.length, b.length);
+ if (a == b) { // Do this after accessing a.length and b.length
+ return 0; // in order to NPE if either a or b is null.
+ }
+ for (int i = 0; i < length; i++) {
+ if (a[i] != b[i]) {
+ return (a[i] & 0xFF) - (b[i] & 0xFF); // "promote" to unsigned.
+ }
+ }
+ return a.length - b.length;
+ }
+
+ /**
+ * {@code memcmp(3)} with a given offset and length.
+ * @param a First non-{@code null} byte array to compare.
+ * @param b Second non-{@code null} byte array to compare.
+ * @param offset The offset at which to start comparing both arrays.
+ * @param length The number of bytes to compare.
+ * @return 0 if the two arrays are identical, otherwise the difference
+ * between the first two different bytes (treated as unsigned), otherwise
+ * the different between their lengths.
+ * @throws IndexOutOfBoundsException if either array isn't large enough.
+ */
+ public static int memcmp(final byte[] a, final byte[] b,
+ final int offset, int length) {
+ if (a == b && a != null) {
+ return 0;
+ }
+ length += offset;
+ for (int i = offset; i < length; i++) {
+ if (a[i] != b[i]) {
+ return (a[i] & 0xFF) - (b[i] & 0xFF); // "promote" to unsigned.
+ }
+ }
+ return 0;
+ }
+
+ /**
+ * De-duplicates two byte arrays.
+ * <p>
+ * If two byte arrays have the same contents but are different, this
+ * function helps to re-use the old one and discard the new copy.
+ * @param old The existing byte array.
+ * @param neww The new byte array we're trying to de-duplicate.
+ * @return {@code old} if {@code neww} is a different array with the same
+ * contents, otherwise {@code neww}.
+ */
+ public static byte[] deDup(final byte[] old, final byte[] neww) {
+ return memcmp(old, neww) == 0 ? old : neww;
+ }
+
+ /**
+ * Tests whether two byte arrays have the same contents.
+ * @param a First non-{@code null} byte array to compare.
+ * @param b Second non-{@code null} byte array to compare.
+ * @return {@code true} if the two arrays are identical,
+ * {@code false} otherwise.
+ */
+ public static boolean equals(final byte[] a, final byte[] b) {
+ return memcmp(a, b) == 0;
+ }
+
+ /**
+ * {@code memcmp(3)} in Java for possibly {@code null} arrays, hooray.
+ * @param a First possibly {@code null} byte array to compare.
+ * @param b Second possibly {@code null} byte array to compare.
+ * @return 0 if the two arrays are identical (or both are {@code null}),
+ * otherwise the difference between the first two different bytes (treated
+ * as unsigned), otherwise the different between their lengths (a {@code
+ * null} byte array is considered shorter than an empty byte array).
+ */
+ public static int memcmpMaybeNull(final byte[] a, final byte[] b) {
+ if (a == null) {
+ if (b == null) {
+ return 0;
+ }
+ return -1;
+ } else if (b == null) {
+ return 1;
+ }
+ return memcmp(a, b);
+ }
+
+ public static int getBitSetSize(int items) {
+ return (items + 7) / 8;
+ }
+
+ public static byte[] fromBitSet(BitSet bits, int colCount) {
+ byte[] bytes = new byte[getBitSetSize(colCount)];
+ for (int i = 0; i < bits.length(); i++) {
+ if (bits.get(i)) {
+ bytes[i / 8] |= 1 << (i % 8);
+ }
+ }
+ return bytes;
+ }
+
+ public static BitSet toBitSet(byte[] b, int offset, int colCount) {
+ BitSet bs = new BitSet(colCount);
+ for (int i = 0; i < colCount; i++) {
+ if ((b[offset + (i / 8)] >> (i % 8) & 1) == 1) {
+ bs.set(i);
+ }
+ }
+ return bs;
+ }
+
+ /**
+ * This method will apply xor on the left most bit of the provided byte. This is used in Kudu to
+ * have unsigned data types sorting correctly.
+ * @param value byte whose left most bit will be xor'd
+ * @return same byte with xor applied on the left most bit
+ */
+ public static byte xorLeftMostBit(byte value) {
+ value ^= (1 << 7);
+ return value;
+ }
+
+ /**
+ * Get the byte array representation of this string, with UTF8 encoding
+ * @param data String get the byte array from
+ * @return UTF8 byte array
+ */
+ public static byte[] fromString(String data) {
+ return UTF8(data);
+ }
+
+ /**
+ * Get a string from the passed byte array, with UTF8 encoding
+ * @param b byte array to convert to string, possibly coming from {@link #fromString(String)}
+ * @return A new string built with the byte array
+ */
+ public static String getString(byte[] b) {
+ return getString(b, 0, b.length);
+ }
+
+ public static String getString(Slice slice) {
+ return slice.toString(CharsetUtil.UTF_8);
+ }
+
+ /**
+ * Get a string from the passed byte array, at the specified offset and for the specified
+ * length, with UTF8 encoding
+ * @param b byte array to convert to string, possibly coming from {@link #fromString(String)}
+ * @param offset where to start reading from in the byte array
+ * @param len how many bytes we should read
+ * @return A new string built with the byte array
+ */
+ public static String getString(byte[] b, int offset, int len) {
+ if (len == 0) {
+ return "";
+ }
+ return new String(b, offset, len, CharsetUtil.UTF_8);
+ }
+
+ /**
+ * Utility method to write a byte array to a data output. Equivalent of doing a writeInt of the
+ * length followed by a write of the byte array. Convert back with {@link #readByteArray}
+ * @param dataOutput
+ * @param b
+ * @throws IOException
+ */
+ public static void writeByteArray(DataOutput dataOutput, byte[] b) throws IOException {
+ dataOutput.writeInt(b.length);
+ dataOutput.write(b);
+ }
+
+ /**
+ * Utility method to read a byte array written the way {@link #writeByteArray} does it.
+ * @param dataInput
+ * @return
+ * @throws IOException
+ */
+ public static byte[] readByteArray(DataInput dataInput) throws IOException {
+ int len = dataInput.readInt();
+ byte[] data = new byte[len];
+ dataInput.readFully(data);
+ return data;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java
new file mode 100644
index 0000000..77cbd5e
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import java.util.List;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.rpc.RpcHeader;
+import org.kududb.util.Slice;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * This class handles information received from an RPC response, providing
+ * access to sidecars and decoded protobufs from the message.
+ */
+@InterfaceAudience.Private
+final class CallResponse {
+ private final ChannelBuffer buf;
+ private final RpcHeader.ResponseHeader header;
+ private final int totalResponseSize;
+
+ // Non-header main message slice is generated upon request and cached.
+ private Slice message = null;
+
+ /**
+ * Performs some sanity checks on the sizes recorded in the packet
+ * referred to by {@code buf}. Assumes that {@code buf} has not been
+ * read from yet, and will only be accessed by this class.
+ *
+ * Afterwards, this constructs the RpcHeader from the buffer.
+ * @param buf Channel buffer which call response reads from.
+ * @throws IllegalArgumentException If either the entire recorded packet
+ * size or recorded response header PB size are not within reasonable
+ * limits as defined by {@link KuduRpc#checkArrayLength(ChannelBuffer, long)}.
+ * @throws IndexOutOfBoundsException if the ChannelBuffer does not contain
+ * the amount of bytes specified by its length prefix.
+ */
+ public CallResponse(final ChannelBuffer buf) {
+ this.buf = buf;
+
+ this.totalResponseSize = buf.readInt();
+ KuduRpc.checkArrayLength(buf, this.totalResponseSize);
+ TabletClient.ensureReadable(buf, this.totalResponseSize);
+
+ final int headerSize = Bytes.readVarInt32(buf);
+ final Slice headerSlice = nextBytes(buf, headerSize);
+ RpcHeader.ResponseHeader.Builder builder = RpcHeader.ResponseHeader.newBuilder();
+ KuduRpc.readProtobuf(headerSlice, builder);
+ this.header = builder.build();
+ }
+
+ /**
+ * @return the parsed header
+ */
+ public RpcHeader.ResponseHeader getHeader() {
+ return this.header;
+ }
+
+ /**
+ * @return the total response size
+ */
+ public int getTotalResponseSize() { return this.totalResponseSize; }
+
+ /**
+ * @return A slice pointing to the section of the packet reserved for the main
+ * protobuf message.
+ * @throws IllegalArgumentException If the recorded size for the main message
+ * is not within reasonable limits as defined by
+ * {@link KuduRpc#checkArrayLength(ChannelBuffer, long)}.
+ * @throws IllegalStateException If the offset for the main protobuf message
+ * is not valid.
+ */
+ public Slice getPBMessage() {
+ cacheMessage();
+ final int mainLength = this.header.getSidecarOffsetsCount() == 0 ?
+ this.message.length() : this.header.getSidecarOffsets(0);
+ if (mainLength < 0 || mainLength > this.message.length()) {
+ throw new IllegalStateException("Main protobuf message invalid. "
+ + "Length is " + mainLength + " while the size of the message "
+ + "excluding the header is " + this.message.length());
+ }
+ return subslice(this.message, 0, mainLength);
+ }
+
+ /**
+ * @param sidecar The index of the sidecar to retrieve.
+ * @return A slice pointing to the desired sidecar.
+ * @throws IllegalStateException If the sidecar offsets specified in the
+ * header response PB are not valid offsets for the array.
+ * @throws IllegalArgumentException If the sidecar with the specified index
+ * does not exist.
+ * @throws IllegalArgumentException If the recorded size for the main message
+ * is not within reasonable limits as defined by
+ * {@link KuduRpc#checkArrayLength(ChannelBuffer, long)}.
+ */
+ public Slice getSidecar(int sidecar) {
+ cacheMessage();
+
+ List<Integer> sidecarList = this.header.getSidecarOffsetsList();
+ if (sidecar < 0 || sidecar > sidecarList.size()) {
+ throw new IllegalArgumentException("Sidecar " + sidecar
+ + " not valid, response has " + sidecarList.size() + " sidecars");
+ }
+
+ final int prevOffset = sidecarList.get(sidecar);
+ final int nextOffset = sidecar + 1 == sidecarList.size() ?
+ this.message.length() : sidecarList.get(sidecar + 1);
+ final int length = nextOffset - prevOffset;
+
+ if (prevOffset < 0 || length < 0 || prevOffset + length > this.message.length()) {
+ throw new IllegalStateException("Sidecar " + sidecar + " invalid "
+ + "(offset = " + prevOffset + ", length = " + length + "). The size "
+ + "of the message " + "excluding the header is " + this.message.length());
+ }
+
+ return subslice(this.message, prevOffset, length);
+ }
+
+ // Reads the message after the header if not read yet
+ private void cacheMessage() {
+ if (this.message != null) return;
+ final int length = Bytes.readVarInt32(buf);
+ this.message = nextBytes(buf, length);
+ }
+
+ // Accounts for a parent slice's offset when making a new one with relative offsets.
+ private static Slice subslice(Slice parent, int offset, int length) {
+ return new Slice(parent.getRawArray(), parent.getRawOffset() + offset, length);
+ }
+
+ // After checking the length, generates a slice for the next 'length'
+ // bytes of 'buf'.
+ private static Slice nextBytes(final ChannelBuffer buf, final int length) {
+ KuduRpc.checkArrayLength(buf, length);
+ byte[] payload;
+ int offset;
+ if (buf.hasArray()) { // Zero copy.
+ payload = buf.array();
+ offset = buf.arrayOffset() + buf.readerIndex();
+ } else { // We have to copy the entire payload out of the buffer :(
+ payload = new byte[length];
+ buf.readBytes(payload);
+ offset = 0;
+ }
+ return new Slice(payload, offset, length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java
new file mode 100644
index 0000000..0e24088
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java
@@ -0,0 +1,387 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+import org.kududb.ColumnSchema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.tserver.Tserver;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A range predicate on one of the columns in the underlying data.
+ * Both boundaries are inclusive.
+ * @deprecated use the {@link KuduPredicate} class instead.
+ */
+@InterfaceAudience.Public
+@Deprecated
+public class ColumnRangePredicate {
+
+ private final Tserver.ColumnRangePredicatePB.Builder pb = Tserver.ColumnRangePredicatePB
+ .newBuilder();
+ private final ColumnSchema column;
+ private byte[] lowerBound = null;
+ private byte[] upperBound = null;
+
+ /**
+ * Create the predicate on the specified column
+ * @param column
+ */
+ public ColumnRangePredicate(ColumnSchema column) {
+ this.column = column;
+ this.pb.setColumn(ProtobufHelper.columnToPb(column));
+ }
+
+ private void setLowerBoundInternal(byte[] value) {
+ this.lowerBound = value;
+ pb.setLowerBound(ZeroCopyLiteralByteString.wrap(this.lowerBound));
+ }
+
+ private void setUpperBoundInternal(byte[] value) {
+ this.upperBound = value;
+ pb.setInclusiveUpperBound(ZeroCopyLiteralByteString.wrap(this.upperBound));
+ }
+
+ /**
+ * Convert a bound into a {@link KuduPredicate}.
+ * @param column the column
+ * @param op the bound comparison operator
+ * @param bound the bound
+ * @return the {@code KuduPredicate}
+ */
+ private static KuduPredicate toKuduPredicate(ColumnSchema column,
+ KuduPredicate.ComparisonOp op,
+ byte[] bound) {
+ if (bound == null) { return null; }
+ switch (column.getType().getDataType()) {
+ case BOOL: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getBoolean(bound));
+ case INT8: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getByte(bound));
+ case INT16: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getShort(bound));
+ case INT32: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getInt(bound));
+ case INT64:
+ case TIMESTAMP: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getLong(bound));
+ case FLOAT: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getFloat(bound));
+ case DOUBLE: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getDouble(bound));
+ case STRING: return KuduPredicate.newComparisonPredicate(column, op, Bytes.getString(bound));
+ case BINARY: return KuduPredicate.newComparisonPredicate(column, op, bound);
+ default:
+ throw new IllegalStateException(String.format("unknown column type %s", column.getType()));
+ }
+ }
+
+ /**
+ * Convert this column range predicate into a {@link KuduPredicate}.
+ * @return the column predicate.
+ */
+ public KuduPredicate toKuduPredicate() {
+ KuduPredicate lower =
+ toKuduPredicate(column, KuduPredicate.ComparisonOp.GREATER_EQUAL, lowerBound);
+ KuduPredicate upper =
+ toKuduPredicate(column, KuduPredicate.ComparisonOp.LESS_EQUAL, upperBound);
+
+ if (upper != null && lower != null) {
+ return upper.merge(lower);
+ } else if (upper != null) {
+ return upper;
+ } else {
+ return lower;
+ }
+ }
+
+ /**
+ * Set a boolean for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(boolean lowerBound) {
+ checkColumn(Type.BOOL);
+ setLowerBoundInternal(Bytes.fromBoolean(lowerBound));
+ }
+
+ /**
+ * Set a byte for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(byte lowerBound) {
+ checkColumn(Type.INT8);
+ setLowerBoundInternal(new byte[] {lowerBound});
+ }
+
+ /**
+ * Set a short for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(short lowerBound) {
+ checkColumn(Type.INT16);
+ setLowerBoundInternal(Bytes.fromShort(lowerBound));
+ }
+
+ /**
+ * Set an int for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(int lowerBound) {
+ checkColumn(Type.INT32);
+ setLowerBoundInternal(Bytes.fromInt(lowerBound));
+ }
+
+ /**
+ * Set a long for the lower bound
+ *
+ * If 'lowerBound' is a timestamp see {@link PartialRow#addLong(String, long)} for the
+ * format.
+ *
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(long lowerBound) {
+ checkColumn(Type.INT64, Type.TIMESTAMP);
+ setLowerBoundInternal(Bytes.fromLong(lowerBound));
+ }
+
+ /**
+ * Set a string for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(String lowerBound) {
+ checkColumn(Type.STRING);
+ setLowerBoundInternal(lowerBound.getBytes());
+ }
+
+ /**
+ * Set a binary value for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(byte[] lowerBound) {
+ checkColumn(Type.BINARY);
+ setLowerBoundInternal(lowerBound);
+ }
+
+ /**
+ * Set a float for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(float lowerBound) {
+ checkColumn(Type.FLOAT);
+ setLowerBoundInternal(Bytes.fromFloat(lowerBound));
+ }
+
+ /**
+ * Set a double for the lower bound
+ * @param lowerBound value for the lower bound
+ */
+ public void setLowerBound(double lowerBound) {
+ checkColumn(Type.DOUBLE);
+ setLowerBoundInternal(Bytes.fromDouble(lowerBound));
+ }
+
+ /**
+ * Set a boolean for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(boolean upperBound) {
+ checkColumn(Type.BOOL);
+ setUpperBoundInternal(Bytes.fromBoolean(upperBound));
+ }
+
+ /**
+ * Set a byte for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(byte upperBound) {
+ checkColumn(Type.INT8);
+ setUpperBoundInternal(new byte[] {upperBound});
+ }
+
+ /**
+ * Set a short for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(short upperBound) {
+ checkColumn(Type.INT16);
+ setUpperBoundInternal(Bytes.fromShort(upperBound));
+ }
+
+ /**
+ * Set an int for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(int upperBound) {
+ checkColumn(Type.INT32);
+ setUpperBoundInternal(Bytes.fromInt(upperBound));
+ }
+
+ /**
+ * Set a long for the upper bound
+ *
+ * If 'upperBound' is a timestamp see {@link PartialRow#addLong(String, long)} for the
+ * format.
+ *
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(long upperBound) {
+ checkColumn(Type.INT64, Type.TIMESTAMP);
+ setUpperBoundInternal(Bytes.fromLong(upperBound));
+ }
+
+ /**
+ * Set a string for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(String upperBound) {
+ checkColumn(Type.STRING);
+ setUpperBoundInternal(upperBound.getBytes());
+ }
+
+ /**
+ * Set a binary value for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(byte[] upperBound) {
+ checkColumn(Type.BINARY);
+ setUpperBoundInternal(upperBound);
+ }
+
+ /**
+ * Set a float for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(float upperBound) {
+ checkColumn(Type.FLOAT);
+ setUpperBoundInternal(Bytes.fromFloat(upperBound));
+ }
+
+ /**
+ * Set a double for the upper bound
+ * @param upperBound value for the upper bound
+ */
+ public void setUpperBound(double upperBound) {
+ checkColumn(Type.DOUBLE);
+ setUpperBoundInternal(Bytes.fromDouble(upperBound));
+ }
+
+ /**
+ * Get the column used by this predicate
+ * @return the column
+ */
+ public ColumnSchema getColumn() {
+ return column;
+ }
+
+ /**
+ * Get the lower bound in its raw representation
+ * @return lower bound as a byte array
+ */
+ public byte[] getLowerBound() {
+ return lowerBound;
+ }
+
+ /**
+ * Get the upper bound in its raw representation
+ * @return upper bound as a byte array
+ */
+ public byte[] getUpperBound() {
+ return upperBound;
+ }
+
+ /**
+ * Converts a list of predicates into an opaque byte array. This is a convenience method for use
+ * cases that require passing predicates as messages.
+ * @param predicates a list of predicates
+ * @return an opaque byte array, or null if the list was empty
+ */
+ public static byte[] toByteArray(List<ColumnRangePredicate> predicates) {
+ if (predicates.isEmpty()) {
+ return null;
+ }
+
+ Tserver.ColumnRangePredicateListPB.Builder predicateListBuilder =
+ Tserver.ColumnRangePredicateListPB.newBuilder();
+
+ for (ColumnRangePredicate crp : predicates) {
+ predicateListBuilder.addRangePredicates(crp.getPb());
+ }
+
+ return predicateListBuilder.build().toByteArray();
+ }
+
+ /**
+ * Converts a given byte array to a list of predicates in their pb format.
+ * @param listBytes bytes obtained from {@link #toByteArray(List)}
+ * @return a list of predicates
+ * @throws IllegalArgumentException thrown when the passed bytes aren't valid
+ */
+ static List<Tserver.ColumnRangePredicatePB> fromByteArray(byte[] listBytes) {
+ List<Tserver.ColumnRangePredicatePB> predicates = new ArrayList<>();
+ if (listBytes == null || listBytes.length == 0) {
+ return predicates;
+ }
+ Tserver.ColumnRangePredicateListPB list = ColumnRangePredicate.getPbFromBytes(listBytes);
+ return list.getRangePredicatesList();
+ }
+
+ /**
+ * Get the predicate in its protobuf form.
+ * @return this predicate in protobuf
+ */
+ Tserver.ColumnRangePredicatePB getPb() {
+ return pb.build();
+ }
+
+ /**
+ * Creates a {@code ColumnRangePredicate} from a protobuf column range predicate message.
+ * @param pb the protobuf message
+ * @return a column range predicate
+ */
+ static ColumnRangePredicate fromPb(Tserver.ColumnRangePredicatePB pb) {
+ ColumnRangePredicate pred =
+ new ColumnRangePredicate(ProtobufHelper.pbToColumnSchema(pb.getColumn()));
+ if (pb.hasLowerBound()) {
+ pred.setLowerBoundInternal(pb.getLowerBound().toByteArray());
+ }
+ if (pb.hasInclusiveUpperBound()) {
+ pred.setUpperBoundInternal(pb.getInclusiveUpperBound().toByteArray());
+ }
+ return pred;
+ }
+
+ /**
+ * Convert a list of predicates given in bytes back to its pb format. It also hides the
+ * InvalidProtocolBufferException.
+ */
+ private static Tserver.ColumnRangePredicateListPB getPbFromBytes(byte[] listBytes) {
+ try {
+ return Tserver.ColumnRangePredicateListPB.parseFrom(listBytes);
+ } catch (InvalidProtocolBufferException e) {
+ // We shade our pb dependency so we can't send out the exception above since other modules
+ // won't know what to expect.
+ throw new IllegalArgumentException("Encountered an invalid column range predicate list: "
+ + Bytes.pretty(listBytes), e);
+ }
+ }
+
+ private void checkColumn(Type... passedTypes) {
+ for (Type type : passedTypes) {
+ if (this.column.getType().equals(type)) return;
+ }
+ throw new IllegalArgumentException(String.format("%s's type isn't %s, it's %s",
+ column.getName(), Arrays.toString(passedTypes), column.getType().getName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
new file mode 100644
index 0000000..20bc4c3
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
@@ -0,0 +1,172 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+import org.kududb.Common;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.master.Master;
+import org.kududb.util.Pair;
+
+/**
+ * This is a builder class for all the options that can be provided while creating a table.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class CreateTableOptions {
+
+ private Master.CreateTableRequestPB.Builder pb = Master.CreateTableRequestPB.newBuilder();
+ private final List<PartialRow> splitRows = Lists.newArrayList();
+ private final List<Pair<PartialRow, PartialRow>> rangeBounds = Lists.newArrayList();
+
+ /**
+ * Add a split point for the table. The table in the end will have splits + 1 tablets.
+ * The row may be reused or modified safely after this call without changing the split point.
+ *
+ * @param row a key row for the split point
+ * @return this instance
+ */
+ public CreateTableOptions addSplitRow(PartialRow row) {
+ splitRows.add(new PartialRow(row));
+ return this;
+ }
+
+ /**
+ * Add a partition range bound to the table with an inclusive lower bound and
+ * exclusive upper bound.
+ *
+ * If either row is empty, then that end of the range will be unbounded. If a
+ * range column is missing a value, the logical minimum value for that column
+ * type will be used as the default.
+ *
+ * Multiple range bounds may be added, but they must not overlap. All split
+ * rows must fall in one of the range bounds. The lower bound must be less
+ * than the upper bound.
+ *
+ * If not provided, the table's range will be unbounded.
+ *
+ * @param lower the inclusive lower bound
+ * @param upper the exclusive upper bound
+ * @return this instance
+ */
+ public CreateTableOptions addRangeBound(PartialRow lower, PartialRow upper) {
+ rangeBounds.add(new Pair<>(new PartialRow(lower), new PartialRow(upper)));
+ return this;
+ }
+
+ /**
+ * Add a set of hash partitions to the table.
+ *
+ * Each column must be a part of the table's primary key, and an individual
+ * column may only appear in a single hash component.
+ *
+ * For each set of hash partitions added to the table, the total number of
+ * table partitions is multiplied by the number of buckets. For example, if a
+ * table is created with 3 split rows, and two hash partitions with 4 and 5
+ * buckets respectively, the total number of table partitions will be 80
+ * (4 range partitions * 4 hash buckets * 5 hash buckets).
+ *
+ * @param columns the columns to hash
+ * @param buckets the number of buckets to hash into
+ * @return this instance
+ */
+ public CreateTableOptions addHashPartitions(List<String> columns, int buckets) {
+ addHashPartitions(columns, buckets, 0);
+ return this;
+ }
+
+ /**
+ * Add a set of hash partitions to the table.
+ *
+ * This constructor takes a seed value, which can be used to randomize the
+ * mapping of rows to hash buckets. Setting the seed may provide some
+ * amount of protection against denial of service attacks when the hashed
+ * columns contain user provided values.
+ *
+ * @param columns the columns to hash
+ * @param buckets the number of buckets to hash into
+ * @param seed a hash seed
+ * @return this instance
+ */
+ public CreateTableOptions addHashPartitions(List<String> columns, int buckets, int seed) {
+ Common.PartitionSchemaPB.HashBucketSchemaPB.Builder hashBucket =
+ pb.getPartitionSchemaBuilder().addHashBucketSchemasBuilder();
+ for (String column : columns) {
+ hashBucket.addColumnsBuilder().setName(column);
+ }
+ hashBucket.setNumBuckets(buckets);
+ hashBucket.setSeed(seed);
+ return this;
+ }
+
+ /**
+ * Set the columns on which the table will be range-partitioned.
+ *
+ * Every column must be a part of the table's primary key. If not set or if
+ * called with an empty vector, the table will be created without range
+ * partitioning.
+ *
+ * Tables must be created with either range, hash, or range and hash
+ * partitioning. To force the use of a single tablet (not recommended),
+ * call this method with an empty list and set no split rows and no hash
+ * partitions.
+ *
+ * @param columns the range partitioned columns
+ * @return this instance
+ */
+ public CreateTableOptions setRangePartitionColumns(List<String> columns) {
+ Common.PartitionSchemaPB.RangeSchemaPB.Builder rangePartition =
+ pb.getPartitionSchemaBuilder().getRangeSchemaBuilder();
+ for (String column : columns) {
+ rangePartition.addColumnsBuilder().setName(column);
+ }
+ return this;
+ }
+
+ /**
+ * Sets the number of replicas that each tablet will have. If not specified, it uses the
+ * server-side default which is usually 3 unless changed by an administrator.
+ *
+ * @param numReplicas the number of replicas to use
+ * @return this instance
+ */
+ public CreateTableOptions setNumReplicas(int numReplicas) {
+ pb.setNumReplicas(numReplicas);
+ return this;
+ }
+
+ Master.CreateTableRequestPB.Builder getBuilder() {
+ if (!splitRows.isEmpty() || !rangeBounds.isEmpty()) {
+ pb.setSplitRowsRangeBounds(new Operation.OperationsEncoder()
+ .encodeSplitRowsRangeBounds(splitRows, rangeBounds));
+ }
+ return pb;
+ }
+
+ List<Integer> getRequiredFeatureFlags() {
+ if (rangeBounds.isEmpty()) {
+ return ImmutableList.of();
+ } else {
+ return ImmutableList.of(Master.MasterFeatures.RANGE_PARTITION_BOUNDS_VALUE);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java
new file mode 100644
index 0000000..31ed9a2
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.master.Master;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * RPC to create new tables
+ */
+@InterfaceAudience.Private
+class CreateTableRequest extends KuduRpc<CreateTableResponse> {
+
+ static final String CREATE_TABLE = "CreateTable";
+
+ private final Schema schema;
+ private final String name;
+ private final Master.CreateTableRequestPB.Builder builder;
+ private final List<Integer> featureFlags;
+
+ CreateTableRequest(KuduTable masterTable, String name, Schema schema,
+ CreateTableOptions builder) {
+ super(masterTable);
+ this.schema = schema;
+ this.name = name;
+ this.builder = builder.getBuilder();
+ featureFlags = builder.getRequiredFeatureFlags();
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ this.builder.setName(this.name);
+ this.builder.setSchema(ProtobufHelper.schemaToPb(this.schema));
+ return toChannelBuffer(header, this.builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return CREATE_TABLE;
+ }
+
+ @Override
+ Pair<CreateTableResponse, Object> deserialize(final CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final Master.CreateTableResponsePB.Builder builder = Master.CreateTableResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), builder);
+ CreateTableResponse response =
+ new CreateTableResponse(deadlineTracker.getElapsedMillis(), tsUUID);
+ return new Pair<CreateTableResponse, Object>(
+ response, builder.hasError() ? builder.getError() : null);
+ }
+
+ @Override
+ Collection<Integer> getRequiredFeatures() {
+ return featureFlags;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java
new file mode 100644
index 0000000..7906c5f
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+@InterfaceAudience.Private
+public class CreateTableResponse extends KuduRpcResponse {
+
+ /**
+ * @param ellapsedMillis Time in milliseconds since RPC creation to now.
+ */
+ CreateTableResponse(long ellapsedMillis, String tsUUID) {
+ super(ellapsedMillis, tsUUID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java b/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
new file mode 100644
index 0000000..fc30074
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Stopwatch;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This is a wrapper class around {@link com.google.common.base.Stopwatch} used to track a relative
+ * deadline in the future.
+ * <p>
+ * The watch starts as soon as this object is created with a deadline of 0,
+ * meaning that there's no deadline.
+ * The deadline has been reached once the stopwatch's elapsed time is equal or greater than the
+ * provided deadline.
+ */
+public class DeadlineTracker {
+ private final Stopwatch stopwatch;
+ /** relative deadline in milliseconds **/
+ private long deadline = 0;
+
+ /**
+ * Creates a new tracker, which starts the stopwatch right now.
+ */
+ public DeadlineTracker() {
+ this(Stopwatch.createUnstarted());
+ }
+
+ /**
+ * Creates a new tracker, using the specified stopwatch, and starts it right now.
+ * The stopwatch is reset if it was already running.
+ * @param stopwatch Specific Stopwatch to use
+ */
+ public DeadlineTracker(Stopwatch stopwatch) {
+ if (stopwatch.isRunning()) {
+ stopwatch.reset();
+ }
+ this.stopwatch = stopwatch.start();
+ }
+
+ /**
+ * Check if we're already past the deadline.
+ * @return true if we're past the deadline, otherwise false. Also returns false if no deadline
+ * was specified
+ */
+ public boolean timedOut() {
+ if (!hasDeadline()) {
+ return false;
+ }
+ return deadline - stopwatch.elapsed(TimeUnit.MILLISECONDS) <= 0;
+ }
+
+ /**
+ * Get the number of milliseconds before the deadline is reached.
+ * <p>
+ * This method is used to pass down the remaining deadline to the RPCs, so has special semantics.
+ * A deadline of 0 is used to indicate an infinite deadline, and negative deadlines are invalid.
+ * Thus, if the deadline has passed (i.e. <tt>deadline - stopwatch.elapsedMillis() <= 0</tt>),
+ * the returned value is floored at <tt>1</tt>.
+ * <p>
+ * Callers who care about this behavior should first check {@link #timedOut()}.
+ *
+ * @return the remaining millis before the deadline is reached, or 1 if the remaining time is
+ * lesser or equal to 0, or Long.MAX_VALUE if no deadline was specified (in which case it
+ * should never be called).
+ * @throws IllegalStateException if this method is called and no deadline was set
+ */
+ public long getMillisBeforeDeadline() {
+ if (!hasDeadline()) {
+ throw new IllegalStateException("This tracker doesn't have a deadline set so it cannot " +
+ "answer getMillisBeforeDeadline()");
+ }
+ long millisBeforeDeadline = deadline - stopwatch.elapsed(TimeUnit.MILLISECONDS);
+ millisBeforeDeadline = millisBeforeDeadline <= 0 ? 1 : millisBeforeDeadline;
+ return millisBeforeDeadline;
+ }
+
+ public long getElapsedMillis() {
+ return this.stopwatch.elapsed(TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Tells if a non-zero deadline was set.
+ * @return true if the deadline is greater than 0, false otherwise.
+ */
+ public boolean hasDeadline() {
+ return deadline != 0;
+ }
+
+ /**
+ * Utility method to check if sleeping for a specified amount of time would put us past the
+ * deadline.
+ * @param plannedSleepTime number of milliseconds for a planned sleep
+ * @return if the planned sleeps goes past the deadline.
+ */
+ public boolean wouldSleepingTimeout(long plannedSleepTime) {
+ if (!hasDeadline()) {
+ return false;
+ }
+ return getMillisBeforeDeadline() - plannedSleepTime <= 0;
+ }
+
+ /**
+ * Sets the deadline to 0 (no deadline) and restarts the stopwatch from scratch.
+ */
+ public void reset() {
+ deadline = 0;
+ stopwatch.reset();
+ stopwatch.start();
+ }
+
+ /**
+ * Get the deadline (in milliseconds).
+ * @return the current deadline
+ */
+ public long getDeadline() {
+ return deadline;
+ }
+
+ /**
+ * Set a new deadline for this tracker. It cannot be smaller than 0,
+ * and if it is 0 then it means that there is no deadline (which is the default behavior).
+ * This method won't call reset().
+ * @param deadline a number of milliseconds greater or equal to 0
+ * @throws IllegalArgumentException if the deadline is lesser than 0
+ */
+ public void setDeadline(long deadline) {
+ if (deadline < 0) {
+ throw new IllegalArgumentException("The deadline must be greater or equal to 0, " +
+ "the passed value is " + deadline);
+ }
+ this.deadline = deadline;
+ }
+
+ public String toString() {
+ StringBuffer buf = new StringBuffer("DeadlineTracker(timeout=");
+ buf.append(deadline);
+ buf.append(", elapsed=").append(stopwatch.elapsed(TimeUnit.MILLISECONDS));
+ buf.append(")");
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java
new file mode 100644
index 0000000..7a068bc
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.ColumnSchema;
+import org.kududb.Type;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Class of Operation for whole row removals.
+ * Only columns which are part of the key can be set.
+ * Instances of this class should not be reused.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Delete extends Operation {
+
+ Delete(KuduTable table) {
+ super(table);
+ }
+
+ @Override
+ ChangeType getChangeType() {
+ return ChangeType.DELETE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java
new file mode 100644
index 0000000..7f8fa51
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.protobuf.Message;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.master.Master;
+import org.kududb.util.Pair;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * RPC to delete tables
+ */
+@InterfaceAudience.Private
+class DeleteTableRequest extends KuduRpc<DeleteTableResponse> {
+
+ static final String DELETE_TABLE = "DeleteTable";
+
+ private final String name;
+
+ DeleteTableRequest(KuduTable table, String name) {
+ super(table);
+ this.name = name;
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ assert header.isInitialized();
+ final Master.DeleteTableRequestPB.Builder builder = Master.DeleteTableRequestPB.newBuilder();
+ Master.TableIdentifierPB tableID =
+ Master.TableIdentifierPB.newBuilder().setTableName(name).build();
+ builder.setTable(tableID);
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() { return MASTER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return DELETE_TABLE;
+ }
+
+ @Override
+ Pair<DeleteTableResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ final Master.DeleteTableResponsePB.Builder builder = Master.DeleteTableResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), builder);
+ DeleteTableResponse response =
+ new DeleteTableResponse(deadlineTracker.getElapsedMillis(), tsUUID);
+ return new Pair<DeleteTableResponse, Object>(
+ response, builder.hasError() ? builder.getError() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java
new file mode 100644
index 0000000..51f3ba7
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class DeleteTableResponse extends KuduRpcResponse {
+
+ /**
+ * @param ellapsedMillis Time in milliseconds since RPC creation to now.
+ */
+ DeleteTableResponse(long ellapsedMillis, String tsUUID) {
+ super(ellapsedMillis, tsUUID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java
new file mode 100644
index 0000000..db2952c
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.base.Preconditions;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+import java.util.ArrayDeque;
+import java.util.Queue;
+
+/**
+ * Class that helps tracking row errors. All methods are thread-safe.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ErrorCollector {
+ private final Queue<RowError> errorQueue;
+ private final int maxCapacity;
+ private boolean overflowed;
+
+ /**
+ * Create a new error collector with a maximum capacity.
+ * @param maxCapacity how many errors can be stored, has to be higher than 0
+ */
+ public ErrorCollector(int maxCapacity) {
+ Preconditions.checkArgument(maxCapacity > 0, "Need to be able to store at least one row error");
+ this.maxCapacity = maxCapacity;
+ this.errorQueue = new ArrayDeque<>(maxCapacity);
+ }
+
+ /**
+ * Add a new error to this collector. If it is already at max capacity, the oldest error will be
+ * discarded before the new one is added.
+ * @param rowError a row error to collect
+ */
+ public synchronized void addError(RowError rowError) {
+ if (errorQueue.size() >= maxCapacity) {
+ errorQueue.poll();
+ overflowed = true;
+ }
+ errorQueue.add(rowError);
+ }
+
+ /**
+ * Get the current count of collected row errors. Cannot be greater than the max capacity this
+ * instance was configured with.
+ * @return the count of errors
+ */
+ public synchronized int countErrors() {
+ return errorQueue.size();
+ }
+
+ /**
+ * Get all the errors that have been collected and an indication if the list overflowed.
+ * The list of errors cleared and the overflow state is reset.
+ * @return an object that contains both the list of row errors and the overflow status
+ */
+ public synchronized RowErrorsAndOverflowStatus getErrors() {
+ RowError[] returnedErrors = new RowError[errorQueue.size()];
+ errorQueue.toArray(returnedErrors);
+ errorQueue.clear();
+
+ RowErrorsAndOverflowStatus returnObject =
+ new RowErrorsAndOverflowStatus(returnedErrors, overflowed);
+ overflowed = false;
+ return returnObject;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java
new file mode 100644
index 0000000..adfb624
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import org.kududb.Common;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * The possible external consistency modes on which Kudu operates.
+ * See {@code src/kudu/common/common.proto} for a detailed explanations on the
+ * meaning and implications of each mode.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum ExternalConsistencyMode {
+ CLIENT_PROPAGATED(Common.ExternalConsistencyMode.CLIENT_PROPAGATED),
+ COMMIT_WAIT(Common.ExternalConsistencyMode.COMMIT_WAIT);
+
+ private Common.ExternalConsistencyMode pbVersion;
+ private ExternalConsistencyMode(Common.ExternalConsistencyMode pbVersion) {
+ this.pbVersion = pbVersion;
+ }
+ @InterfaceAudience.Private
+ public Common.ExternalConsistencyMode pbVersion() {
+ return pbVersion;
+ }
+}
[05/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestScannerMultiTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestScannerMultiTablet.java b/java/kudu-client/src/test/java/org/kududb/client/TestScannerMultiTablet.java
deleted file mode 100644
index 251057f..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestScannerMultiTablet.java
+++ /dev/null
@@ -1,236 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.stumbleupon.async.Deferred;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.ArrayList;
-
-import static org.junit.Assert.assertNull;
-import static org.kududb.Type.STRING;
-import static org.junit.Assert.assertEquals;
-
-public class TestScannerMultiTablet extends BaseKuduTest {
- // Generate a unique table name
- private static final String TABLE_NAME =
- TestScannerMultiTablet.class.getName()+"-"+System.currentTimeMillis();
-
- private static Schema schema = getSchema();
- private static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- // create a 4-tablets table for scanning
- CreateTableOptions builder =
- new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key1", "key2"));
-
- for (int i = 1; i < 4; i++){
- PartialRow splitRow = schema.newPartialRow();
- splitRow.addString("key1", "" + i);
- splitRow.addString("key2", "");
- builder.addSplitRow(splitRow);
- }
-
- createTable(TABLE_NAME, schema, builder);
-
- table = openTable(TABLE_NAME);
-
- AsyncKuduSession session = client.newSession();
- session.setFlushMode(AsyncKuduSession.FlushMode.AUTO_FLUSH_SYNC);
-
- // The data layout ends up like this:
- // tablet '', '1': no rows
- // tablet '1', '2': '111', '122', '133'
- // tablet '2', '3': '211', '222', '233'
- // tablet '3', '': '311', '322', '333'
- String[] keys = new String[] {"1", "2", "3"};
- for (String key1 : keys) {
- for (String key2 : keys) {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addString(0, key1);
- row.addString(1, key2);
- row.addString(2, key2);
- Deferred<OperationResponse> d = session.apply(insert);
- d.join(DEFAULT_SLEEP);
- }
- }
- }
-
- // Test various combinations of start/end row keys.
- @Test(timeout = 100000)
- public void testKeyStartEnd() throws Exception {
- assertEquals(0,
- countRowsInScan(getScanner("", "", "1", ""))); // There's nothing in the 1st tablet
- assertEquals(1, countRowsInScan(getScanner("", "", "1", "2"))); // Grab the very first row
- assertEquals(3, countRowsInScan(getScanner("1", "1", "1", "4"))); // Grab the whole 2nd tablet
- assertEquals(3, countRowsInScan(getScanner("1", "1", "2", ""))); // Same, and peek at the 3rd
- assertEquals(3, countRowsInScan(getScanner("1", "1", "2", "0"))); // Same, different peek
- assertEquals(4,
- countRowsInScan(getScanner("1", "2", "2", "3"))); // Middle of 2nd to middle of 3rd
- assertEquals(3,
- countRowsInScan(getScanner("1", "4", "2", "4"))); // Peek at the 2nd then whole 3rd
- assertEquals(6, countRowsInScan(getScanner("1", "5", "3", "4"))); // Whole 3rd and 4th
- assertEquals(9, countRowsInScan(getScanner("", "", "4", ""))); // Full table scan
-
- assertEquals(9,
- countRowsInScan(getScanner("", "", null, null))); // Full table scan with empty upper
- assertEquals(9,
- countRowsInScan(getScanner(null, null, "4", ""))); // Full table scan with empty lower
- assertEquals(9,
- countRowsInScan(getScanner(null, null, null, null))); // Full table scan with empty bounds
-
- // Test that we can close a scanner while in between two tablets. We start on the second
- // tablet and our first nextRows() will get 3 rows. At that moment we want to close the scanner
- // before getting on the 3rd tablet.
- AsyncKuduScanner scanner = getScanner("1", "", null, null);
- Deferred<RowResultIterator> d = scanner.nextRows();
- RowResultIterator rri = d.join(DEFAULT_SLEEP);
- assertEquals(3, rri.getNumRows());
- d = scanner.close();
- rri = d.join(DEFAULT_SLEEP);
- assertNull(rri);
- }
-
- // Test mixing start/end row keys with predicates.
- @Test(timeout = 100000)
- public void testKeysAndPredicates() throws Exception {
- // First row from the 2nd tablet.
- ColumnRangePredicate predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
- predicate.setLowerBound("1");
- predicate.setUpperBound("1");
- assertEquals(1, countRowsInScan(getScanner("1", "", "2", "", predicate)));
-
- // All the 2nd tablet.
- predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
- predicate.setLowerBound("1");
- predicate.setUpperBound("3");
- assertEquals(3, countRowsInScan(getScanner("1", "", "2", "", predicate)));
-
- // Value that doesn't exist.
- predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
- predicate.setLowerBound("4");
- assertEquals(0, countRowsInScan(getScanner("1", "", "2", "", predicate)));
-
- // First row from every tablet.
- predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
- predicate.setLowerBound("1");
- predicate.setUpperBound("1");
- assertEquals(3, countRowsInScan(getScanner(null, null, null, null, predicate)));
-
- // All the rows.
- predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
- predicate.setLowerBound("1");
- assertEquals(9, countRowsInScan(getScanner(null, null, null, null, predicate)));
- }
-
- @Test(timeout = 100000)
- public void testProjections() throws Exception {
- // Test with column names.
- AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
- builder.setProjectedColumnNames(Lists.newArrayList(schema.getColumnByIndex(0).getName(),
- schema.getColumnByIndex(1).getName()));
- buildScannerAndCheckColumnsCount(builder, 2);
-
- // Test with column indexes.
- builder = client.newScannerBuilder(table);
- builder.setProjectedColumnIndexes(Lists.newArrayList(0, 1));
- buildScannerAndCheckColumnsCount(builder, 2);
-
- // Test with column names overriding indexes.
- builder = client.newScannerBuilder(table);
- builder.setProjectedColumnIndexes(Lists.newArrayList(0, 1));
- builder.setProjectedColumnNames(Lists.newArrayList(schema.getColumnByIndex(0).getName()));
- buildScannerAndCheckColumnsCount(builder, 1);
-
- // Test with keys last with indexes.
- builder = client.newScannerBuilder(table);
- builder.setProjectedColumnIndexes(Lists.newArrayList(2, 1, 0));
- buildScannerAndCheckColumnsCount(builder, 3);
-
- // Test with keys last with column names.
- builder = client.newScannerBuilder(table);
- builder.setProjectedColumnNames(Lists.newArrayList(schema.getColumnByIndex(2).getName(),
- schema.getColumnByIndex(0).getName()));
- buildScannerAndCheckColumnsCount(builder, 2);
- }
-
- private AsyncKuduScanner getScanner(String lowerBoundKeyOne,
- String lowerBoundKeyTwo,
- String exclusiveUpperBoundKeyOne,
- String exclusiveUpperBoundKeyTwo) {
- return getScanner(lowerBoundKeyOne, lowerBoundKeyTwo,
- exclusiveUpperBoundKeyOne, exclusiveUpperBoundKeyTwo, null);
- }
-
- private AsyncKuduScanner getScanner(String lowerBoundKeyOne,
- String lowerBoundKeyTwo,
- String exclusiveUpperBoundKeyOne,
- String exclusiveUpperBoundKeyTwo,
- ColumnRangePredicate predicate) {
- AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
-
- if (lowerBoundKeyOne != null) {
- PartialRow lowerBoundRow = schema.newPartialRow();
- lowerBoundRow.addString(0, lowerBoundKeyOne);
- lowerBoundRow.addString(1, lowerBoundKeyTwo);
- builder.lowerBound(lowerBoundRow);
- }
-
- if (exclusiveUpperBoundKeyOne != null) {
- PartialRow upperBoundRow = schema.newPartialRow();
- upperBoundRow.addString(0, exclusiveUpperBoundKeyOne);
- upperBoundRow.addString(1, exclusiveUpperBoundKeyTwo);
- builder.exclusiveUpperBound(upperBoundRow);
- }
-
- if (predicate != null) {
- builder.addColumnRangePredicate(predicate);
- }
-
- return builder.build();
- }
-
- private void buildScannerAndCheckColumnsCount(AsyncKuduScanner.AsyncKuduScannerBuilder builder,
- int count) throws Exception {
- AsyncKuduScanner scanner = builder.build();
- scanner.nextRows().join(DEFAULT_SLEEP);
- RowResultIterator rri = scanner.nextRows().join(DEFAULT_SLEEP);
- assertEquals(count, rri.next().getSchema().getColumns().size());
- }
-
- private static Schema getSchema() {
- ArrayList<ColumnSchema> columns = new ArrayList<>(3);
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key1", STRING)
- .key(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("key2", STRING)
- .key(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder("val", STRING)
- .nullable(true) // Important because we need to make sure it gets passed in projections
- .build());
- return new Schema(columns);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestStatistics.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestStatistics.java b/java/kudu-client/src/test/java/org/kududb/client/TestStatistics.java
deleted file mode 100644
index 6cfbee7..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestStatistics.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.kududb.client.Statistics.Statistic;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestStatistics extends BaseKuduTest {
-
- private static final String TABLE_NAME = TestStatistics.class.getName() + "-"
- + System.currentTimeMillis();
- private static KuduTable table;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- BaseKuduTest.setUpBeforeClass();
- CreateTableOptions options = getBasicCreateTableOptions().setNumReplicas(1);
- table = createTable(TABLE_NAME, basicSchema, options);
- }
-
- @Test(timeout = 10000)
- public void test() throws Exception {
- KuduSession session = syncClient.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- int rowCount = 20;
- for (int i = 0; i < rowCount; i++) {
- Insert insert = createBasicSchemaInsert(table, i);
- session.apply(insert);
- if (i % 2 == 1) {
- session.flush();
- }
- }
- Statistics statistics = syncClient.getStatistics();
- assertEquals(rowCount / 2, statistics.getClientStatistic(Statistic.WRITE_RPCS));
- assertEquals(rowCount, statistics.getClientStatistic(Statistic.WRITE_OPS));
- assertEquals(0, statistics.getClientStatistic(Statistic.RPC_ERRORS));
- assertEquals(0, statistics.getClientStatistic(Statistic.OPS_ERRORS));
-
- // Use default flush mode.
- session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
- // Insert duplicate rows, expect to get ALREADY_PRESENT error.
- long byteSize = 0;
- for (int i = 0; i < rowCount; i++) {
- Insert insert = createBasicSchemaInsert(table, i);
- session.apply(insert);
- byteSize += insert.getRowOperationSizeBytes();
- }
- assertEquals(rowCount + rowCount / 2, statistics.getClientStatistic(Statistic.WRITE_RPCS));
- assertEquals(rowCount, statistics.getClientStatistic(Statistic.WRITE_OPS));
- assertEquals(0, statistics.getClientStatistic(Statistic.RPC_ERRORS));
- assertEquals(rowCount, statistics.getClientStatistic(Statistic.OPS_ERRORS));
- assertEquals(byteSize * 2, statistics.getClientStatistic(Statistic.BYTES_WRITTEN));
-
- assertEquals(1, statistics.getTableSet().size());
- assertEquals(1, statistics.getTabletSet().size());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestStatus.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestStatus.java b/java/kudu-client/src/test/java/org/kududb/client/TestStatus.java
deleted file mode 100644
index 11bd23b..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestStatus.java
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.Test;
-import org.kududb.client.Status;
-
-import static org.junit.Assert.*;
-
-public class TestStatus {
-
- @Test
- public void testOKStatus() {
- Status s = Status.OK();
- assertTrue(s.ok());
- assertFalse(s.isNotAuthorized());
- assertEquals(-1, s.getPosixCode());
- assertEquals("OK", s.toString());
- }
-
- @Test
- public void testStatusNonPosix() {
- Status s = Status.Aborted("foo");
- assertFalse(s.ok());
- assertTrue(s.isAborted());
- assertEquals("ABORTED", s.getCodeName());
- assertEquals("foo", s.getMessage());
- assertEquals(-1, s.getPosixCode());
- assertEquals("Aborted: foo", s.toString());
- }
-
- @Test
- public void testPosixCode() {
- Status s = Status.NotFound("File not found", 2);
- assertFalse(s.ok());
- assertFalse(s.isAborted());
- assertTrue(s.isNotFound());
- assertEquals(2, s.getPosixCode());
- assertEquals("Not found: File not found (error 2)", s.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestTestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestTestUtils.java b/java/kudu-client/src/test/java/org/kududb/client/TestTestUtils.java
deleted file mode 100644
index b150f8e..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestTestUtils.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.junit.After;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests for non-trivial helper methods in TestUtils.
- */
-public class TestTestUtils {
-
- public static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
-
- private Process proc;
-
- @After
- public void tearDown() {
- if (proc != null) {
- proc.destroy();
- }
- }
-
- /**
- * Starts a process that executes the "yes" command (which prints 'y' in a loop),
- * sends a SIGSTOP to the process, and ensures that SIGSTOP does indeed pause the process.
- * Afterwards, sends a SIGCONT to the process and ensures that the process resumes.
- */
- @Test(timeout = 2000)
- public void testPauseAndResume() throws Exception {
- ProcessBuilder processBuilder = new ProcessBuilder("yes");
- proc = processBuilder.start();
- LineCounterRunnable lineCounter = new LineCounterRunnable(proc.getInputStream());
- Thread thread = new Thread(lineCounter);
- thread.setDaemon(true);
- thread.start();
- TestUtils.pauseProcess(proc);
- long prevCount;
- do {
- prevCount = lineCounter.getCount();
- Thread.sleep(10);
- } while (prevCount != lineCounter.getCount());
- assertEquals(prevCount, lineCounter.getCount());
- TestUtils.resumeProcess(proc);
- do {
- prevCount = lineCounter.getCount();
- Thread.sleep(10);
- } while (prevCount == lineCounter.getCount());
- assertTrue(lineCounter.getCount() > prevCount);
- }
-
- /**
- * Counts the number of lines in a specified input stream.
- */
- static class LineCounterRunnable implements Runnable {
- private final AtomicLong counter;
- private final InputStream is;
-
- public LineCounterRunnable(InputStream is) {
- this.is = is;
- counter = new AtomicLong(0);
- }
-
- @Override
- public void run() {
- BufferedReader in = null;
- try {
- in = new BufferedReader(new InputStreamReader(is));
- while (in.readLine() != null) {
- counter.incrementAndGet();
- }
- } catch (Exception e) {
- LOG.error("Error while reading from the process", e);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- LOG.error("Error closing the stream", e);
- }
- }
- }
- }
-
- public long getCount() {
- return counter.get();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestTimeouts.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestTimeouts.java b/java/kudu-client/src/test/java/org/kududb/client/TestTimeouts.java
deleted file mode 100644
index 3e78918..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestTimeouts.java
+++ /dev/null
@@ -1,68 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import com.stumbleupon.async.TimeoutException;
-import org.junit.Test;
-
-public class TestTimeouts extends BaseKuduTest {
-
- private static final String TABLE_NAME =
- TestTimeouts.class.getName() + "-" + System.currentTimeMillis();
-
- /**
- * This test case tries different methods that should all timeout, while relying on the client to
- * pass down the timeouts to the session and scanner.
- */
- @Test(timeout = 100000)
- public void testLowTimeouts() throws Exception {
- KuduClient lowTimeoutsClient = new KuduClient.KuduClientBuilder(masterAddresses)
- .defaultAdminOperationTimeoutMs(1)
- .defaultOperationTimeoutMs(1)
- .build();
-
- try {
- lowTimeoutsClient.listTabletServers();
- fail("Should have timed out");
- } catch (KuduException ex) {
- // Expected.
- }
-
- createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions());
- KuduTable table = openTable(TABLE_NAME);
-
- KuduSession lowTimeoutSession = lowTimeoutsClient.newSession();
-
- try {
- lowTimeoutSession.apply(createBasicSchemaInsert(table, 1));
- fail("Should have timed out");
- } catch (KuduException ex) {
- assertTrue(ex.getStatus().isTimedOut());
- }
-
- KuduScanner lowTimeoutScanner = lowTimeoutsClient.newScannerBuilder(table).build();
- try {
- lowTimeoutScanner.nextRows();
- fail("Should have timed out");
- } catch (KuduException ex) {
- assertTrue(ex.getStatus().isTimedOut());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java b/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
deleted file mode 100644
index 392bed9..0000000
--- a/java/kudu-client/src/test/java/org/kududb/client/TestUtils.java
+++ /dev/null
@@ -1,289 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
-import com.sun.security.auth.module.UnixSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import sun.management.VMManagement;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.util.List;
-import java.util.Set;
-
-/**
- * A grouping of methods that help unit testing.
- */
-public class TestUtils {
- private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
-
- // Used by pidOfProcess()
- private static String UNIX_PROCESS_CLS_NAME = "java.lang.UNIXProcess";
- private static Set<String> VALID_SIGNALS = ImmutableSet.of("STOP", "CONT", "TERM", "KILL");
-
- private static final String BIN_DIR_PROP = "binDir";
-
- /**
- * @return the path of the flags file to pass to daemon processes
- * started by the tests
- */
- public static String getFlagsPath() {
- URL u = BaseKuduTest.class.getResource("/flags");
- if (u == null) {
- throw new RuntimeException("Unable to find 'flags' file");
- }
- if (u.getProtocol() == "file") {
- return urlToPath(u);
- }
- // If the flags are inside a JAR, extract them into our temporary
- // test directory.
- try {
- // Somewhat unintuitively, createTempFile() actually creates the file,
- // not just the path, so we have to use REPLACE_EXISTING below.
- Path tmpFile = Files.createTempFile(
- Paths.get(getBaseDir()), "kudu-flags", ".flags");
- Files.copy(BaseKuduTest.class.getResourceAsStream("/flags"), tmpFile,
- StandardCopyOption.REPLACE_EXISTING);
- return tmpFile.toAbsolutePath().toString();
- } catch (IOException e) {
- throw new RuntimeException("Unable to extract flags file into tmp", e);
- }
- }
-
- /**
- * Return the path portion of a file URL, after decoding the escaped
- * components. This fixes issues when trying to build within a
- * working directory with special characters.
- */
- private static String urlToPath(URL u) {
- try {
- return URLDecoder.decode(u.getPath(), "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static String findBuildDir() {
- URL myUrl = BaseKuduTest.class.getProtectionDomain().getCodeSource().getLocation();
- File myPath = new File(urlToPath(myUrl));
- while (myPath != null) {
- if (new File(myPath, ".git").isDirectory()) {
- return new File(myPath, "build/latest/bin").getAbsolutePath();
- }
- myPath = myPath.getParentFile();
- }
- LOG.warn("Unable to find build dir! myUrl={}", myUrl);
- return null;
- }
-
- /**
- * @param binName the binary to look for (eg 'kudu-tserver')
- * @return the absolute path of that binary
- * @throws FileNotFoundException if no such binary is found
- */
- public static String findBinary(String binName) throws FileNotFoundException {
- String binDir = System.getProperty(BIN_DIR_PROP);
- if (binDir != null) {
- LOG.info("Using binary directory specified by property: {}",
- binDir);
- } else {
- binDir = findBuildDir();
- }
-
- File candidate = new File(binDir, binName);
- if (candidate.canExecute()) {
- return candidate.getAbsolutePath();
- }
- throw new FileNotFoundException("Cannot find binary " + binName +
- " in binary directory " + binDir);
- }
-
- /**
- * @return the base directory within which we will store server data
- */
- public static String getBaseDir() {
- String s = System.getenv("TEST_TMPDIR");
- if (s == null) {
- s = String.format("/tmp/kudutest-%d", new UnixSystem().getUid());
- }
- File f = new File(s);
- f.mkdirs();
- return f.getAbsolutePath();
- }
-
- /**
- * Finds the next free port, starting with the one passed. Keep in mind the
- * time-of-check-time-of-use nature of this method, the returned port might become occupied
- * after it was checked for availability.
- * @param startPort first port to be probed
- * @return a currently usable port
- * @throws IOException IOE is thrown if we can't close a socket we tried to open or if we run
- * out of ports to try
- */
- public static int findFreePort(int startPort) throws IOException {
- ServerSocket ss;
- for(int i = startPort; i < 65536; i++) {
- try {
- ss = new ServerSocket();
- SocketAddress address = new InetSocketAddress(getUniqueLocalhost(), i);
- ss.bind(address);
- } catch (IOException e) {
- continue;
- }
- ss.close();
- return i;
- }
- throw new IOException("Ran out of ports.");
- }
-
- /**
- * Finds a specified number of parts, starting with one passed. Keep in mind the
- * time-of-check-time-of-use nature of this method.
- * @see {@link #findFreePort(int)}
- * @param startPort First port to be probed.
- * @param numPorts Number of ports to reserve.
- * @return A list of currently usable ports.
- * @throws IOException IOE Is thrown if we can't close a socket we tried to open or if run
- * out of ports to try.
- */
- public static List<Integer> findFreePorts(int startPort, int numPorts) throws IOException {
- List<Integer> ports = Lists.newArrayListWithCapacity(numPorts);
- for (int i = 0; i < numPorts; i++) {
- startPort = findFreePort(startPort);
- ports.add(startPort++);
- }
- return ports;
- }
-
- /**
- * Gets the pid of a specified process. Relies on reflection and only works on
- * UNIX process, not guaranteed to work on JDKs other than Oracle and OpenJDK.
- * @param proc The specified process.
- * @return The process UNIX pid.
- * @throws IllegalArgumentException If the process is not a UNIXProcess.
- * @throws Exception If there are other getting the pid via reflection.
- */
- static int pidOfProcess(Process proc) throws Exception {
- Class<?> procCls = proc.getClass();
- if (!procCls.getName().equals(UNIX_PROCESS_CLS_NAME)) {
- throw new IllegalArgumentException("stopProcess() expects objects of class " +
- UNIX_PROCESS_CLS_NAME + ", but " + procCls.getName() + " was passed in instead!");
- }
- Field pidField = procCls.getDeclaredField("pid");
- pidField.setAccessible(true);
- return (Integer) pidField.get(proc);
- }
-
- /**
- * Send a code specified by its string representation to the specified process.
- * TODO: Use a JNR/JNR-Posix instead of forking the JVM to exec "kill".
- * @param proc The specified process.
- * @param sig The string representation of the process (e.g., STOP for SIGSTOP).
- * @throws IllegalArgumentException If the signal type is not supported.
- * @throws IllegalStateException If we are unable to send the specified signal.
- */
- static void signalProcess(Process proc, String sig) throws Exception {
- if (!VALID_SIGNALS.contains(sig)) {
- throw new IllegalArgumentException(sig + " is not a supported signal, only " +
- Joiner.on(",").join(VALID_SIGNALS) + " are supported");
- }
- int pid = pidOfProcess(proc);
- int rv = Runtime.getRuntime()
- .exec(String.format("kill -%s %d", sig, pid))
- .waitFor();
- if (rv != 0) {
- throw new IllegalStateException(String.format("unable to send SIG%s to process %s(pid=%d): " +
- "expected return code from kill, but got %d instead", sig, proc, pid, rv));
- }
- }
-
- /**
- * Pause the specified process by sending a SIGSTOP using the kill command.
- * @param proc The specified process.
- * @throws Exception If error prevents us from pausing the process.
- */
- static void pauseProcess(Process proc) throws Exception {
- signalProcess(proc, "STOP");
- }
-
- /**
- * Resumes the specified process by sending a SIGCONT using the kill command.
- * @param proc The specified process.
- * @throws Exception If error prevents us from resuming the process.
- */
- static void resumeProcess(Process proc) throws Exception {
- signalProcess(proc, "CONT");
- }
-
- /**
- * This is used to generate unique loopback IPs for parallel test running.
- * @return the local PID of this process
- */
- static int getPid() {
- try {
- RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
- java.lang.reflect.Field jvm = runtime.getClass().getDeclaredField("jvm");
- jvm.setAccessible(true);
- VMManagement mgmt = (VMManagement)jvm.get(runtime);
- Method pid_method = mgmt.getClass().getDeclaredMethod("getProcessId");
- pid_method.setAccessible(true);
-
- return (Integer)pid_method.invoke(mgmt);
- } catch (Exception e) {
- LOG.warn("Cannot get PID", e);
- return 1;
- }
- }
-
- /**
- * The generated IP is based on pid, so this requires that the parallel tests
- * run in separate VMs.
- *
- * On OSX, the above trick doesn't work, so we can't run parallel tests on OSX.
- * Given that, we just return the normal localhost IP.
- *
- * @return a unique loopback IP address for this PID. This allows running
- * tests in parallel, since 127.0.0.0/8 all act as loopbacks on Linux.
- */
- static String getUniqueLocalhost() {
- if ("Mac OS X".equals(System.getProperty("os.name"))) {
- return "127.0.0.1";
- }
-
- int pid = getPid();
- return "127." + ((pid & 0xff00) >> 8) + "." + (pid & 0xff) + ".1";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/util/TestAsyncUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/util/TestAsyncUtil.java b/java/kudu-client/src/test/java/org/kududb/util/TestAsyncUtil.java
deleted file mode 100644
index fce7ddc..0000000
--- a/java/kudu-client/src/test/java/org/kududb/util/TestAsyncUtil.java
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.util;
-
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test for {@link AsyncUtil}.
- */
-public class TestAsyncUtil {
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- @Test
- public void testAddCallbacksDeferring() throws Exception {
- Deferred<String> d = new Deferred<String>();
- TestCallback cb = new TestCallback();
- TestErrback eb = new TestErrback();
-
- // Test normal callbacks.
- AsyncUtil.addCallbacksDeferring(d, cb, eb);
- final String testStr = "hello world";
- d.callback(testStr);
- assertEquals(d.join(), "callback: " + testStr);
-
- d = new Deferred<String>();
- AsyncUtil.addCallbacksDeferring(d, cb, eb);
- d.callback(new IllegalArgumentException());
- assertEquals(d.join(), "illegal arg");
-
- d = new Deferred<String>();
- AsyncUtil.addCallbacksDeferring(d, cb, eb);
- d.callback(new IllegalStateException());
- exception.expect(IllegalStateException.class);
- d.join();
- }
-
- final class TestCallback implements Callback<Deferred<String>, String> {
- @Override
- public Deferred<String> call(String arg) throws Exception {
- return Deferred.fromResult("callback: " + arg);
- }
- }
-
- final class TestErrback implements Callback<Deferred<String>, Exception> {
- @Override
- public Deferred<String> call(Exception arg) {
- if (arg instanceof IllegalArgumentException) {
- return Deferred.fromResult("illegal arg");
- }
- return Deferred.fromError(arg);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/util/TestMurmurHash.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/util/TestMurmurHash.java b/java/kudu-client/src/test/java/org/kududb/util/TestMurmurHash.java
deleted file mode 100644
index 051107c..0000000
--- a/java/kudu-client/src/test/java/org/kududb/util/TestMurmurHash.java
+++ /dev/null
@@ -1,46 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.util;
-
-import com.google.common.primitives.UnsignedLongs;
-import com.sangupta.murmur.Murmur2;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test Murmur2 Hash64 returns the expected values for inputs.
- *
- * These tests are duplicated on the C++ side to ensure that hash computations
- * are stable across both platforms.
- */
-public class TestMurmurHash {
-
- @Test
- public void testMurmur2Hash64() throws Exception {
- long hash;
-
- hash = Murmur2.hash64("ab".getBytes("UTF-8"), 2, 0);
- assertEquals(UnsignedLongs.parseUnsignedLong("7115271465109541368"), hash);
-
- hash = Murmur2.hash64("abcdefg".getBytes("UTF-8"), 7, 0);
- assertEquals(UnsignedLongs.parseUnsignedLong("2601573339036254301"), hash);
-
- hash = Murmur2.hash64("quick brown fox".getBytes("UTF-8"), 15, 42);
- assertEquals(UnsignedLongs.parseUnsignedLong("3575930248840144026"), hash);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/test/java/org/kududb/util/TestNetUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/util/TestNetUtil.java b/java/kudu-client/src/test/java/org/kududb/util/TestNetUtil.java
deleted file mode 100644
index 5c003ae..0000000
--- a/java/kudu-client/src/test/java/org/kududb/util/TestNetUtil.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.util;
-
-import com.google.common.net.HostAndPort;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test for {@link NetUtil}.
- */
-public class TestNetUtil {
-
- /**
- * Tests parsing strings into {@link HostAndPort} objects with and without specifying
- * the port in the string.
- */
- @Test
- public void testParseString() {
- String aStringWithPort = "1.2.3.4:1234";
- HostAndPort hostAndPortForAStringWithPort = NetUtil.parseString(aStringWithPort, 0);
- assertEquals(hostAndPortForAStringWithPort.getHostText(), "1.2.3.4");
- assertEquals(hostAndPortForAStringWithPort.getPort(), 1234);
-
- String aStringWithoutPort = "1.2.3.4";
- HostAndPort hostAndPortForAStringWithoutPort = NetUtil.parseString(aStringWithoutPort, 12345);
- assertEquals(hostAndPortForAStringWithoutPort.getHostText(), aStringWithoutPort);
- assertEquals(hostAndPortForAStringWithoutPort.getPort(), 12345);
- }
-
- /**
- * Tests parsing comma separated list of "host:port" pairs and hosts into a list of
- * {@link HostAndPort} objects.
- */
- @Test
- public void testParseStrings() {
- String testAddrs = "1.2.3.4.5,10.0.0.1:5555,127.0.0.1:7777";
- List<HostAndPort> hostsAndPorts = NetUtil.parseStrings(testAddrs, 3333);
- assertArrayEquals(hostsAndPorts.toArray(),
- new HostAndPort[] { HostAndPort.fromParts("1.2.3.4.5", 3333),
- HostAndPort.fromParts("10.0.0.1", 5555),
- HostAndPort.fromParts("127.0.0.1", 7777) }
- );
- }
-
- @Test
- public void testHostsAndPortsToString() {
- List<HostAndPort> hostsAndPorts = Arrays.asList(
- HostAndPort.fromParts("127.0.0.1", 1111),
- HostAndPort.fromParts("1.2.3.4.5", 0)
- );
- assertEquals(NetUtil.hostsAndPortsToString(hostsAndPorts), "127.0.0.1:1111,1.2.3.4.5:0");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java
new file mode 100644
index 0000000..7166300
--- /dev/null
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduEventProducer.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.kududb.flume.sink;
+
+import org.apache.flume.Event;
+import org.apache.flume.conf.Configurable;
+import org.apache.flume.conf.ConfigurableComponent;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.KuduTable;
+import org.kududb.client.Operation;
+
+import java.util.List;
+
+/**
+ * Interface for an event producer which produces Kudu Operations to write
+ * the headers and body of an event in a Kudu table. This is configurable,
+ * so any config params required should be taken through this. The columns
+ * should exist in the table specified in the configuration for the KuduSink.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface KuduEventProducer extends Configurable, ConfigurableComponent {
+ /**
+ * Initialize the event producer.
+ * @param event to be written to Kudu
+ * @param table the KuduTable object used for creating Kudu Operation objects
+ */
+ void initialize(Event event, KuduTable table);
+
+ /**
+ * Get the operations that should be written out to Kudu as a result of this
+ * event. This list is written to Kudu using the Kudu client API.
+ * @return List of {@link org.kududb.client.Operation} which
+ * are written as such to Kudu
+ */
+ List<Operation> getOperations();
+
+ /*
+ * Clean up any state. This will be called when the sink is being stopped.
+ */
+ void close();
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
new file mode 100644
index 0000000..080cda2
--- /dev/null
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSink.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.kududb.flume.sink;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import org.apache.flume.Channel;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.Transaction;
+import org.apache.flume.conf.Configurable;
+import org.apache.flume.instrumentation.SinkCounter;
+import org.apache.flume.sink.AbstractSink;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.client.AsyncKuduClient;
+import org.kududb.client.KuduClient;
+import org.kududb.client.KuduSession;
+import org.kududb.client.KuduTable;
+import org.kududb.client.Operation;
+import org.kududb.client.OperationResponse;
+import org.kududb.client.SessionConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * <p>A Flume sink that reads events from a channel and writes them to a Kudu table.
+ *
+ * <p><strong>Flume Kudu Sink configuration parameters</strong>
+ *
+ * <table cellpadding=3 cellspacing=0 border=1>
+ * <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr>
+ * <tr><td>channel</td><td></td><td>Yes</td><td>The name of the Flume channel to read from.</td></tr>
+ * <tr><td>type</td><td></td><td>Yes</td><td>Component name. Must be {@code org.kududb.flume.sink.KuduSink}</td></tr>
+ * <tr><td>masterAddresses</td><td></td><td>Yes</td><td>Comma-separated list of "host:port" pairs of the Kudu master servers. The port is optional.</td></tr>
+ * <tr><td>tableName</td><td></td><td>Yes</td><td>The name of the Kudu table to write to.</td></tr>
+ * <tr><td>batchSize</td><td>100</td><td>No</td><td>The maximum number of events the sink will attempt to take from the channel per transaction.</td></tr>
+ * <tr><td>ignoreDuplicateRows</td><td>true</td><td>No</td><td>Whether to ignore errors indicating that we attempted to insert duplicate rows into Kudu.</td></tr>
+ * <tr><td>timeoutMillis</td><td>10000</td><td>No</td><td>Timeout period for Kudu write operations, in milliseconds.</td></tr>
+ * <tr><td>producer</td><td>{@link org.kududb.flume.sink.SimpleKuduEventProducer}</td><td>No</td><td>The fully qualified class name of the {@link KuduEventProducer} the sink should use.</td></tr>
+ * <tr><td>producer.*</td><td></td><td>(Varies by event producer)</td><td>Configuration properties to pass to the event producer implementation.</td></tr>
+ * </table>
+ *
+ * <p><strong>Installation</strong>
+ *
+ * <p>After building the sink, in order to use it with Flume, place the file named
+ * <tt>kudu-flume-sink-<em>VERSION</em>-jar-with-dependencies.jar</tt> in the
+ * Flume <tt>plugins.d</tt> directory under <tt>kudu-flume-sink/lib/</tt>.
+ *
+ * <p>For detailed instructions on using Flume's plugins.d mechanism, please see the plugins.d
+ * section of the <a href="https://flume.apache.org/FlumeUserGuide.html#the-plugins-d-directory">Flume User Guide</a>.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduSink extends AbstractSink implements Configurable {
+ private static final Logger logger = LoggerFactory.getLogger(KuduSink.class);
+ private static final Long DEFAULT_BATCH_SIZE = 100L;
+ private static final Long DEFAULT_TIMEOUT_MILLIS =
+ AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
+ private static final String DEFAULT_KUDU_EVENT_PRODUCER =
+ "org.kududb.flume.sink.SimpleKuduEventProducer";
+ private static final boolean DEFAULT_IGNORE_DUPLICATE_ROWS = true;
+
+ private String masterAddresses;
+ private String tableName;
+ private long batchSize;
+ private long timeoutMillis;
+ private boolean ignoreDuplicateRows;
+ private KuduTable table;
+ private KuduSession session;
+ private KuduClient client;
+ private KuduEventProducer eventProducer;
+ private String eventProducerType;
+ private Context producerContext;
+ private SinkCounter sinkCounter;
+
+ public KuduSink() {
+ this(null);
+ }
+
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ public KuduSink(KuduClient kuduClient) {
+ this.client = kuduClient;
+ }
+
+ @Override
+ public void start() {
+ Preconditions.checkState(table == null && session == null, "Please call stop " +
+ "before calling start on an old instance.");
+
+ // This is not null only inside tests
+ if (client == null) {
+ client = new KuduClient.KuduClientBuilder(masterAddresses).build();
+ }
+ session = client.newSession();
+ session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
+ session.setTimeoutMillis(timeoutMillis);
+ session.setIgnoreAllDuplicateRows(ignoreDuplicateRows);
+
+ try {
+ table = client.openTable(tableName);
+ } catch (Exception e) {
+ sinkCounter.incrementConnectionFailedCount();
+ String msg = String.format("Could not open table '%s' from Kudu", tableName);
+ logger.error(msg, e);
+ throw new FlumeException(msg, e);
+ }
+
+ super.start();
+ sinkCounter.incrementConnectionCreatedCount();
+ sinkCounter.start();
+ }
+
+ @Override
+ public void stop() {
+ try {
+ if (client != null) {
+ client.shutdown();
+ }
+ client = null;
+ table = null;
+ session = null;
+ } catch (Exception e) {
+ throw new FlumeException("Error closing client.", e);
+ }
+ sinkCounter.incrementConnectionClosedCount();
+ sinkCounter.stop();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void configure(Context context) {
+ masterAddresses = context.getString(KuduSinkConfigurationConstants.MASTER_ADDRESSES);
+ tableName = context.getString(KuduSinkConfigurationConstants.TABLE_NAME);
+
+ batchSize = context.getLong(
+ KuduSinkConfigurationConstants.BATCH_SIZE, DEFAULT_BATCH_SIZE);
+ timeoutMillis = context.getLong(
+ KuduSinkConfigurationConstants.TIMEOUT_MILLIS, DEFAULT_TIMEOUT_MILLIS);
+ ignoreDuplicateRows = context.getBoolean(
+ KuduSinkConfigurationConstants.IGNORE_DUPLICATE_ROWS, DEFAULT_IGNORE_DUPLICATE_ROWS);
+ eventProducerType = context.getString(KuduSinkConfigurationConstants.PRODUCER);
+
+ Preconditions.checkNotNull(masterAddresses,
+ "Master address cannot be empty, please specify '" +
+ KuduSinkConfigurationConstants.MASTER_ADDRESSES +
+ "' in configuration file");
+ Preconditions.checkNotNull(tableName,
+ "Table name cannot be empty, please specify '" +
+ KuduSinkConfigurationConstants.TABLE_NAME +
+ "' in configuration file");
+
+ // Check for event producer, if null set event producer type.
+ if (eventProducerType == null || eventProducerType.isEmpty()) {
+ eventProducerType = DEFAULT_KUDU_EVENT_PRODUCER;
+ logger.info("No Kudu event producer defined, will use default");
+ }
+
+ producerContext = new Context();
+ producerContext.putAll(context.getSubProperties(
+ KuduSinkConfigurationConstants.PRODUCER_PREFIX));
+
+ try {
+ Class<? extends KuduEventProducer> clazz =
+ (Class<? extends KuduEventProducer>)
+ Class.forName(eventProducerType);
+ eventProducer = clazz.newInstance();
+ eventProducer.configure(producerContext);
+ } catch (Exception e) {
+ logger.error("Could not instantiate Kudu event producer." , e);
+ Throwables.propagate(e);
+ }
+ sinkCounter = new SinkCounter(this.getName());
+ }
+
+ public KuduClient getClient() {
+ return client;
+ }
+
+ @Override
+ public Status process() throws EventDeliveryException {
+ if (session.hasPendingOperations()) {
+ // If for whatever reason we have pending operations then just refuse to process
+ // and tell caller to try again a bit later. We don't want to pile on the kudu
+ // session object.
+ return Status.BACKOFF;
+ }
+
+ Channel channel = getChannel();
+ Transaction txn = channel.getTransaction();
+
+ txn.begin();
+
+ try {
+ long txnEventCount = 0;
+ for (; txnEventCount < batchSize; txnEventCount++) {
+ Event event = channel.take();
+ if (event == null) {
+ break;
+ }
+
+ eventProducer.initialize(event, table);
+ List<Operation> operations = eventProducer.getOperations();
+ for (Operation o : operations) {
+ session.apply(o);
+ }
+ }
+
+ logger.debug("Flushing {} events", txnEventCount);
+ List<OperationResponse> responses = session.flush();
+ if (responses != null) {
+ for (OperationResponse response : responses) {
+ // Throw an EventDeliveryException if at least one of the responses was
+ // a row error. Row errors can occur for example when an event is inserted
+ // into Kudu successfully but the Flume transaction is rolled back for some reason,
+ // and a subsequent replay of the same Flume transaction leads to a
+ // duplicate key error since the row already exists in Kudu.
+ // (Kudu doesn't support "insert or overwrite" semantics yet.)
+ // Note: Duplicate keys will not be reported as errors if ignoreDuplicateRows
+ // is enabled in the config.
+ if (response.hasRowError()) {
+ throw new EventDeliveryException("Failed to flush one or more changes. " +
+ "Transaction rolled back: " + response.getRowError().toString());
+ }
+ }
+ }
+
+ if (txnEventCount == 0) {
+ sinkCounter.incrementBatchEmptyCount();
+ } else if (txnEventCount == batchSize) {
+ sinkCounter.incrementBatchCompleteCount();
+ } else {
+ sinkCounter.incrementBatchUnderflowCount();
+ }
+
+ txn.commit();
+
+ if (txnEventCount == 0) {
+ return Status.BACKOFF;
+ }
+
+ sinkCounter.addToEventDrainSuccessCount(txnEventCount);
+ return Status.READY;
+
+ } catch (Throwable e) {
+ txn.rollback();
+
+ String msg = "Failed to commit transaction. Transaction rolled back.";
+ logger.error(msg, e);
+ if (e instanceof Error || e instanceof RuntimeException) {
+ Throwables.propagate(e);
+ } else {
+ logger.error(msg, e);
+ throw new EventDeliveryException(msg, e);
+ }
+ } finally {
+ txn.close();
+ }
+
+ return Status.BACKOFF;
+ }
+
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ KuduEventProducer getEventProducer() {
+ return eventProducer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java
new file mode 100644
index 0000000..6486137
--- /dev/null
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/KuduSinkConfigurationConstants.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.kududb.flume.sink;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+
+/**
+ * Constants used for configuration of KuduSink
+ */
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class KuduSinkConfigurationConstants {
+ /**
+ * Comma-separated list of "host:port" pairs of the masters (port optional).
+ */
+ public static final String MASTER_ADDRESSES = "masterAddresses";
+
+ /**
+ * The name of the table in Kudu to write to.
+ */
+ public static final String TABLE_NAME = "tableName";
+
+ /**
+ * The fully qualified class name of the Kudu event producer the sink should use.
+ */
+ public static final String PRODUCER = "producer";
+
+ /**
+ * Configuration to pass to the Kudu event producer.
+ */
+ public static final String PRODUCER_PREFIX = PRODUCER + ".";
+
+ /**
+ * Maximum number of events the sink should take from the channel per
+ * transaction, if available.
+ */
+ public static final String BATCH_SIZE = "batchSize";
+
+ /**
+ * Timeout period for Kudu operations, in milliseconds.
+ */
+ public static final String TIMEOUT_MILLIS = "timeoutMillis";
+
+ /**
+ * Whether to ignore errors indicating that we attempted to insert duplicate rows into Kudu.
+ */
+ public static final String IGNORE_DUPLICATE_ROWS = "ignoreDuplicateRows";
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java
new file mode 100644
index 0000000..b5be054
--- /dev/null
+++ b/java/kudu-flume-sink/src/main/java/org/apache/kudu/flume/sink/SimpleKuduEventProducer.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.kududb.flume.sink;
+
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.FlumeException;
+import org.apache.flume.conf.ComponentConfiguration;
+import org.kududb.client.Insert;
+import org.kududb.client.KuduTable;
+import org.kududb.client.Operation;
+import org.kududb.client.PartialRow;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * <p>A simple serializer that generates one {@link Insert} per {@link Event} by writing the event
+ * body into a BINARY column. The headers are discarded.
+ *
+ * <p><strong>Simple Kudu Event Producer configuration parameters</strong>
+ *
+ * <table cellpadding=3 cellspacing=0 border=1>
+ * <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr>
+ * <tr><td>producer.payloadColumn</td><td>payload</td><td>No</td><td>The name of the BINARY column to write the Flume the event body to.</td></tr>
+ * </table>
+ */
+public class SimpleKuduEventProducer implements KuduEventProducer {
+ private byte[] payload;
+ private KuduTable table;
+ private String payloadColumn;
+
+ public SimpleKuduEventProducer(){
+ }
+
+ @Override
+ public void configure(Context context) {
+ payloadColumn = context.getString("payloadColumn","payload");
+ }
+
+ @Override
+ public void configure(ComponentConfiguration conf) {
+ }
+
+ @Override
+ public void initialize(Event event, KuduTable table) {
+ this.payload = event.getBody();
+ this.table = table;
+ }
+
+ @Override
+ public List<Operation> getOperations() throws FlumeException {
+ try {
+ Insert insert = table.newInsert();
+ PartialRow row = insert.getRow();
+ row.addBinary(payloadColumn, payload);
+
+ return Collections.singletonList((Operation) insert);
+ } catch (Exception e){
+ throw new FlumeException("Failed to create Kudu Insert object!", e);
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduEventProducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduEventProducer.java b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduEventProducer.java
deleted file mode 100644
index 7166300..0000000
--- a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduEventProducer.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.kududb.flume.sink;
-
-import org.apache.flume.Event;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.conf.ConfigurableComponent;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
-
-import java.util.List;
-
-/**
- * Interface for an event producer which produces Kudu Operations to write
- * the headers and body of an event in a Kudu table. This is configurable,
- * so any config params required should be taken through this. The columns
- * should exist in the table specified in the configuration for the KuduSink.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public interface KuduEventProducer extends Configurable, ConfigurableComponent {
- /**
- * Initialize the event producer.
- * @param event to be written to Kudu
- * @param table the KuduTable object used for creating Kudu Operation objects
- */
- void initialize(Event event, KuduTable table);
-
- /**
- * Get the operations that should be written out to Kudu as a result of this
- * event. This list is written to Kudu using the Kudu client API.
- * @return List of {@link org.kududb.client.Operation} which
- * are written as such to Kudu
- */
- List<Operation> getOperations();
-
- /*
- * Clean up any state. This will be called when the sink is being stopped.
- */
- void close();
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java
deleted file mode 100644
index 080cda2..0000000
--- a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.kududb.flume.sink;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import org.apache.flume.Channel;
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.Transaction;
-import org.apache.flume.conf.Configurable;
-import org.apache.flume.instrumentation.SinkCounter;
-import org.apache.flume.sink.AbstractSink;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.AsyncKuduClient;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduSession;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
-import org.kududb.client.OperationResponse;
-import org.kududb.client.SessionConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * <p>A Flume sink that reads events from a channel and writes them to a Kudu table.
- *
- * <p><strong>Flume Kudu Sink configuration parameters</strong>
- *
- * <table cellpadding=3 cellspacing=0 border=1>
- * <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr>
- * <tr><td>channel</td><td></td><td>Yes</td><td>The name of the Flume channel to read from.</td></tr>
- * <tr><td>type</td><td></td><td>Yes</td><td>Component name. Must be {@code org.kududb.flume.sink.KuduSink}</td></tr>
- * <tr><td>masterAddresses</td><td></td><td>Yes</td><td>Comma-separated list of "host:port" pairs of the Kudu master servers. The port is optional.</td></tr>
- * <tr><td>tableName</td><td></td><td>Yes</td><td>The name of the Kudu table to write to.</td></tr>
- * <tr><td>batchSize</td><td>100</td><td>No</td><td>The maximum number of events the sink will attempt to take from the channel per transaction.</td></tr>
- * <tr><td>ignoreDuplicateRows</td><td>true</td><td>No</td><td>Whether to ignore errors indicating that we attempted to insert duplicate rows into Kudu.</td></tr>
- * <tr><td>timeoutMillis</td><td>10000</td><td>No</td><td>Timeout period for Kudu write operations, in milliseconds.</td></tr>
- * <tr><td>producer</td><td>{@link org.kududb.flume.sink.SimpleKuduEventProducer}</td><td>No</td><td>The fully qualified class name of the {@link KuduEventProducer} the sink should use.</td></tr>
- * <tr><td>producer.*</td><td></td><td>(Varies by event producer)</td><td>Configuration properties to pass to the event producer implementation.</td></tr>
- * </table>
- *
- * <p><strong>Installation</strong>
- *
- * <p>After building the sink, in order to use it with Flume, place the file named
- * <tt>kudu-flume-sink-<em>VERSION</em>-jar-with-dependencies.jar</tt> in the
- * Flume <tt>plugins.d</tt> directory under <tt>kudu-flume-sink/lib/</tt>.
- *
- * <p>For detailed instructions on using Flume's plugins.d mechanism, please see the plugins.d
- * section of the <a href="https://flume.apache.org/FlumeUserGuide.html#the-plugins-d-directory">Flume User Guide</a>.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduSink extends AbstractSink implements Configurable {
- private static final Logger logger = LoggerFactory.getLogger(KuduSink.class);
- private static final Long DEFAULT_BATCH_SIZE = 100L;
- private static final Long DEFAULT_TIMEOUT_MILLIS =
- AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS;
- private static final String DEFAULT_KUDU_EVENT_PRODUCER =
- "org.kududb.flume.sink.SimpleKuduEventProducer";
- private static final boolean DEFAULT_IGNORE_DUPLICATE_ROWS = true;
-
- private String masterAddresses;
- private String tableName;
- private long batchSize;
- private long timeoutMillis;
- private boolean ignoreDuplicateRows;
- private KuduTable table;
- private KuduSession session;
- private KuduClient client;
- private KuduEventProducer eventProducer;
- private String eventProducerType;
- private Context producerContext;
- private SinkCounter sinkCounter;
-
- public KuduSink() {
- this(null);
- }
-
- @VisibleForTesting
- @InterfaceAudience.Private
- public KuduSink(KuduClient kuduClient) {
- this.client = kuduClient;
- }
-
- @Override
- public void start() {
- Preconditions.checkState(table == null && session == null, "Please call stop " +
- "before calling start on an old instance.");
-
- // This is not null only inside tests
- if (client == null) {
- client = new KuduClient.KuduClientBuilder(masterAddresses).build();
- }
- session = client.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- session.setTimeoutMillis(timeoutMillis);
- session.setIgnoreAllDuplicateRows(ignoreDuplicateRows);
-
- try {
- table = client.openTable(tableName);
- } catch (Exception e) {
- sinkCounter.incrementConnectionFailedCount();
- String msg = String.format("Could not open table '%s' from Kudu", tableName);
- logger.error(msg, e);
- throw new FlumeException(msg, e);
- }
-
- super.start();
- sinkCounter.incrementConnectionCreatedCount();
- sinkCounter.start();
- }
-
- @Override
- public void stop() {
- try {
- if (client != null) {
- client.shutdown();
- }
- client = null;
- table = null;
- session = null;
- } catch (Exception e) {
- throw new FlumeException("Error closing client.", e);
- }
- sinkCounter.incrementConnectionClosedCount();
- sinkCounter.stop();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void configure(Context context) {
- masterAddresses = context.getString(KuduSinkConfigurationConstants.MASTER_ADDRESSES);
- tableName = context.getString(KuduSinkConfigurationConstants.TABLE_NAME);
-
- batchSize = context.getLong(
- KuduSinkConfigurationConstants.BATCH_SIZE, DEFAULT_BATCH_SIZE);
- timeoutMillis = context.getLong(
- KuduSinkConfigurationConstants.TIMEOUT_MILLIS, DEFAULT_TIMEOUT_MILLIS);
- ignoreDuplicateRows = context.getBoolean(
- KuduSinkConfigurationConstants.IGNORE_DUPLICATE_ROWS, DEFAULT_IGNORE_DUPLICATE_ROWS);
- eventProducerType = context.getString(KuduSinkConfigurationConstants.PRODUCER);
-
- Preconditions.checkNotNull(masterAddresses,
- "Master address cannot be empty, please specify '" +
- KuduSinkConfigurationConstants.MASTER_ADDRESSES +
- "' in configuration file");
- Preconditions.checkNotNull(tableName,
- "Table name cannot be empty, please specify '" +
- KuduSinkConfigurationConstants.TABLE_NAME +
- "' in configuration file");
-
- // Check for event producer, if null set event producer type.
- if (eventProducerType == null || eventProducerType.isEmpty()) {
- eventProducerType = DEFAULT_KUDU_EVENT_PRODUCER;
- logger.info("No Kudu event producer defined, will use default");
- }
-
- producerContext = new Context();
- producerContext.putAll(context.getSubProperties(
- KuduSinkConfigurationConstants.PRODUCER_PREFIX));
-
- try {
- Class<? extends KuduEventProducer> clazz =
- (Class<? extends KuduEventProducer>)
- Class.forName(eventProducerType);
- eventProducer = clazz.newInstance();
- eventProducer.configure(producerContext);
- } catch (Exception e) {
- logger.error("Could not instantiate Kudu event producer." , e);
- Throwables.propagate(e);
- }
- sinkCounter = new SinkCounter(this.getName());
- }
-
- public KuduClient getClient() {
- return client;
- }
-
- @Override
- public Status process() throws EventDeliveryException {
- if (session.hasPendingOperations()) {
- // If for whatever reason we have pending operations then just refuse to process
- // and tell caller to try again a bit later. We don't want to pile on the kudu
- // session object.
- return Status.BACKOFF;
- }
-
- Channel channel = getChannel();
- Transaction txn = channel.getTransaction();
-
- txn.begin();
-
- try {
- long txnEventCount = 0;
- for (; txnEventCount < batchSize; txnEventCount++) {
- Event event = channel.take();
- if (event == null) {
- break;
- }
-
- eventProducer.initialize(event, table);
- List<Operation> operations = eventProducer.getOperations();
- for (Operation o : operations) {
- session.apply(o);
- }
- }
-
- logger.debug("Flushing {} events", txnEventCount);
- List<OperationResponse> responses = session.flush();
- if (responses != null) {
- for (OperationResponse response : responses) {
- // Throw an EventDeliveryException if at least one of the responses was
- // a row error. Row errors can occur for example when an event is inserted
- // into Kudu successfully but the Flume transaction is rolled back for some reason,
- // and a subsequent replay of the same Flume transaction leads to a
- // duplicate key error since the row already exists in Kudu.
- // (Kudu doesn't support "insert or overwrite" semantics yet.)
- // Note: Duplicate keys will not be reported as errors if ignoreDuplicateRows
- // is enabled in the config.
- if (response.hasRowError()) {
- throw new EventDeliveryException("Failed to flush one or more changes. " +
- "Transaction rolled back: " + response.getRowError().toString());
- }
- }
- }
-
- if (txnEventCount == 0) {
- sinkCounter.incrementBatchEmptyCount();
- } else if (txnEventCount == batchSize) {
- sinkCounter.incrementBatchCompleteCount();
- } else {
- sinkCounter.incrementBatchUnderflowCount();
- }
-
- txn.commit();
-
- if (txnEventCount == 0) {
- return Status.BACKOFF;
- }
-
- sinkCounter.addToEventDrainSuccessCount(txnEventCount);
- return Status.READY;
-
- } catch (Throwable e) {
- txn.rollback();
-
- String msg = "Failed to commit transaction. Transaction rolled back.";
- logger.error(msg, e);
- if (e instanceof Error || e instanceof RuntimeException) {
- Throwables.propagate(e);
- } else {
- logger.error(msg, e);
- throw new EventDeliveryException(msg, e);
- }
- } finally {
- txn.close();
- }
-
- return Status.BACKOFF;
- }
-
- @VisibleForTesting
- @InterfaceAudience.Private
- KuduEventProducer getEventProducer() {
- return eventProducer;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSinkConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSinkConfigurationConstants.java b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSinkConfigurationConstants.java
deleted file mode 100644
index 6486137..0000000
--- a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSinkConfigurationConstants.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.kududb.flume.sink;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Constants used for configuration of KuduSink
- */
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class KuduSinkConfigurationConstants {
- /**
- * Comma-separated list of "host:port" pairs of the masters (port optional).
- */
- public static final String MASTER_ADDRESSES = "masterAddresses";
-
- /**
- * The name of the table in Kudu to write to.
- */
- public static final String TABLE_NAME = "tableName";
-
- /**
- * The fully qualified class name of the Kudu event producer the sink should use.
- */
- public static final String PRODUCER = "producer";
-
- /**
- * Configuration to pass to the Kudu event producer.
- */
- public static final String PRODUCER_PREFIX = PRODUCER + ".";
-
- /**
- * Maximum number of events the sink should take from the channel per
- * transaction, if available.
- */
- public static final String BATCH_SIZE = "batchSize";
-
- /**
- * Timeout period for Kudu operations, in milliseconds.
- */
- public static final String TIMEOUT_MILLIS = "timeoutMillis";
-
- /**
- * Whether to ignore errors indicating that we attempted to insert duplicate rows into Kudu.
- */
- public static final String IGNORE_DUPLICATE_ROWS = "ignoreDuplicateRows";
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java
deleted file mode 100644
index b5be054..0000000
--- a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.kududb.flume.sink;
-
-import org.apache.flume.Context;
-import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
-import org.apache.flume.conf.ComponentConfiguration;
-import org.kududb.client.Insert;
-import org.kududb.client.KuduTable;
-import org.kududb.client.Operation;
-import org.kududb.client.PartialRow;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * <p>A simple serializer that generates one {@link Insert} per {@link Event} by writing the event
- * body into a BINARY column. The headers are discarded.
- *
- * <p><strong>Simple Kudu Event Producer configuration parameters</strong>
- *
- * <table cellpadding=3 cellspacing=0 border=1>
- * <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr>
- * <tr><td>producer.payloadColumn</td><td>payload</td><td>No</td><td>The name of the BINARY column to write the Flume the event body to.</td></tr>
- * </table>
- */
-public class SimpleKuduEventProducer implements KuduEventProducer {
- private byte[] payload;
- private KuduTable table;
- private String payloadColumn;
-
- public SimpleKuduEventProducer(){
- }
-
- @Override
- public void configure(Context context) {
- payloadColumn = context.getString("payloadColumn","payload");
- }
-
- @Override
- public void configure(ComponentConfiguration conf) {
- }
-
- @Override
- public void initialize(Event event, KuduTable table) {
- this.payload = event.getBody();
- this.table = table;
- }
-
- @Override
- public List<Operation> getOperations() throws FlumeException {
- try {
- Insert insert = table.newInsert();
- PartialRow row = insert.getRow();
- row.addBinary(payloadColumn, payload);
-
- return Collections.singletonList((Operation) insert);
- } catch (Exception e){
- throw new FlumeException("Failed to create Kudu Insert object!", e);
- }
- }
-
- @Override
- public void close() {
- }
-}
[31/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsvMapper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsvMapper.java b/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsvMapper.java
deleted file mode 100644
index 21f43b5..0000000
--- a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/ImportCsvMapper.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.mapreduce.tools;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.*;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Mapper;
-
-import java.io.IOException;
-
-/**
- * Mapper that ingests CSV lines and turns them into Kudu Inserts.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ImportCsvMapper extends Mapper<LongWritable, Text, NullWritable, Operation> {
-
- private static final NullWritable NULL_KEY = NullWritable.get();
-
- /** Column seperator */
- private String separator;
-
- /** Should skip bad lines */
- private boolean skipBadLines;
- private Counter badLineCount;
-
- private CsvParser parser;
-
- private KuduTable table;
- private Schema schema;
-
- /**
- * Handles initializing this class with objects specific to it (i.e., the parser).
- */
- @Override
- protected void setup(Context context) {
- Configuration conf = context.getConfiguration();
-
- this.separator = conf.get(ImportCsv.SEPARATOR_CONF_KEY);
- if (this.separator == null) {
- this.separator = ImportCsv.DEFAULT_SEPARATOR;
- }
-
- this.skipBadLines = conf.getBoolean(ImportCsv.SKIP_LINES_CONF_KEY, true);
- this.badLineCount = context.getCounter(ImportCsv.Counters.BAD_LINES);
-
- this.parser = new CsvParser(conf.get(ImportCsv.COLUMNS_NAMES_KEY), this.separator);
-
- this.table = KuduTableMapReduceUtil.getTableFromContext(context);
- this.schema = this.table.getSchema();
- }
-
- /**
- * Convert a line of CSV text into a Kudu Insert
- */
- @Override
- public void map(LongWritable offset, Text value,
- Context context)
- throws IOException {
- byte[] lineBytes = value.getBytes();
-
- try {
- CsvParser.ParsedLine parsed = this.parser.parse(lineBytes, value.getLength());
-
- Insert insert = this.table.newInsert();
- PartialRow row = insert.getRow();
- for (int i = 0; i < parsed.getColumnCount(); i++) {
- String colName = parsed.getColumnName(i);
- ColumnSchema col = this.schema.getColumn(colName);
- String colValue = Bytes.getString(parsed.getLineBytes(), parsed.getColumnOffset(i),
- parsed.getColumnLength(i));
- switch (col.getType()) {
- case BOOL:
- row.addBoolean(colName, Boolean.parseBoolean(colValue));
- break;
- case INT8:
- row.addByte(colName, Byte.parseByte(colValue));
- break;
- case INT16:
- row.addShort(colName, Short.parseShort(colValue));
- break;
- case INT32:
- row.addInt(colName, Integer.parseInt(colValue));
- break;
- case INT64:
- row.addLong(colName, Long.parseLong(colValue));
- break;
- case STRING:
- row.addString(colName, colValue);
- break;
- case FLOAT:
- row.addFloat(colName, Float.parseFloat(colValue));
- break;
- case DOUBLE:
- row.addDouble(colName, Double.parseDouble(colValue));
- break;
- default:
- throw new IllegalArgumentException("Type " + col.getType() + " not recognized");
- }
- }
- context.write(NULL_KEY, insert);
- } catch (CsvParser.BadCsvLineException badLine) {
- if (this.skipBadLines) {
- System.err.println("Bad line at offset: " + offset.get() + ":\n" + badLine.getMessage());
- this.badLineCount.increment(1);
- return;
- } else {
- throw new IOException("Failing task because of a bad line", badLine);
- }
- } catch (IllegalArgumentException e) {
- if (this.skipBadLines) {
- System.err.println("Bad line at offset: " + offset.get() + ":\n" + e.getMessage());
- this.badLineCount.increment(1);
- return;
- } else {
- throw new IOException("Failing task because of an illegal argument", e);
- }
- } catch (InterruptedException e) {
- throw new IOException("Failing task since it was interrupted", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/IntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/IntegrationTestBigLinkedList.java
deleted file mode 100644
index 549e117..0000000
--- a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/IntegrationTestBigLinkedList.java
+++ /dev/null
@@ -1,1662 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. See accompanying LICENSE file.
- */
-package org.kududb.mapreduce.tools;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.*;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
-import org.kududb.util.Pair;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * <p>
- * This is an integration test borrowed from goraci, written by Keith Turner,
- * which is in turn inspired by the Accumulo test called continous ingest (ci).
- * The original source code can be found here:
- * </p>
- * <ul>
- * <li>
- * <a href="https://github.com/keith-turner/goraci">https://github.com/keith-turner/goraci</a>
- * </li>
- * <li>
- * <a href="https://github.com/enis/goraci/">https://github.com/enis/goraci/</a>
- * </li>
- * </ul>
- *
- * <p>
- * Apache Accumulo has a simple test suite that verifies that data is not
- * lost at scale. This test suite is called continuous ingest. This test runs
- * many ingest clients that continually create linked lists containing 25
- * million nodes. At some point the clients are stopped and a map reduce job is
- * run to ensure no linked list has a hole. A hole indicates data was lost.
- * </p>
- *
- * <p>
- * The nodes in the linked list are random. This causes each linked list to
- * spread across the table. Therefore if one part of a table loses data, then it
- * will be detected by references in another part of the table.
- * </p>
- *
- * <h3>
- * THE ANATOMY OF THE TEST
- * </h3>
- *
- * <p>
- * Below is rough sketch of how data is written. For specific details look at
- * the Generator code.
- * </p>
- * <ol>
- * <li>
- * Write out 1 million nodes
- * </li>
- * <li>
- * Flush the client
- * </li>
- * <li>
- * Write out 1 million that reference previous million
- * </li>
- * <li>
- * If this is the 25th set of 1 million nodes, then update 1st set of million to point to last
- * </li>
- * <li>
- * Goto 1
- * </li>
- * </ol>
- *
- * <p>
- * The key is that nodes only reference flushed nodes. Therefore a node should
- * never reference a missing node, even if the ingest client is killed at any
- * point in time.
- * </p>
- *
- * <p>
- * When running this test suite w/ Accumulo there is a script running in
- * parallel called the Agitator that randomly and continuously kills server
- * processes. The outcome was that many data loss bugs were found in Accumulo
- * by doing this. This test suite can also help find bugs that impact uptime
- * and stability when run for days or weeks.
- * </p>
- *
- * <p>
- * This test suite consists the following:
- * </p>
- * <ul>
- * <li>
- * A few Java programs
- * </li>
- * <li>
- * A little helper script to run the java programs
- * </li>
- * <li>
- * A maven script to build it.
- * </li>
- * </ul>
- *
- * <p>
- * When generating data, its best to have each map task generate a multiple of
- * 25 million. The reason for this is that circular linked list are generated
- * every 25M. Not generating a multiple in 25M will result in some nodes in the
- * linked list not having references. The loss of an unreferenced node can not
- * be detected.
- * </p>
- *
- * <h3>
- * Below is a description of the Java programs
- * </h3>
- *
- * <ul>
- * <li>
- * Generator - A map only job that generates data. As stated previously,
- * its best to generate data in multiples of 25M.
- * </li>
- * <li>
- * Verify - A map reduce job that looks for holes. Look at the counts after running. REFERENCED and
- * UNREFERENCED are ok, any UNDEFINED counts are bad. Do not run at the same
- * time as the Generator.
- * </li>
- * <li>
- * Print - A standalone program that prints nodes in the linked list
- * </li>
- * <li>
- * Delete - Disabled. A standalone program that deletes a single node
- * </li>
- * <li>
- * Walker - Disabled. A standalong program that start following a linked list and emits timing
- * info.
- * </li>
- * </ul>
- *
- * <h3>
- * KUDU-SPECIFIC CHANGES
- * </h3>
- *
- * <ul>
- * <li>
- * The 16 bytes row key is divided into two 8 byte long since we don't have a "bytes" type in
- * Kudu. Note that the C++ client can store bytes directly in string columns. Using longs
- * enables us to pretty print human readable keys than can then be passed back just as easily.
- * </li>
- * <li>
- * The table can be pre-split when running the Generator. The row keys' first component will be
- * spread over the Long.MIN_VALUE - Long.MAX_VALUE keyspace.
- * </li>
- * <li>
- * The Walker and Deleter progams were disabled to save some time but they can be re-enabled then
- * ported to Kudu without too much effort.
- * </li>
- * </ul>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class IntegrationTestBigLinkedList extends Configured implements Tool {
- private static final byte[] NO_KEY = new byte[1];
-
- protected static final String TABLE_NAME_KEY = "IntegrationTestBigLinkedList.table";
-
- protected static final String DEFAULT_TABLE_NAME = "IntegrationTestBigLinkedList";
-
- protected static final String HEADS_TABLE_NAME_KEY = "IntegrationTestBigLinkedList.heads_table";
-
- protected static final String DEFAULT_HEADS_TABLE_NAME = "IntegrationTestBigLinkedListHeads";
-
- /** Row key, two times 8 bytes. */
- private static final String COLUMN_KEY_ONE = "key1";
- private static final String COLUMN_KEY_TWO = "key2";
-
- /** Link to the id of the prev node in the linked list, two times 8 bytes. */
- private static final String COLUMN_PREV_ONE = "prev1";
- private static final String COLUMN_PREV_TWO = "prev2";
-
- /** identifier of the mapred task that generated this row. */
- private static final String COLUMN_CLIENT = "client";
-
- /** the id of the row within the same client. */
- private static final String COLUMN_ROW_ID = "row_id";
-
- /** The number of times this row was updated. */
- private static final String COLUMN_UPDATE_COUNT = "update_count";
-
- /** How many rows to write per map task. This has to be a multiple of 25M. */
- private static final String GENERATOR_NUM_ROWS_PER_MAP_KEY
- = "IntegrationTestBigLinkedList.generator.num_rows";
-
- private static final String GENERATOR_NUM_MAPPERS_KEY
- = "IntegrationTestBigLinkedList.generator.map.tasks";
-
- private static final String GENERATOR_WIDTH_KEY
- = "IntegrationTestBigLinkedList.generator.width";
-
- private static final String GENERATOR_WRAP_KEY
- = "IntegrationTestBigLinkedList.generator.wrap";
-
- private static final int WIDTH_DEFAULT = 1000000;
- private static final int WRAP_DEFAULT = 25;
- private static final int ROWKEY_LENGTH = 16;
-
- private String toRun;
- private String[] otherArgs;
-
- static class CINode {
- String key;
- String prev;
- String client;
- long rowId;
- int updateCount;
- }
-
- static Schema getTableSchema() {
- List<ColumnSchema> columns = new ArrayList<ColumnSchema>(7);
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_ONE, Type.INT64)
- .key(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_TWO, Type.INT64)
- .key(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_PREV_ONE, Type.INT64)
- .nullable(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_PREV_TWO, Type.INT64)
- .nullable(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_ROW_ID, Type.INT64)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_CLIENT, Type.STRING)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_UPDATE_COUNT, Type.INT32)
- .build());
- return new Schema(columns);
- }
-
- static Schema getHeadsTableSchema() {
- List<ColumnSchema> columns = new ArrayList<ColumnSchema>(2);
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_ONE, Type.INT64)
- .key(true)
- .build());
- columns.add(new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_TWO, Type.INT64)
- .key(true)
- .build());
- return new Schema(columns);
- }
-
- /**
- * A Map only job that generates random linked list and stores them.
- */
- static class Generator extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(Generator.class);
-
- static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> {
- static class GeneratorInputSplit extends InputSplit implements Writable {
- @Override
- public long getLength() throws IOException, InterruptedException {
- return 1;
- }
- @Override
- public String[] getLocations() throws IOException, InterruptedException {
- return new String[0];
- }
- @Override
- public void readFields(DataInput arg0) throws IOException {
- }
- @Override
- public void write(DataOutput arg0) throws IOException {
- }
- }
-
- static class GeneratorRecordReader extends RecordReader<BytesWritable,NullWritable> {
- private long count;
- private long numNodes;
- private Random rand;
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public BytesWritable getCurrentKey() throws IOException, InterruptedException {
- byte[] bytes = new byte[ROWKEY_LENGTH];
- rand.nextBytes(bytes);
- return new BytesWritable(bytes);
- }
-
- @Override
- public NullWritable getCurrentValue() throws IOException, InterruptedException {
- return NullWritable.get();
- }
-
- @Override
- public float getProgress() throws IOException, InterruptedException {
- return (float)(count / (double)numNodes);
- }
-
- @Override
- public void initialize(InputSplit arg0, TaskAttemptContext context)
- throws IOException, InterruptedException {
- numNodes = context.getConfiguration().getLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, 25000000);
- // Use SecureRandom to avoid issue described in HBASE-13382.
- rand = new SecureRandom();
- }
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- return count++ < numNodes;
- }
-
- }
-
- @Override
- public RecordReader<BytesWritable,NullWritable> createRecordReader(
- InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
- GeneratorRecordReader rr = new GeneratorRecordReader();
- rr.initialize(split, context);
- return rr;
- }
-
- @Override
- public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
- int numMappers = job.getConfiguration().getInt(GENERATOR_NUM_MAPPERS_KEY, 1);
-
- ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numMappers);
-
- for (int i = 0; i < numMappers; i++) {
- splits.add(new GeneratorInputSplit());
- }
-
- return splits;
- }
- }
-
- /** Ensure output files from prev-job go to map inputs for current job */
- static class OneFilePerMapperSFIF<K, V> extends SequenceFileInputFormat<K, V> {
- @Override
- protected boolean isSplitable(JobContext context, Path filename) {
- return false;
- }
- }
-
- /**
- * Some ASCII art time:
- * [ . . . ] represents one batch of random longs of length WIDTH
- *
- * _________________________
- * | ______ |
- * | | ||
- * __+_________________+_____ ||
- * v v v |||
- * first = [ . . . . . . . . . . . ] |||
- * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ |||
- * | | | | | | | | | | | |||
- * prev = [ . . . . . . . . . . . ] |||
- * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ |||
- * | | | | | | | | | | | |||
- * current = [ . . . . . . . . . . . ] |||
- * |||
- * ... |||
- * |||
- * last = [ . . . . . . . . . . . ] |||
- * | | | | | | | | | | |-----|||
- * | |--------||
- * |___________________________|
- */
- static class GeneratorMapper
- extends Mapper<BytesWritable, NullWritable, NullWritable, NullWritable> {
-
- private byte[][] first = null;
- private byte[][] prev = null;
- private byte[][] current = null;
- private String id;
- private long rowId = 0;
- private int i;
- private KuduClient client;
- private KuduTable table;
- private KuduSession session;
- private KuduTable headsTable;
- private long numNodes;
- private long wrap;
- private int width;
-
- @Override
- protected void setup(Context context) throws IOException, InterruptedException {
- id = "Job: " + context.getJobID() + " Task: " + context.getTaskAttemptID();
- Configuration conf = context.getConfiguration();
- CommandLineParser parser = new CommandLineParser(conf);
- client = parser.getClient();
- try {
- table = client.openTable(getTableName(conf));
- headsTable = client.openTable(getHeadsTable(conf));
- } catch (Exception e) {
- throw new IOException(e);
- }
- session = client.newSession();
- session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
- session.setMutationBufferSpace(WIDTH_DEFAULT);
- session.setIgnoreAllDuplicateRows(true);
-
- this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT);
- current = new byte[this.width][];
- int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT);
- this.wrap = (long)wrapMultiplier * width;
- this.numNodes = context.getConfiguration().getLong(
- GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT);
- if (this.numNodes < this.wrap) {
- this.wrap = this.numNodes;
- }
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- try {
- session.close();
- client.shutdown();
- } catch (Exception ex) {
- // ugh.
- throw new IOException(ex);
- }
- }
-
- @Override
- protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
- current[i] = new byte[key.getLength()];
- System.arraycopy(key.getBytes(), 0, current[i], 0, key.getLength());
- if (++i == current.length) {
- persist(output, current, false);
- i = 0;
-
- // Keep track of the first row so that we can point to it at the end.
- if (first == null) {
- first = current;
- }
- prev = current;
- current = new byte[this.width][];
-
- rowId += current.length;
- output.setStatus("Count " + rowId);
-
- // Check if it's time to wrap up this batch.
- if (rowId % wrap == 0) {
- // this block of code turns the 1 million linked list of length 25 into one giant
- // circular linked list of 25 million.
- circularLeftShift(first);
-
- persist(output, first, true);
-
- Operation insert = headsTable.newInsert();
- PartialRow row = insert.getRow();
- row.addLong(COLUMN_KEY_ONE, Bytes.getLong(first[0]));
- row.addLong(COLUMN_KEY_TWO, Bytes.getLong(first[0], 8));
- try {
- session.apply(insert);
- session.flush();
- } catch (Exception e) {
- throw new IOException("Couldn't flush the head row, " + insert, e);
- }
-
- first = null;
- prev = null;
- }
- }
- }
-
- private static <T> void circularLeftShift(T[] first) {
- T ez = first[0];
- for (int i = 0; i < first.length - 1; i++)
- first[i] = first[i + 1];
- first[first.length - 1] = ez;
- }
-
- private void persist(Context output, byte[][] data, boolean update)
- throws IOException {
- try {
- for (int i = 0; i < data.length; i++) {
- Operation put = update ? table.newUpdate() : table.newInsert();
- PartialRow row = put.getRow();
-
- long keyOne = Bytes.getLong(data[i]);
- long keyTwo = Bytes.getLong(data[i], 8);
-
- row.addLong(COLUMN_KEY_ONE, keyOne);
- row.addLong(COLUMN_KEY_TWO, keyTwo);
-
- // prev is null for the first line, we'll update it at the end.
- if (prev == null) {
- row.setNull(COLUMN_PREV_ONE);
- row.setNull(COLUMN_PREV_TWO);
- } else {
- row.addLong(COLUMN_PREV_ONE, Bytes.getLong(prev[i]));
- row.addLong(COLUMN_PREV_TWO, Bytes.getLong(prev[i], 8));
- }
-
- if (!update) {
- // We only add those for new inserts, we don't update the heads with a new row, etc.
- row.addLong(COLUMN_ROW_ID, rowId + i);
- row.addString(COLUMN_CLIENT, id);
- row.addInt(COLUMN_UPDATE_COUNT, 0);
- }
- session.apply(put);
-
- if (i % 1000 == 0) {
- // Tickle progress every so often else maprunner will think us hung
- output.progress();
- }
- }
-
- session.flush();
- } catch (Exception ex) {
- throw new IOException(ex);
- }
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- if (args.length < 4) {
- System.out.println("Usage : " + Generator.class.getSimpleName() +
- " <num mappers> <num nodes per map> <num_tablets> <tmp output dir> [<width> <wrap " +
- "multiplier>]");
- System.out.println(" where <num nodes per map> should be a multiple of " +
- " width*wrap multiplier, 25M by default");
- return 0;
- }
-
- int numMappers = Integer.parseInt(args[0]);
- long numNodes = Long.parseLong(args[1]);
- int numTablets = Integer.parseInt(args[2]);
- Path tmpOutput = new Path(args[3]);
- Integer width = (args.length < 5) ? null : Integer.parseInt(args[4]);
- Integer wrapMultiplier = (args.length < 6) ? null : Integer.parseInt(args[5]);
- return run(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier);
- }
-
- protected void createTables(int numTablets) throws Exception {
-
- createSchema(getTableName(getConf()), getTableSchema(), numTablets);
- createSchema(getHeadsTable(getConf()), getHeadsTableSchema(), numTablets);
- }
-
- protected void createSchema(String tableName, Schema schema, int numTablets) throws Exception {
- CommandLineParser parser = new CommandLineParser(getConf());
- KuduClient client = parser.getClient();
- try {
- if (numTablets < 1) {
- numTablets = 1;
- }
-
- if (client.tableExists(tableName)) {
- return;
- }
-
- CreateTableOptions builder =
- new CreateTableOptions().setNumReplicas(parser.getNumReplicas())
- .setRangePartitionColumns(ImmutableList.of("key1", "key2"));
- if (numTablets > 1) {
- BigInteger min = BigInteger.valueOf(Long.MIN_VALUE);
- BigInteger max = BigInteger.valueOf(Long.MAX_VALUE);
- BigInteger step = max.multiply(BigInteger.valueOf(2)).divide(BigInteger.valueOf
- (numTablets));
- LOG.info(min.longValue());
- LOG.info(max.longValue());
- LOG.info(step.longValue());
- PartialRow splitRow = schema.newPartialRow();
- splitRow.addLong("key2", Long.MIN_VALUE);
- for (int i = 1; i < numTablets; i++) {
- long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue();
- LOG.info("key " + key);
- splitRow.addLong("key1", key);
- builder.addSplitRow(splitRow);
- }
- }
-
- client.createTable(tableName, schema, builder);
- } finally {
- // Done with this client.
- client.shutdown();
- }
- }
-
- public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,
- Integer width, Integer wrapMultiplier) throws Exception {
- LOG.info("Running RandomInputGenerator with numMappers=" + numMappers
- + ", numNodes=" + numNodes);
- Job job = new Job(getConf());
-
- job.setJobName("Random Input Generator");
- job.setNumReduceTasks(0);
- job.setJarByClass(getClass());
-
- job.setInputFormatClass(GeneratorInputFormat.class);
- job.setOutputKeyClass(BytesWritable.class);
- job.setOutputValueClass(NullWritable.class);
-
- setJobConf(job, numMappers, numNodes, width, wrapMultiplier);
-
- job.setMapperClass(Mapper.class); //identity mapper
-
- FileOutputFormat.setOutputPath(job, tmpOutput);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
-
- boolean success = job.waitForCompletion(true);
-
- return success ? 0 : 1;
- }
-
- public int runGenerator(int numMappers, long numNodes, int numTablets, Path tmpOutput,
- Integer width, Integer wrapMultiplier) throws Exception {
- LOG.info("Running Generator with numMappers=" + numMappers +", numNodes=" + numNodes);
- createTables(numTablets);
-
- Job job = new Job(getConf());
-
- job.setJobName("Link Generator");
- job.setNumReduceTasks(0);
- job.setJarByClass(getClass());
-
- FileInputFormat.setInputPaths(job, tmpOutput);
- job.setInputFormatClass(OneFilePerMapperSFIF.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(NullWritable.class);
-
- setJobConf(job, numMappers, numNodes, width, wrapMultiplier);
-
- job.setMapperClass(GeneratorMapper.class);
-
- job.setOutputFormatClass(NullOutputFormat.class);
-
- job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
- // If we fail, retrying will fail again in case we were able to flush at least once since
- // we'll be creating duplicate rows. Better to just have one try.
- job.getConfiguration().setInt("mapreduce.map.maxattempts", 1);
- // Lack of YARN-445 means we can't auto-jstack on timeout, so disabling the timeout gives
- // us a chance to do it manually.
- job.getConfiguration().setInt("mapreduce.task.timeout", 0);
- KuduTableMapReduceUtil.addDependencyJars(job);
-
- boolean success = job.waitForCompletion(true);
-
- return success ? 0 : 1;
- }
-
- public int run(int numMappers, long numNodes, int numTablets, Path tmpOutput,
- Integer width, Integer wrapMultiplier) throws Exception {
- int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier);
- if (ret > 0) {
- return ret;
- }
- return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier);
- }
- }
-
- /**
- * A Map Reduce job that verifies that the linked lists generated by
- * {@link Generator} do not have any holes.
- */
- static class Verify extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(Verify.class);
- private static final BytesWritable DEF = new BytesWritable(NO_KEY);
- private static final Joiner COMMA_JOINER = Joiner.on(",");
- private static final byte[] rowKey = new byte[ROWKEY_LENGTH];
- private static final byte[] prev = new byte[ROWKEY_LENGTH];
-
- private Job job;
-
- public static class VerifyMapper extends Mapper<NullWritable, RowResult,
- BytesWritable, BytesWritable> {
- private BytesWritable row = new BytesWritable();
- private BytesWritable ref = new BytesWritable();
-
- @Override
- protected void map(NullWritable key, RowResult value, Mapper.Context context)
- throws IOException ,InterruptedException {
- Bytes.setLong(rowKey, value.getLong(0));
- Bytes.setLong(rowKey, value.getLong(1), 8);
-
- row.set(rowKey, 0, rowKey.length);
- // Emit that the row is defined
- context.write(row, DEF);
- if (value.isNull(2)) {
- LOG.warn(String.format("Prev is not set for: %s", Bytes.pretty(rowKey)));
- } else {
- Bytes.setLong(prev, value.getLong(2));
- Bytes.setLong(prev, value.getLong(3), 8);
- ref.set(prev, 0, prev.length);
- // Emit which row is referenced by this row.
- context.write(ref, row);
- }
- }
- }
-
- public enum Counts {
- UNREFERENCED, UNDEFINED, REFERENCED, EXTRAREFERENCES
- }
-
- public static class VerifyReducer extends Reducer<BytesWritable,BytesWritable,Text,Text> {
- private ArrayList<byte[]> refs = new ArrayList<byte[]>();
-
- private AtomicInteger rows = new AtomicInteger(0);
-
- @Override
- public void reduce(BytesWritable key, Iterable<BytesWritable> values, Context context)
- throws IOException, InterruptedException {
-
- int defCount = 0;
-
- refs.clear();
- // We only expect two values, a DEF and a reference, but there might be more.
- for (BytesWritable type : values) {
- if (type.getLength() == DEF.getLength()) {
- defCount++;
- } else {
- byte[] bytes = new byte[type.getLength()];
- System.arraycopy(type.getBytes(), 0, bytes, 0, type.getLength());
- refs.add(bytes);
- }
- }
-
- // TODO check for more than one def, should not happen
-
- List<String> refsList = new ArrayList<>(refs.size());
- String keyString = null;
- if (defCount == 0 || refs.size() != 1) {
- for (byte[] ref : refs) {
- refsList.add(COMMA_JOINER.join(Bytes.getLong(ref), Bytes.getLong(ref, 8)));
- }
- keyString = COMMA_JOINER.join(Bytes.getLong(key.getBytes()),
- Bytes.getLong(key.getBytes(), 8));
-
- LOG.error("Linked List error: Key = " + keyString + " References = " + refsList);
- }
-
- if (defCount == 0 && refs.size() > 0) {
- // this is bad, found a node that is referenced but not defined. It must have been
- // lost, emit some info about this node for debugging purposes.
- context.write(new Text(keyString), new Text(refsList.toString()));
- context.getCounter(Counts.UNDEFINED).increment(1);
- } else if (defCount > 0 && refs.size() == 0) {
- // node is defined but not referenced
- context.write(new Text(keyString), new Text("none"));
- context.getCounter(Counts.UNREFERENCED).increment(1);
- } else {
- if (refs.size() > 1) {
- if (refsList != null) {
- context.write(new Text(keyString), new Text(refsList.toString()));
- }
- context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1);
- }
- // node is defined and referenced
- context.getCounter(Counts.REFERENCED).increment(1);
- }
-
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
-
- if (args.length != 2) {
- System.out.println("Usage : " + Verify.class.getSimpleName() + " <output dir> <num reducers>");
- return 0;
- }
-
- String outputDir = args[0];
- int numReducers = Integer.parseInt(args[1]);
-
- return run(outputDir, numReducers);
- }
-
- public int run(String outputDir, int numReducers) throws Exception {
- return run(new Path(outputDir), numReducers);
- }
-
- public int run(Path outputDir, int numReducers) throws Exception {
- LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);
-
- job = new Job(getConf());
-
- job.setJobName("Link Verifier");
- job.setNumReduceTasks(numReducers);
- job.setJarByClass(getClass());
-
- Joiner columnsToQuery = Joiner.on(",");
-
- new KuduTableMapReduceUtil.TableInputFormatConfiguratorWithCommandLineParser(
- job, getTableName(getConf()),
- columnsToQuery.join(COLUMN_KEY_ONE, COLUMN_KEY_TWO, COLUMN_PREV_ONE, COLUMN_PREV_TWO))
- .configure();
- job.setMapperClass(VerifyMapper.class);
- job.setMapOutputKeyClass(BytesWritable.class);
- job.setMapOutputValueClass(BytesWritable.class);
- job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
-
- job.setReducerClass(VerifyReducer.class);
- job.setOutputFormatClass(TextOutputFormat.class);
- TextOutputFormat.setOutputPath(job, outputDir);
-
- boolean success = job.waitForCompletion(true);
-
- return success ? 0 : 1;
- }
-
- @SuppressWarnings("deprecation")
- public boolean verify(long expectedReferenced) throws Exception {
- if (job == null) {
- throw new IllegalStateException("You should call run() first");
- }
-
- Counters counters = job.getCounters();
-
- Counter referenced = counters.findCounter(Counts.REFERENCED);
- Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
- Counter undefined = counters.findCounter(Counts.UNDEFINED);
- Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES);
-
- boolean success = true;
- //assert
- if (expectedReferenced != referenced.getValue()) {
- LOG.error("Expected referenced count does not match with actual referenced count. " +
- "Expected referenced=" + expectedReferenced + ", actual=" + referenced.getValue());
- success = false;
- }
-
- if (unreferenced.getValue() > 0) {
- boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue());
- LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue()
- + (couldBeMultiRef ? "; could be due to duplicate random numbers" : ""));
- success = false;
- }
-
- if (undefined.getValue() > 0) {
- LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
- success = false;
- }
-
- // TODO Add the rows' location on failure.
- if (!success) {
- //Configuration conf = job.getConfiguration();
- //HConnection conn = HConnectionManager.getConnection(conf);
- //TableName tableName = getTableName(conf);
- CounterGroup g = counters.getGroup("undef");
- Iterator<Counter> it = g.iterator();
- while (it.hasNext()) {
- String keyString = it.next().getName();
- //byte[] key = Bytes.toBytes(keyString);
- //HRegionLocation loc = conn.relocateRegion(tableName, key);
- LOG.error("undefined row " + keyString /*+ ", " + loc*/);
- }
- g = counters.getGroup("unref");
- it = g.iterator();
- while (it.hasNext()) {
- String keyString = it.next().getName();
- //byte[] key = Bytes.toBytes(keyString);
- //HRegionLocation loc = conn.relocateRegion(tableName, key);
- LOG.error("unreferred row " + keyString /*+ ", " + loc*/);
- }
- }
- return success;
- }
- }
-
- /**
- * Executes Generate and Verify in a loop. Data is not cleaned between runs, so each iteration
- * adds more data.
- */
- static class Loop extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(Loop.class);
-
- IntegrationTestBigLinkedList it;
-
- FileSystem fs;
-
- protected void runGenerator(int numMappers, long numNodes, int numTablets,
- String outputDir, Integer width, Integer wrapMultiplier) throws Exception {
- Path outputPath = new Path(outputDir);
- UUID uuid = UUID.randomUUID(); //create a random UUID.
- Path generatorOutput = new Path(outputPath, uuid.toString());
-
- Generator generator = new Generator();
- generator.setConf(getConf());
- int retCode = generator.run(numMappers, numNodes, numTablets, generatorOutput, width,
- wrapMultiplier);
- if (retCode > 0) {
- throw new RuntimeException("Generator failed with return code: " + retCode);
- }
- fs.delete(generatorOutput, true);
- }
-
- protected void runVerify(String outputDir,
- int numReducers,
- long expectedNumNodes,
- int retries) throws Exception {
- // Kudu doesn't fully support snapshot consistency so we might start reading from a node that
- // doesn't have all the data. This happens often with under "chaos monkey"-type of setups.
- for (int i = 0; i < retries; i++) {
- if (i > 0) {
- long sleep = 60 * 1000;
- LOG.info("Retrying in " + sleep + "ms");
- Thread.sleep(sleep);
- }
-
- Path outputPath = new Path(outputDir);
- UUID uuid = UUID.randomUUID(); //create a random UUID.
- Path iterationOutput = new Path(outputPath, uuid.toString());
-
- Verify verify = new Verify();
- verify.setConf(getConf());
- int retCode = verify.run(iterationOutput, numReducers);
- if (retCode > 0) {
- LOG.warn("Verify.run failed with return code: " + retCode);
- } else if (!verify.verify(expectedNumNodes)) {
- LOG.warn("Verify.verify failed");
- } else {
- fs.delete(iterationOutput, true);
- LOG.info("Verify finished with success. Total nodes=" + expectedNumNodes);
- return;
- }
- }
- throw new RuntimeException("Ran out of retries to verify");
- }
-
- @Override
- public int run(String[] args) throws Exception {
- if (args.length < 6) {
- System.err.println("Usage: Loop <num iterations> <num mappers> <num nodes per mapper> " +
- "<num_tablets> <output dir> <num reducers> [<width> <wrap multiplier>" +
- "<start expected nodes> <num_verify_retries>]");
- return 1;
- }
- LOG.info("Running Loop with args:" + Arrays.deepToString(args));
-
- int numIterations = Integer.parseInt(args[0]);
- int numMappers = Integer.parseInt(args[1]);
- long numNodes = Long.parseLong(args[2]);
- int numTablets = Integer.parseInt(args[3]);
- String outputDir = args[4];
- int numReducers = Integer.parseInt(args[5]);
- Integer width = (args.length < 7) ? null : Integer.parseInt(args[6]);
- Integer wrapMultiplier = (args.length < 8) ? null : Integer.parseInt(args[7]);
- long expectedNumNodes = (args.length < 9) ? 0 : Long.parseLong(args[8]);
- int numVerifyRetries = (args.length < 10) ? 3 : Integer.parseInt(args[9]);
-
- if (numIterations < 0) {
- numIterations = Integer.MAX_VALUE; // run indefinitely (kind of)
- }
-
- fs = FileSystem.get(getConf());
-
- for (int i = 0; i < numIterations; i++) {
- LOG.info("Starting iteration = " + i);
- runGenerator(numMappers, numNodes, numTablets, outputDir, width, wrapMultiplier);
- expectedNumNodes += numMappers * numNodes;
-
- runVerify(outputDir, numReducers, expectedNumNodes, numVerifyRetries);
- }
-
- return 0;
- }
- }
-
- /**
- * A stand alone program that prints out portions of a list created by {@link Generator}
- */
- private static class Print extends Configured implements Tool {
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
- options.addOption("s", "start", true, "start key, only the first component");
- options.addOption("e", "end", true, "end key (exclusive), only the first component");
- options.addOption("l", "limit", true, "number to print");
-
- GnuParser parser = new GnuParser();
- CommandLine cmd = null;
- try {
- cmd = parser.parse(options, args);
- if (cmd.getArgs().length != 0) {
- throw new ParseException("Command takes no arguments");
- }
- } catch (ParseException e) {
- System.err.println("Failed to parse command line " + e.getMessage());
- System.err.println();
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(getClass().getSimpleName(), options);
- System.exit(-1);
- }
-
- CommandLineParser cmdLineParser = new CommandLineParser(getConf());
- long timeout = cmdLineParser.getOperationTimeoutMs();
- KuduClient client = cmdLineParser.getClient();
-
- KuduTable table = client.openTable(getTableName(getConf()));
- KuduScanner.KuduScannerBuilder builder =
- client.newScannerBuilder(table)
- .scanRequestTimeout(timeout);
-
-
- if (cmd.hasOption("s")) {
- PartialRow row = table.getSchema().newPartialRow();
- row.addLong(0, Long.parseLong(cmd.getOptionValue("s")));
- builder.lowerBound(row);
- }
- if (cmd.hasOption("e")) {
- PartialRow row = table.getSchema().newPartialRow();
- row.addLong(0, Long.parseLong(cmd.getOptionValue("e")));
- builder.exclusiveUpperBound(row);
- }
-
- int limit = cmd.hasOption("l") ? Integer.parseInt(cmd.getOptionValue("l")) : 100;
-
- int count = 0;
-
- KuduScanner scanner = builder.build();
- while (scanner.hasMoreRows() && count < limit) {
- RowResultIterator rowResults = scanner.nextRows();
- count = printNodesAndGetNewCount(count, limit, rowResults);
- }
- RowResultIterator rowResults = scanner.close();
- printNodesAndGetNewCount(count, limit, rowResults);
-
- client.shutdown();
-
- return 0;
- }
-
- private static int printNodesAndGetNewCount(int oldCount, int limit,
- RowResultIterator rowResults) {
- int newCount = oldCount;
- if (rowResults == null) {
- return newCount;
- }
-
- CINode node = new CINode();
- for (RowResult result : rowResults) {
- newCount++;
- node = getCINode(result, node);
- printCINodeString(node);
- if (newCount == limit) {
- break;
- }
- }
- return newCount;
- }
- }
-
- /**
- * This tool needs to be run separately from the Generator-Verify loop. It can run while the
- * other two are running or in between loops.
- *
- * Each mapper scans a "heads" table and, for each row, follows the circular linked list and
- * updates their counter until it reaches the head of the list again.
- */
- private static class Updater extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(Updater.class);
-
- private static final String MAX_LINK_UPDATES_PER_MAPPER = "kudu.updates.per.mapper";
-
- public enum Counts {
- // Stats on what we're updating.
- UPDATED_LINKS,
- UPDATED_NODES,
- FIRST_UPDATE,
- SECOND_UPDATE,
- THIRD_UPDATE,
- FOURTH_UPDATE,
- MORE_THAN_FOUR_UPDATES,
- // Stats on what's broken.
- BROKEN_LINKS,
- BAD_UPDATE_COUNTS
- }
-
- public static class UpdaterMapper extends Mapper<NullWritable, RowResult,
- NullWritable, NullWritable> {
- private KuduClient client;
- private KuduTable table;
- private KuduSession session;
-
- /**
- * Schema we use when getting rows from the linked list, we only need the reference and
- * its update count.
- */
- private final List<String> SCAN_COLUMN_NAMES = ImmutableList.of(
- COLUMN_PREV_ONE, COLUMN_PREV_TWO, COLUMN_UPDATE_COUNT, COLUMN_CLIENT);
-
- private long numUpdatesPerMapper;
-
- /**
- * Processing each linked list takes minutes, meaning that it's easily possible for our
- * scanner to timeout. Instead, we gather all the linked list heads that we need and
- * process them all at once in the first map invocation.
- */
- private List<Pair<Long, Long>> headsCache;
-
- @Override
- protected void setup(Context context) throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
- CommandLineParser parser = new CommandLineParser(conf);
- client = parser.getClient();
- try {
- table = client.openTable(getTableName(conf));
- } catch (Exception e) {
- throw new IOException("Couldn't open the linked list table", e);
- }
- session = client.newSession();
-
- Schema tableSchema = table.getSchema();
-
-
- numUpdatesPerMapper = conf.getLong(MAX_LINK_UPDATES_PER_MAPPER, 1);
- headsCache = new ArrayList<Pair<Long, Long>>((int)numUpdatesPerMapper);
- }
-
- @Override
- protected void map(NullWritable key, RowResult value, Mapper.Context context)
- throws IOException, InterruptedException {
- // Add as many heads as we need, then we skip the rest.
- do {
- if (headsCache.size() < numUpdatesPerMapper) {
- value = (RowResult)context.getCurrentValue();
- headsCache.add(new Pair<Long, Long>(value.getLong(0), value.getLong(1)));
- }
- } while (context.nextKeyValue());
-
- // At this point we've exhausted the scanner and hopefully gathered all the linked list
- // heads we needed.
- LOG.info("Processing " + headsCache.size() +
- " linked lists, out of " + numUpdatesPerMapper);
- processAllHeads(context);
- }
-
- private void processAllHeads(Mapper.Context context) throws IOException {
- for (Pair<Long, Long> value : headsCache) {
- processHead(value, context);
- }
- }
-
- private void processHead(Pair<Long, Long> head, Mapper.Context context) throws IOException {
- long headKeyOne = head.getFirst();
- long headKeyTwo = head.getSecond();
- long prevKeyOne = headKeyOne;
- long prevKeyTwo = headKeyTwo;
- int currentCount = -1;
- int newCount = -1;
- String client = null;
-
- // Always printing this out, really useful when debugging.
- LOG.info("Head: " + getStringFromKeys(headKeyOne, headKeyTwo));
-
- do {
- RowResult prev = nextNode(prevKeyOne, prevKeyTwo);
- if (prev == null) {
- context.getCounter(Counts.BROKEN_LINKS).increment(1);
- LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " doesn't exist");
- break;
- }
-
- // It's possible those columns are null, let's not break trying to read them.
- if (prev.isNull(0) || prev.isNull(1)) {
- context.getCounter(Counts.BROKEN_LINKS).increment(1);
- LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " isn't referencing anywhere");
- break;
- }
-
- int prevCount = prev.getInt(2);
- String prevClient = prev.getString(3);
- if (currentCount == -1) {
- // First time we loop we discover what the count was and set the new one.
- currentCount = prevCount;
- newCount = currentCount + 1;
- client = prevClient;
- }
-
- if (prevCount != currentCount) {
- context.getCounter(Counts.BAD_UPDATE_COUNTS).increment(1);
- LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " has a wrong updateCount, " +
- prevCount + " instead of " + currentCount);
- // Game over, there's corruption.
- break;
- }
-
- if (!prevClient.equals(client)) {
- context.getCounter(Counts.BROKEN_LINKS).increment(1);
- LOG.warn(getStringFromKeys(prevKeyOne, prevKeyTwo) + " has the wrong client, " +
- "bad reference? Bad client= " + prevClient);
- break;
- }
-
- updateRow(prevKeyOne, prevKeyTwo, newCount);
- context.getCounter(Counts.UPDATED_NODES).increment(1);
- if (prevKeyOne % 10 == 0) {
- context.progress();
- }
- prevKeyOne = prev.getLong(0);
- prevKeyTwo = prev.getLong(1);
- } while (headKeyOne != prevKeyOne && headKeyTwo != prevKeyTwo);
-
- updateStatCounters(context, newCount);
- context.getCounter(Counts.UPDATED_LINKS).increment(1);
- }
-
- /**
- * Finds the next node in the linked list.
- */
- private RowResult nextNode(long prevKeyOne, long prevKeyTwo) throws IOException {
- KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table)
- .setProjectedColumnNames(SCAN_COLUMN_NAMES);
-
- configureScannerForRandomRead(builder, table, prevKeyOne, prevKeyTwo);
-
- try {
- return getOneRowResult(builder.build());
- } catch (Exception e) {
- // Goes right out and fails the job.
- throw new IOException("Couldn't read the following row: " +
- getStringFromKeys(prevKeyOne, prevKeyTwo), e);
- }
- }
-
- private void updateRow(long keyOne, long keyTwo, int newCount) throws IOException {
- Update update = table.newUpdate();
- PartialRow row = update.getRow();
- row.addLong(COLUMN_KEY_ONE, keyOne);
- row.addLong(COLUMN_KEY_TWO, keyTwo);
- row.addInt(COLUMN_UPDATE_COUNT, newCount);
- try {
- session.apply(update);
- } catch (Exception e) {
- // Goes right out and fails the job.
- throw new IOException("Couldn't update the following row: " +
- getStringFromKeys(keyOne, keyTwo), e);
- }
- }
-
- /**
- * We keep some statistics about the linked list we update so that we can get a feel of
- * what's being updated.
- */
- private void updateStatCounters(Mapper.Context context, int newCount) {
- switch (newCount) {
- case -1:
- case 0:
- // TODO We didn't event get the first node?
- break;
- case 1:
- context.getCounter(Counts.FIRST_UPDATE).increment(1);
- break;
- case 2:
- context.getCounter(Counts.SECOND_UPDATE).increment(1);
- break;
- case 3:
- context.getCounter(Counts.THIRD_UPDATE).increment(1);
- break;
- case 4:
- context.getCounter(Counts.FOURTH_UPDATE).increment(1);
- break;
- default:
- context.getCounter(Counts.MORE_THAN_FOUR_UPDATES).increment(1);
- break;
- }
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- try {
- session.close();
- client.shutdown();
- } catch (Exception ex) {
- // Goes right out and fails the job.
- throw new IOException("Coulnd't close the scanner after the task completed", ex);
- }
- }
- }
-
- public int run(long maxLinkUpdatesPerMapper) throws Exception {
- LOG.info("Running Updater with maxLinkUpdatesPerMapper=" + maxLinkUpdatesPerMapper);
-
- Job job = new Job(getConf());
-
- job.setJobName("Link Updater");
- job.setNumReduceTasks(0);
- job.setJarByClass(getClass());
-
- Joiner columnsToQuery = Joiner.on(",");
-
- new KuduTableMapReduceUtil.TableInputFormatConfiguratorWithCommandLineParser(
- job, getHeadsTable(getConf()),
- columnsToQuery.join(COLUMN_KEY_ONE, COLUMN_KEY_TWO))
- .configure();
-
- job.setMapperClass(UpdaterMapper.class);
- job.setMapOutputKeyClass(BytesWritable.class);
- job.setMapOutputValueClass(BytesWritable.class);
- job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
- // If something fails we want to exit ASAP.
- job.getConfiguration().setInt("mapreduce.map.maxattempts", 1);
- // Lack of YARN-445 means we can't auto-jstack on timeout, so disabling the timeout gives
- // us a chance to do it manually.
- job.getConfiguration().setInt("mapreduce.task.timeout", 0);
- job.getConfiguration().setLong(MAX_LINK_UPDATES_PER_MAPPER, maxLinkUpdatesPerMapper);
-
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(NullWritable.class);
- job.setOutputFormatClass(NullOutputFormat.class);
-
- KuduTableMapReduceUtil.addDependencyJars(job);
-
- boolean success = job.waitForCompletion(true);
-
- Counters counters = job.getCounters();
-
- if (success) {
- // Let's not continue looping if we have broken linked lists.
- Counter brokenLinks = counters.findCounter(Counts.BROKEN_LINKS);
- Counter badUpdates = counters.findCounter(Counts.BAD_UPDATE_COUNTS);
- if (brokenLinks.getValue() > 0 || badUpdates.getValue() > 0) {
- LOG.error("Corruption was detected, see the job's counters. Ending the update loop.");
- success = false;
- }
- }
- return success ? 0 : 1;
- }
-
- @Override
- public int run(String[] args) throws Exception {
- if (args.length < 2) {
- System.err.println("Usage: Update <num iterations> <max link updates per mapper>");
- System.err.println(" where <num iterations> will be 'infinite' if passed a negative value" +
- " or zero");
- return 1;
- }
- LOG.info("Running Loop with args:" + Arrays.deepToString(args));
-
- int numIterations = Integer.parseInt(args[0]);
- long maxUpdates = Long.parseLong(args[1]);
-
- if (numIterations <= 0) {
- numIterations = Integer.MAX_VALUE;
- }
-
- if (maxUpdates < 1) {
- maxUpdates = 1;
- }
-
- for (int i = 0; i < numIterations; i++) {
- LOG.info("Starting iteration = " + i);
- int ret = run(maxUpdates);
- if (ret != 0) {
- LOG.error("Can't continue updating, last run failed.");
- return ret;
- }
- }
-
- return 0;
- }
- }
-
- /**
- * A stand alone program that deletes a single node.
- * TODO
- */
- /*private static class Delete extends Configured implements Tool {
- @Override
- public int run(String[] args) throws Exception {
- if (args.length != 1) {
- System.out.println("Usage : " + Delete.class.getSimpleName() + " <node to delete>");
- return 0;
- }
- byte[] val = Bytes.toBytesBinary(args[0]);
-
- org.apache.hadoop.hbase.client.Delete delete
- = new org.apache.hadoop.hbase.client.Delete(val);
-
- HTable table = new HTable(getConf(), getTableName(getConf()));
-
- table.delete(delete);
- table.flushCommits();
- table.close();
-
- System.out.println("Delete successful");
- return 0;
- }
- }*/
-
- /**
- * A stand alone program that follows a linked list created by {@link Generator}
- * and prints timing info.
- *
- */
- private static class Walker extends Configured implements Tool {
-
- private KuduClient client;
- private KuduTable table;
-
- @Override
- public int run(String[] args) throws IOException {
- if (args.length < 1) {
- System.err.println("Usage: Walker <start key> [<num nodes>]");
- System.err.println(" where <num nodes> defaults to 100 nodes that will be printed out");
- return 1;
- }
- int maxNumNodes = 100;
- if (args.length == 2) {
- maxNumNodes = Integer.parseInt(args[1]);
- }
- System.out.println("Running Walker with args:" + Arrays.deepToString(args));
-
- String[] keys = args[0].split(",");
- if (keys.length != 2) {
- System.err.println("The row key must be formatted like key1,key2");
- return 1;
- }
-
- long keyOne = Long.parseLong(keys[0]);
- long keyTwo = Long.parseLong(keys[1]);
-
- System.out.println("Walking with " + getStringFromKeys(keyOne, keyTwo));
-
- try {
- walk(keyOne, keyTwo, maxNumNodes);
- } catch (Exception e) {
- throw new IOException(e);
- }
- return 0;
- }
-
- private void walk(long headKeyOne, long headKeyTwo, int maxNumNodes) throws Exception {
- CommandLineParser parser = new CommandLineParser(getConf());
- client = parser.getClient();
- table = client.openTable(getTableName(getConf()));
-
- long prevKeyOne = headKeyOne;
- long prevKeyTwo = headKeyTwo;
- CINode node = new CINode();
- int nodesCount = 0;
-
- do {
- RowResult rr = nextNode(prevKeyOne, prevKeyTwo);
- if (rr == null) {
- System.err.println(getStringFromKeys(prevKeyOne, prevKeyTwo) + " doesn't exist!");
- break;
- }
- getCINode(rr, node);
- printCINodeString(node);
- if (rr.isNull(2) || rr.isNull(3)) {
- System.err.println("Last node didn't have a reference, breaking");
- break;
- }
- prevKeyOne = rr.getLong(2);
- prevKeyTwo = rr.getLong(3);
- nodesCount++;
- } while ((headKeyOne != prevKeyOne && headKeyTwo != prevKeyTwo) && (nodesCount <
- maxNumNodes));
- }
-
- private RowResult nextNode(long keyOne, long keyTwo) throws Exception {
- KuduScanner.KuduScannerBuilder builder = client.newScannerBuilder(table);
- configureScannerForRandomRead(builder, table, keyOne, keyTwo);
-
- return getOneRowResult(builder.build());
- }
- }
-
- private static void configureScannerForRandomRead(AbstractKuduScannerBuilder builder,
- KuduTable table,
- long keyOne,
- long keyTwo) {
- PartialRow lowerBound = table.getSchema().newPartialRow();
- lowerBound.addLong(0, keyOne);
- lowerBound.addLong(1, keyTwo);
- builder.lowerBound(lowerBound);
-
- PartialRow upperBound = table.getSchema().newPartialRow();
- // Adding 1 since we want a single row, and the upper bound is exclusive.
- upperBound.addLong(0, keyOne + 1);
- upperBound.addLong(1, keyTwo + 1);
- builder.exclusiveUpperBound(upperBound);
- }
-
- private static String getTableName(Configuration conf) {
- return conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME);
- }
-
- private static String getHeadsTable(Configuration conf) {
- return conf.get(HEADS_TABLE_NAME_KEY, DEFAULT_HEADS_TABLE_NAME);
- }
-
- private static CINode getCINode(RowResult result, CINode node) {
-
- node.key = getStringFromKeys(result.getLong(0), result.getLong(1));
- if (result.isNull(2) || result.isNull(3)) {
- node.prev = "NO_REFERENCE";
- } else {
- node.prev = getStringFromKeys(result.getLong(2), result.getLong(3));
- }
- node.rowId = result.getInt(4);
- node.client = result.getString(5);
- node.updateCount = result.getInt(6);
- return node;
- }
-
- private static void printCINodeString(CINode node) {
- System.out.printf("%s:%s:%012d:%s:%s\n", node.key, node.prev, node.rowId, node.client,
- node.updateCount);
- }
-
- private static String getStringFromKeys(long key1, long key2) {
- return new StringBuilder().append(key1).append(",").append(key2).toString();
- }
-
- private static RowResult getOneRowResult(KuduScanner scanner) throws Exception {
- RowResultIterator rowResults;
- rowResults = scanner.nextRows();
- if (rowResults.getNumRows() == 0) {
- return null;
- }
- if (rowResults.getNumRows() > 1) {
- throw new Exception("Received too many rows from scanner " + scanner);
- }
- return rowResults.next();
- }
-
- private void usage() {
- System.err.println("Usage: " + this.getClass().getSimpleName() + " COMMAND [COMMAND options]");
- System.err.println(" where COMMAND is one of:");
- System.err.println("");
- System.err.println(" Generator A map only job that generates data.");
- System.err.println(" Verify A map reduce job that looks for holes");
- System.err.println(" Look at the counts after running");
- System.err.println(" REFERENCED and UNREFERENCED are ok");
- System.err.println(" any UNDEFINED counts are bad. Do not");
- System.err.println(" run at the same time as the Generator.");
- System.err.println(" Print A standalone program that prints nodes");
- System.err.println(" in the linked list.");
- System.err.println(" Loop A program to Loop through Generator and");
- System.err.println(" Verify steps");
- System.err.println(" Update A program to updade the nodes");
- /* System.err.println(" Delete A standalone program that deletes a");
- System.err.println(" single node.");*/
- System.err.println(" Walker A standalong program that starts ");
- System.err.println(" following a linked list");
- System.err.println("\t ");
- System.err.flush();
- }
-
- protected void processOptions(String[] args) {
- //get the class, run with the conf
- if (args.length < 1) {
- usage();
- throw new RuntimeException("Incorrect Number of args.");
- }
- toRun = args[0];
- otherArgs = Arrays.copyOfRange(args, 1, args.length);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Tool tool = null;
- processOptions(args);
- if (toRun.equals("Generator")) {
- tool = new Generator();
- } else if (toRun.equals("Verify")) {
- tool = new Verify();
- } else if (toRun.equals("Loop")) {
- Loop loop = new Loop();
- loop.it = this;
- tool = loop;
-
- } else if (toRun.equals("Print")) {
- tool = new Print();
- } else if (toRun.equals("Update")) {
- tool = new Updater();
- } else if (toRun.equals("Walker")) {
- tool = new Walker();
- } /*else if (toRun.equals("Delete")) {
- tool = new Delete();
- }*/ else {
- usage();
- throw new RuntimeException("Unknown arg");
- }
-
- return ToolRunner.run(getConf(), tool, otherArgs);
- }
-
- private static void setJobConf(Job job, int numMappers, long numNodes,
- Integer width, Integer wrapMultiplier) {
- job.getConfiguration().setInt(GENERATOR_NUM_MAPPERS_KEY, numMappers);
- job.getConfiguration().setLong(GENERATOR_NUM_ROWS_PER_MAP_KEY, numNodes);
- if (width != null) {
- job.getConfiguration().setInt(GENERATOR_WIDTH_KEY, width);
- }
- if (wrapMultiplier != null) {
- job.getConfiguration().setInt(GENERATOR_WRAP_KEY, wrapMultiplier);
- }
- }
-
- public static void main(String[] args) throws Exception {
- int ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
- System.exit(ret);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/RowCounter.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/RowCounter.java b/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/RowCounter.java
deleted file mode 100644
index 45e7837..0000000
--- a/java/kudu-client-tools/src/main/java/org/kududb/mapreduce/tools/RowCounter.java
+++ /dev/null
@@ -1,127 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.mapreduce.tools;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
-import org.kududb.client.RowResult;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-
-/**
- * Map-only job that counts all the rows in the provided table.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class RowCounter extends Configured implements Tool {
-
- static final String NAME = "rowcounter";
- static final String COLUMN_PROJECTION_KEY = "rowcounter.column.projection";
-
- /** Counter enumeration to count the actual rows. */
- public static enum Counters { ROWS }
-
- /**
- * Simple row counter
- */
- static class RowCounterMapper extends
- Mapper<NullWritable, RowResult, NullWritable, NullWritable> {
-
- @Override
- protected void map(NullWritable key, RowResult value, Context context) throws IOException,
- InterruptedException {
- context.getCounter(Counters.ROWS).increment(1);
- }
- }
-
- /**
- * Sets up the actual job.
- *
- * @param conf The current configuration.
- * @param args The command line parameters.
- * @return The newly created job.
- * @throws java.io.IOException When setting up the job fails.
- */
- @SuppressWarnings("deprecation")
- public static Job createSubmittableJob(Configuration conf, String[] args)
- throws IOException, ClassNotFoundException {
-
- String columnProjection = conf.get(COLUMN_PROJECTION_KEY);
-
- Class<RowCounterMapper> mapperClass = RowCounterMapper.class;
- String tableName = args[0];
-
- String jobName = NAME + "_" + tableName;
- Job job = new Job(conf, jobName);
- job.setJarByClass(mapperClass);
- job.setMapperClass(mapperClass);
- job.setNumReduceTasks(0);
- job.setOutputFormatClass(NullOutputFormat.class);
- new KuduTableMapReduceUtil.TableInputFormatConfiguratorWithCommandLineParser(
- job,
- tableName,
- columnProjection)
- .configure();
- return job;
- }
-
- /*
- * @param errorMsg Error message. Can be null.
- */
- private static void usage(final String errorMsg) {
- if (errorMsg != null && errorMsg.length() > 0) {
- System.err.println("ERROR: " + errorMsg);
- }
- String usage =
- "Usage: " + NAME + " <table.name>\n\n" +
- "Counts all the rows in the given table.\n" +
- "\n" +
- "Other options that may be specified with -D include:\n" +
- " -D" + COLUMN_PROJECTION_KEY + "=a,b,c - comma-separated list of columns to read " +
- "as part of the row count. By default, none are read so that the count is as fast " +
- "as possible. When specifying columns that are keys, they must be at the beginning" +
- ".\n" +
- CommandLineParser.getHelpSnippet();
-
- System.err.println(usage);
- }
-
- @Override
- public int run(String[] otherArgs) throws Exception {
- if (otherArgs.length != 1) {
- usage("Wrong number of arguments: " + otherArgs.length);
- return -1;
- }
- Job job = createSubmittableJob(getConf(), otherArgs);
- return job.waitForCompletion(true) ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- int status = ToolRunner.run(new RowCounter(), args);
- System.exit(status);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
new file mode 100644
index 0000000..d7b8352
--- /dev/null
+++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce.tools;
+
+import org.kududb.ColumnSchema;
+import org.kududb.Schema;
+import org.kududb.Type;
+import org.kududb.mapreduce.CommandLineParser;
+import org.kududb.mapreduce.HadoopTestingUtility;
+import org.kududb.client.BaseKuduTest;
+import org.kududb.client.CreateTableOptions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+
+public class ITImportCsv extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ ITImportCsv.class.getName() + "-" + System.currentTimeMillis();
+
+ private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
+
+ private static Schema schema;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+
+ ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4);
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32)
+ .key(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column2_d", Type.DOUBLE)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column3_s", Type.STRING)
+ .nullable(true)
+ .build());
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("column4_b", Type.BOOL)
+ .build());
+ schema = new Schema(columns);
+
+ createTable(TABLE_NAME, schema,
+ new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key")));
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ try {
+ BaseKuduTest.tearDownAfterClass();
+ } finally {
+ HADOOP_UTIL.cleanup();
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ Configuration conf = new Configuration();
+ String testHome =
+ HADOOP_UTIL.setupAndGetTestDir(ITImportCsv.class.getName(), conf).getAbsolutePath();
+
+ // Create a 2 lines input file
+ File data = new File(testHome, "data.csv");
+ writeCsvFile(data);
+
+ StringBuilder sb = new StringBuilder();
+ for (ColumnSchema col : schema.getColumns()) {
+ sb.append(col.getName());
+ sb.append(",");
+ }
+ sb.deleteCharAt(sb.length() - 1);
+ String[] args = new String[] {
+ "-D" + CommandLineParser.MASTER_ADDRESSES_KEY + "=" + getMasterAddresses(),
+ sb.toString(), TABLE_NAME, data.toString()};
+
+ GenericOptionsParser parser = new GenericOptionsParser(conf, args);
+ Job job = ImportCsv.createSubmittableJob(parser.getConfiguration(), parser.getRemainingArgs());
+ assertTrue("Test job did not end properly", job.waitForCompletion(true));
+
+ assertEquals(1, job.getCounters().findCounter(ImportCsv.Counters.BAD_LINES).getValue());
+
+ assertEquals(3, countRowsInScan(
+ client.newScannerBuilder(openTable(TABLE_NAME)).build()));
+ // TODO: should verify the actual returned rows, not just the count!
+ }
+
+ private void writeCsvFile(File data) throws IOException {
+ FileOutputStream fos = new FileOutputStream(data);
+ fos.write("1\t3\t2.3\tsome string\ttrue\n".getBytes());
+ fos.write("2\t5\t4.5\tsome more\tfalse\n".getBytes());
+ fos.write("3\t7\twait this is not a double\tbad row\ttrue\n".getBytes());
+ fos.write("4\t9\t10\ttrailing separator isn't bad mkay?\ttrue\t\n".getBytes());
+ fos.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java
new file mode 100644
index 0000000..311c5ee
--- /dev/null
+++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.kududb.mapreduce.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Test;
+import org.kududb.client.BaseKuduTest;
+import org.kududb.mapreduce.CommandLineParser;
+import org.kududb.mapreduce.HadoopTestingUtility;
+
+public class ITIntegrationTestBigLinkedList extends BaseKuduTest {
+
+ private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ try {
+ BaseKuduTest.tearDownAfterClass();
+ } finally {
+ HADOOP_UTIL.cleanup();
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ Configuration conf = new Configuration();
+ HADOOP_UTIL.setupAndGetTestDir(
+ ITIntegrationTestBigLinkedList.class.getName(),conf).getAbsolutePath();
+
+ String[] args = new String[] {
+ "-D" + CommandLineParser.MASTER_ADDRESSES_KEY + "=" + getMasterAddresses(),
+ "Loop",
+ "2", // Two iterations
+ "1", // 1 mapper
+ "2500", // 2.5k rows to insert
+ "1", // 1 tablet
+ "/tmp/itbll", // output dir
+ "1", // 1 reduce
+ "100", // create 100 columns
+ "25", // wrap them together after 25 rows
+ "0"
+ };
+ int ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
+ Assert.assertEquals(0, ret);
+
+ args[2] = "1"; // Just one iteration this time
+ args[10] = "5000"; // 2 * 2500 from previous run
+ ret = ToolRunner.run(new IntegrationTestBigLinkedList(), args);
+ Assert.assertEquals(0, ret);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java
new file mode 100644
index 0000000..52984bb
--- /dev/null
+++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.mapreduce.tools;
+
+import org.kududb.mapreduce.CommandLineParser;
+import org.kududb.mapreduce.HadoopTestingUtility;
+import org.kududb.client.BaseKuduTest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ITRowCounter extends BaseKuduTest {
+
+ private static final String TABLE_NAME =
+ ITRowCounter.class.getName() + "-" + System.currentTimeMillis();
+
+ private static final HadoopTestingUtility HADOOP_UTIL = new HadoopTestingUtility();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ BaseKuduTest.setUpBeforeClass();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ try {
+ BaseKuduTest.tearDownAfterClass();
+ } finally {
+ HADOOP_UTIL.cleanup();
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ Configuration conf = new Configuration();
+ HADOOP_UTIL.setupAndGetTestDir(ITRowCounter.class.getName(), conf).getAbsolutePath();
+
+ createFourTabletsTableWithNineRows(TABLE_NAME);
+
+ String[] args = new String[] {
+ "-D" + CommandLineParser.MASTER_ADDRESSES_KEY + "=" + getMasterAddresses(), TABLE_NAME};
+ GenericOptionsParser parser = new GenericOptionsParser(conf, args);
+ Job job = RowCounter.createSubmittableJob(parser.getConfiguration(), parser.getRemainingArgs());
+ assertTrue("Job did not end properly", job.waitForCompletion(true));
+
+ assertEquals(9, job.getCounters().findCounter(RowCounter.Counters.ROWS).getValue());
+ }
+}
[28/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
new file mode 100644
index 0000000..7699536
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
@@ -0,0 +1,894 @@
+/*
+ * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the StumbleUpon nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.Message;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.kududb.ColumnSchema;
+import org.kududb.Common;
+import org.kududb.Schema;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.tserver.Tserver;
+import org.kududb.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.kududb.tserver.Tserver.NewScanRequestPB;
+import static org.kududb.tserver.Tserver.ScanRequestPB;
+import static org.kududb.tserver.Tserver.ScanResponsePB;
+import static org.kududb.tserver.Tserver.TabletServerErrorPB;
+
+/**
+ * Creates a scanner to read data from Kudu.
+ * <p>
+ * This class is <strong>not synchronized</strong> as it's expected to be
+ * used from a single thread at a time. It's rarely (if ever?) useful to
+ * scan concurrently from a shared scanner using multiple threads. If you
+ * want to optimize large table scans using extra parallelism, create a few
+ * scanners and give each of them a partition of the table to scan. Or use
+ * MapReduce.
+ * <p>
+ * There's no method in this class to explicitly open the scanner. It will open
+ * itself automatically when you start scanning by calling {@link #nextRows()}.
+ * Also, the scanner will automatically call {@link #close} when it reaches the
+ * end key. If, however, you would like to stop scanning <i>before reaching the
+ * end key</i>, you <b>must</b> call {@link #close} before disposing of the scanner.
+ * Note that it's always safe to call {@link #close} on a scanner.
+ * <p>
+ * A {@code AsyncKuduScanner} is not re-usable. Should you want to scan the same rows
+ * or the same table again, you must create a new one.
+ *
+ * <h1>A note on passing {@code byte} arrays in argument</h1>
+ * None of the method that receive a {@code byte[]} in argument will copy it.
+ * For more info, please refer to the documentation of {@link KuduRpc}.
+ * <h1>A note on passing {@code String}s in argument</h1>
+ * All strings are assumed to use the platform's default charset.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class AsyncKuduScanner {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncKuduScanner.class);
+
+ /**
+ * The possible read modes for scanners.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public enum ReadMode {
+ /**
+ * When READ_LATEST is specified the server will always return committed writes at
+ * the time the request was received. This type of read does not return a snapshot
+ * timestamp and is not repeatable.
+ *
+ * In ACID terms this corresponds to Isolation mode: "Read Committed"
+ *
+ * This is the default mode.
+ */
+ READ_LATEST(Common.ReadMode.READ_LATEST),
+
+ /**
+ * When READ_AT_SNAPSHOT is specified the server will attempt to perform a read
+ * at the provided timestamp. If no timestamp is provided the server will take the
+ * current time as the snapshot timestamp. In this mode reads are repeatable, i.e.
+ * all future reads at the same timestamp will yield the same data. This is
+ * performed at the expense of waiting for in-flight transactions whose timestamp
+ * is lower than the snapshot's timestamp to complete, so it might incur a latency
+ * penalty.
+ *
+ * In ACID terms this, by itself, corresponds to Isolation mode "Repeatable
+ * Read". If all writes to the scanned tablet are made externally consistent,
+ * then this corresponds to Isolation mode "Strict-Serializable".
+ *
+ * Note: there currently "holes", which happen in rare edge conditions, by which writes
+ * are sometimes not externally consistent even when action was taken to make them so.
+ * In these cases Isolation may degenerate to mode "Read Committed". See KUDU-430.
+ */
+ READ_AT_SNAPSHOT(Common.ReadMode.READ_AT_SNAPSHOT);
+
+ private Common.ReadMode pbVersion;
+ ReadMode(Common.ReadMode pbVersion) {
+ this.pbVersion = pbVersion;
+ }
+
+ @InterfaceAudience.Private
+ public Common.ReadMode pbVersion() {
+ return this.pbVersion;
+ }
+ }
+
+ //////////////////////////
+ // Initial configurations.
+ //////////////////////////
+
+ private final AsyncKuduClient client;
+ private final KuduTable table;
+ private final Schema schema;
+
+ /**
+ * Map of column name to predicate.
+ */
+ private final Map<String, KuduPredicate> predicates;
+
+ /**
+ * Maximum number of bytes returned by the scanner, on each batch.
+ */
+ private final int batchSizeBytes;
+
+ /**
+ * The maximum number of rows to scan.
+ */
+ private final long limit;
+
+ /**
+ * The start partition key of the next tablet to scan.
+ *
+ * Each time the scan exhausts a tablet, this is updated to that tablet's end partition key.
+ */
+ private byte[] nextPartitionKey;
+
+ /**
+ * The end partition key of the last tablet to scan.
+ */
+ private final byte[] endPartitionKey;
+
+ /**
+ * Set in the builder. If it's not set by the user, it will default to EMPTY_ARRAY.
+ * It is then reset to the new start primary key of each tablet we open a scanner on as the scan
+ * moves from one tablet to the next.
+ */
+ private final byte[] startPrimaryKey;
+
+ /**
+ * Set in the builder. If it's not set by the user, it will default to EMPTY_ARRAY.
+ * It's never modified after that.
+ */
+ private final byte[] endPrimaryKey;
+
+ private final boolean prefetching;
+
+ private final boolean cacheBlocks;
+
+ private final ReadMode readMode;
+
+ private final Common.OrderMode orderMode;
+
+ private final long htTimestamp;
+
+ /////////////////////
+ // Runtime variables.
+ /////////////////////
+
+ private boolean closed = false;
+
+ private boolean hasMore = true;
+
+ /**
+ * The tabletSlice currently being scanned.
+ * If null, we haven't started scanning.
+ * If == DONE, then we're done scanning.
+ * Otherwise it contains a proper tabletSlice name, and we're currently scanning.
+ */
+ private AsyncKuduClient.RemoteTablet tablet;
+
+ /**
+ * This is the scanner ID we got from the TabletServer.
+ * It's generated randomly so any value is possible.
+ */
+ private byte[] scannerId;
+
+ /**
+ * The sequence ID of this call. The sequence ID should start at 0
+ * with the request for a new scanner, and after each successful request,
+ * the client should increment it by 1. When retrying a request, the client
+ * should _not_ increment this value. If the server detects that the client
+ * missed a chunk of rows from the middle of a scan, it will respond with an
+ * error.
+ */
+ private int sequenceId;
+
+ private Deferred<RowResultIterator> prefetcherDeferred;
+
+ private boolean inFirstTablet = true;
+
+ final long scanRequestTimeout;
+
+ private static final AtomicBoolean PARTITION_PRUNE_WARN = new AtomicBoolean(true);
+
+ AsyncKuduScanner(AsyncKuduClient client, KuduTable table, List<String> projectedNames,
+ List<Integer> projectedIndexes, ReadMode readMode, Common.OrderMode orderMode,
+ long scanRequestTimeout,
+ Map<String, KuduPredicate> predicates, long limit,
+ boolean cacheBlocks, boolean prefetching,
+ byte[] startPrimaryKey, byte[] endPrimaryKey,
+ byte[] startPartitionKey, byte[] endPartitionKey,
+ long htTimestamp, int batchSizeBytes) {
+ checkArgument(batchSizeBytes > 0, "Need a strictly positive number of bytes, " +
+ "got %s", batchSizeBytes);
+ checkArgument(limit > 0, "Need a strictly positive number for the limit, " +
+ "got %s", limit);
+ if (htTimestamp != AsyncKuduClient.NO_TIMESTAMP) {
+ checkArgument(htTimestamp >= 0, "Need non-negative number for the scan, " +
+ " timestamp got %s", htTimestamp);
+ checkArgument(readMode == ReadMode.READ_AT_SNAPSHOT, "When specifying a " +
+ "HybridClock timestamp, the read mode needs to be set to READ_AT_SNAPSHOT");
+ }
+ if (orderMode == Common.OrderMode.ORDERED) {
+ checkArgument(readMode == ReadMode.READ_AT_SNAPSHOT, "Returning rows in primary key order " +
+ "requires the read mode to be set to READ_AT_SNAPSHOT");
+ }
+
+ this.client = client;
+ this.table = table;
+ this.readMode = readMode;
+ this.orderMode = orderMode;
+ this.scanRequestTimeout = scanRequestTimeout;
+ this.predicates = predicates;
+ this.limit = limit;
+ this.cacheBlocks = cacheBlocks;
+ this.prefetching = prefetching;
+ this.startPrimaryKey = startPrimaryKey;
+ this.endPrimaryKey = endPrimaryKey;
+ this.htTimestamp = htTimestamp;
+ this.batchSizeBytes = batchSizeBytes;
+
+ if (!table.getPartitionSchema().isSimpleRangePartitioning() &&
+ (startPrimaryKey != AsyncKuduClient.EMPTY_ARRAY ||
+ endPrimaryKey != AsyncKuduClient.EMPTY_ARRAY) &&
+ PARTITION_PRUNE_WARN.getAndSet(false)) {
+ LOG.warn("Starting full table scan. " +
+ "In the future this scan may be automatically optimized with partition pruning.");
+ }
+
+ if (table.getPartitionSchema().isSimpleRangePartitioning()) {
+ // If the table is simple range partitioned, then the partition key space
+ // is isomorphic to the primary key space. We can potentially reduce the
+ // scan length by only scanning the intersection of the primary key range
+ // and the partition key range. This is a stop-gap until real partition
+ // pruning is in place that can work across any partitioning type.
+
+ if ((endPartitionKey.length != 0 && Bytes.memcmp(startPrimaryKey, endPartitionKey) >= 0) ||
+ (endPrimaryKey.length != 0 && Bytes.memcmp(startPartitionKey, endPrimaryKey) >= 0)) {
+ // The primary key range and the partition key range do not intersect;
+ // the scan will be empty.
+ this.nextPartitionKey = startPartitionKey;
+ this.endPartitionKey = endPartitionKey;
+ } else {
+ // Assign the scan's partition key range to the intersection of the
+ // primary key and partition key ranges.
+ if (Bytes.memcmp(startPartitionKey, startPrimaryKey) < 0) {
+ this.nextPartitionKey = startPrimaryKey;
+ } else {
+ this.nextPartitionKey = startPartitionKey;
+ }
+ if (endPrimaryKey.length != 0 && Bytes.memcmp(endPartitionKey, endPrimaryKey) > 0) {
+ this.endPartitionKey = endPrimaryKey;
+ } else {
+ this.endPartitionKey = endPartitionKey;
+ }
+ }
+ } else {
+ this.nextPartitionKey = startPartitionKey;
+ this.endPartitionKey = endPartitionKey;
+ }
+
+ // Map the column names to actual columns in the table schema.
+ // If the user set this to 'null', we scan all columns.
+ if (projectedNames != null) {
+ List<ColumnSchema> columns = new ArrayList<ColumnSchema>();
+ for (String columnName : projectedNames) {
+ ColumnSchema originalColumn = table.getSchema().getColumn(columnName);
+ columns.add(getStrippedColumnSchema(originalColumn));
+ }
+ this.schema = new Schema(columns);
+ } else if (projectedIndexes != null) {
+ List<ColumnSchema> columns = new ArrayList<ColumnSchema>();
+ for (Integer columnIndex : projectedIndexes) {
+ ColumnSchema originalColumn = table.getSchema().getColumnByIndex(columnIndex);
+ columns.add(getStrippedColumnSchema(originalColumn));
+ }
+ this.schema = new Schema(columns);
+ } else {
+ this.schema = table.getSchema();
+ }
+
+ // If any of the column predicates are of type None (the predicate is known
+ // to match no rows), then the scan can be short circuited without
+ // contacting any tablet servers.
+ boolean shortCircuit = false;
+ for (KuduPredicate predicate : this.predicates.values()) {
+ if (predicate.getType() == KuduPredicate.PredicateType.NONE) {
+ shortCircuit = true;
+ break;
+ }
+ }
+ if (shortCircuit) {
+ LOG.debug("Short circuiting scan with predicates: {}", predicates.values());
+ this.hasMore = false;
+ this.closed = true;
+ }
+ }
+
+ /**
+ * Clone the given column schema instance. The new instance will include only the name, type, and
+ * nullability of the passed one.
+ * @return a new column schema
+ */
+ private static ColumnSchema getStrippedColumnSchema(ColumnSchema columnToClone) {
+ return new ColumnSchema.ColumnSchemaBuilder(columnToClone.getName(), columnToClone.getType())
+ .nullable(columnToClone.isNullable())
+ .build();
+ }
+
+ /**
+ * Returns the maximum number of rows that this scanner was configured to return.
+ * @return a long representing the maximum number of rows that can be returned
+ */
+ public long getLimit() {
+ return this.limit;
+ }
+
+ /**
+ * Tells if the last rpc returned that there might be more rows to scan.
+ * @return true if there might be more data to scan, else false
+ */
+ public boolean hasMoreRows() {
+ return this.hasMore;
+ }
+
+ /**
+ * Returns if this scanner was configured to cache data blocks or not.
+ * @return true if this scanner will cache blocks, else else.
+ */
+ public boolean getCacheBlocks() {
+ return this.cacheBlocks;
+ }
+
+ /**
+ * Returns the maximum number of bytes returned by the scanner, on each batch.
+ * @return a long representing the maximum number of bytes that a scanner can receive at once
+ * from a tablet server
+ */
+ public long getBatchSizeBytes() {
+ return this.batchSizeBytes;
+ }
+
+ /**
+ * Returns the ReadMode for this scanner.
+ * @return the configured read mode for this scanner
+ */
+ public ReadMode getReadMode() {
+ return this.readMode;
+ }
+
+ private Common.OrderMode getOrderMode() {
+ return this.orderMode;
+ }
+
+ /**
+ * Returns the projection schema of this scanner. If specific columns were
+ * not specified during scanner creation, the table schema is returned.
+ * @return the projection schema for this scanner
+ */
+ public Schema getProjectionSchema() {
+ return this.schema;
+ }
+
+ long getSnapshotTimestamp() {
+ return this.htTimestamp;
+ }
+
+ /**
+ * Scans a number of rows.
+ * <p>
+ * Once this method returns {@code null} once (which indicates that this
+ * {@code Scanner} is done scanning), calling it again leads to an undefined
+ * behavior.
+ * @return a deferred list of rows.
+ */
+ public Deferred<RowResultIterator> nextRows() {
+ if (closed) { // We're already done scanning.
+ return Deferred.fromResult(null);
+ } else if (tablet == null) {
+
+ Callback<Deferred<RowResultIterator>, AsyncKuduScanner.Response> cb =
+ new Callback<Deferred<RowResultIterator>, Response>() {
+ @Override
+ public Deferred<RowResultIterator> call(Response resp) throws Exception {
+ if (!resp.more || resp.scanner_id == null) {
+ scanFinished();
+ return Deferred.fromResult(resp.data); // there might be data to return
+ }
+ scannerId = resp.scanner_id;
+ sequenceId++;
+ hasMore = resp.more;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Scanner " + Bytes.pretty(scannerId) + " opened on " + tablet);
+ }
+ return Deferred.fromResult(resp.data);
+ }
+ public String toString() {
+ return "scanner opened";
+ }
+ };
+
+ Callback<Deferred<RowResultIterator>, Exception> eb =
+ new Callback<Deferred<RowResultIterator>, Exception>() {
+ @Override
+ public Deferred<RowResultIterator> call(Exception e) throws Exception {
+ invalidate();
+ if (e instanceof NonCoveredRangeException) {
+ NonCoveredRangeException ncre = (NonCoveredRangeException) e;
+ nextPartitionKey = ncre.getNonCoveredRangeEnd();
+
+ // Stop scanning if the non-covered range is past the end partition key.
+ if (ncre.getNonCoveredRangeEnd().length == 0
+ || (endPartitionKey != AsyncKuduClient.EMPTY_ARRAY
+ && Bytes.memcmp(endPartitionKey, ncre.getNonCoveredRangeEnd()) <= 0)) {
+ hasMore = false;
+ closed = true; // the scanner is closed on the other side at this point
+ return Deferred.fromResult(RowResultIterator.empty());
+ }
+ nextPartitionKey = ncre.getNonCoveredRangeEnd();
+ scannerId = null;
+ sequenceId = 0;
+ return nextRows();
+ } else {
+ LOG.warn("Can not open scanner", e);
+ // Don't let the scanner think it's opened on this tablet.
+ return Deferred.fromError(e); // Let the error propogate.
+ }
+ }
+ public String toString() {
+ return "open scanner errback";
+ }
+ };
+
+ // We need to open the scanner first.
+ return client.sendRpcToTablet(getOpenRequest()).addCallbackDeferring(cb).addErrback(eb);
+ } else if (prefetching && prefetcherDeferred != null) {
+ // TODO KUDU-1260 - Check if this works and add a test
+ prefetcherDeferred.chain(new Deferred<RowResultIterator>().addCallback(prefetch));
+ return prefetcherDeferred;
+ }
+ final Deferred<RowResultIterator> d =
+ client.scanNextRows(this).addCallbacks(got_next_row, nextRowErrback());
+ if (prefetching) {
+ d.chain(new Deferred<RowResultIterator>().addCallback(prefetch));
+ }
+ return d;
+ }
+
+ private final Callback<RowResultIterator, RowResultIterator> prefetch =
+ new Callback<RowResultIterator, RowResultIterator>() {
+ @Override
+ public RowResultIterator call(RowResultIterator arg) throws Exception {
+ if (hasMoreRows()) {
+ prefetcherDeferred = client.scanNextRows(AsyncKuduScanner.this).addCallbacks
+ (got_next_row, nextRowErrback());
+ }
+ return null;
+ }
+ };
+
+ /**
+ * Singleton callback to handle responses of "next" RPCs.
+ * This returns an {@code ArrayList<ArrayList<KeyValue>>} (possibly inside a
+ * deferred one).
+ */
+ private final Callback<RowResultIterator, Response> got_next_row =
+ new Callback<RowResultIterator, Response>() {
+ public RowResultIterator call(final Response resp) {
+ if (!resp.more) { // We're done scanning this tablet.
+ scanFinished();
+ return resp.data;
+ }
+ sequenceId++;
+ hasMore = resp.more;
+ //LOG.info("Scan.next is returning rows: " + resp.data.getNumRows());
+ return resp.data;
+ }
+ public String toString() {
+ return "get nextRows response";
+ }
+ };
+
+ /**
+ * Creates a new errback to handle errors while trying to get more rows.
+ */
+ private final Callback<Exception, Exception> nextRowErrback() {
+ return new Callback<Exception, Exception>() {
+ public Exception call(final Exception error) {
+ final AsyncKuduClient.RemoteTablet old_tablet = tablet; // Save before invalidate().
+ String message = old_tablet + " pretends to not know " + AsyncKuduScanner.this;
+ LOG.warn(message, error);
+ invalidate(); // If there was an error, don't assume we're still OK.
+ return error; // Let the error propagate.
+ }
+ public String toString() {
+ return "NextRow errback";
+ }
+ };
+ }
+
+ void scanFinished() {
+ Partition partition = tablet.getPartition();
+ // Stop scanning if we have scanned until or past the end partition key.
+ if (partition.isEndPartition()
+ || (this.endPartitionKey != AsyncKuduClient.EMPTY_ARRAY
+ && Bytes.memcmp(this.endPartitionKey, partition.getPartitionKeyEnd()) <= 0)) {
+ hasMore = false;
+ closed = true; // the scanner is closed on the other side at this point
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Done scanning tablet {} for partition {} with scanner id {}",
+ tablet.getTabletIdAsString(), tablet.getPartition(), Bytes.pretty(scannerId));
+ }
+ nextPartitionKey = partition.getPartitionKeyEnd();
+ scannerId = null;
+ sequenceId = 0;
+ invalidate();
+ }
+
+ /**
+ * Closes this scanner (don't forget to call this when you're done with it!).
+ * <p>
+ * Closing a scanner already closed has no effect. The deferred returned
+ * will be called back immediately.
+ * @return A deferred object that indicates the completion of the request.
+ * The {@link Object} can be null, a RowResultIterator if there was data left
+ * in the scanner, or an Exception.
+ */
+ public Deferred<RowResultIterator> close() {
+ if (closed) {
+ return Deferred.fromResult(null);
+ }
+ final Deferred<RowResultIterator> d =
+ client.closeScanner(this).addCallback(closedCallback()); // TODO errBack ?
+ return d;
+ }
+
+ /** Callback+Errback invoked when the TabletServer closed our scanner. */
+ private Callback<RowResultIterator, Response> closedCallback() {
+ return new Callback<RowResultIterator, Response>() {
+ public RowResultIterator call(Response response) {
+ closed = true;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Scanner " + Bytes.pretty(scannerId) + " closed on "
+ + tablet);
+ }
+ tablet = null;
+ scannerId = "client debug closed".getBytes(); // Make debugging easier.
+ return response == null ? null : response.data;
+ }
+ public String toString() {
+ return "scanner closed";
+ }
+ };
+ }
+
+ public String toString() {
+ final String tablet = this.tablet == null ? "null" : this.tablet.getTabletIdAsString();
+ final StringBuilder buf = new StringBuilder();
+ buf.append("KuduScanner(table=");
+ buf.append(table.getName());
+ buf.append(", tablet=").append(tablet);
+ buf.append(", scannerId=").append(Bytes.pretty(scannerId));
+ buf.append(", scanRequestTimeout=").append(scanRequestTimeout);
+ buf.append(')');
+ return buf.toString();
+ }
+
+ // ---------------------- //
+ // Package private stuff. //
+ // ---------------------- //
+
+ KuduTable table() {
+ return table;
+ }
+
+ /**
+ * Sets the name of the tabletSlice that's hosting {@code this.start_key}.
+ * @param tablet The tabletSlice we're currently supposed to be scanning.
+ */
+ void setTablet(final AsyncKuduClient.RemoteTablet tablet) {
+ this.tablet = tablet;
+ }
+
+ /**
+ * Invalidates this scanner and makes it assume it's no longer opened.
+ * When a TabletServer goes away while we're scanning it, or some other type
+ * of access problem happens, this method should be called so that the
+ * scanner will have to re-locate the TabletServer and re-open itself.
+ */
+ void invalidate() {
+ tablet = null;
+ }
+
+ /**
+ * Returns the tabletSlice currently being scanned, if any.
+ */
+ AsyncKuduClient.RemoteTablet currentTablet() {
+ return tablet;
+ }
+
+ /**
+ * Returns an RPC to open this scanner.
+ */
+ KuduRpc<Response> getOpenRequest() {
+ checkScanningNotStarted();
+ // This is the only point where we know we haven't started scanning and where the scanner
+ // should be fully configured
+ if (this.inFirstTablet) {
+ this.inFirstTablet = false;
+ }
+ return new ScanRequest(table, State.OPENING);
+ }
+
+ /**
+ * Returns an RPC to fetch the next rows.
+ */
+ KuduRpc<Response> getNextRowsRequest() {
+ return new ScanRequest(table, State.NEXT);
+ }
+
+ /**
+ * Returns an RPC to close this scanner.
+ */
+ KuduRpc<Response> getCloseRequest() {
+ return new ScanRequest(table, State.CLOSING);
+ }
+
+ /**
+ * Throws an exception if scanning already started.
+ * @throws IllegalStateException if scanning already started.
+ */
+ private void checkScanningNotStarted() {
+ if (tablet != null) {
+ throw new IllegalStateException("scanning already started");
+ }
+ }
+
+ /**
+ * Helper object that contains all the info sent by a TS after a Scan request.
+ */
+ static final class Response {
+ /** The ID associated with the scanner that issued the request. */
+ private final byte[] scanner_id;
+ /** The actual payload of the response. */
+ private final RowResultIterator data;
+
+ /**
+ * If false, the filter we use decided there was no more data to scan.
+ * In this case, the server has automatically closed the scanner for us,
+ * so we don't need to explicitly close it.
+ */
+ private final boolean more;
+
+ Response(final byte[] scanner_id,
+ final RowResultIterator data,
+ final boolean more) {
+ this.scanner_id = scanner_id;
+ this.data = data;
+ this.more = more;
+ }
+
+ public String toString() {
+ return "AsyncKuduScanner$Response(scannerId=" + Bytes.pretty(scanner_id)
+ + ", data=" + data + ", more=" + more + ") ";
+ }
+ }
+
+ private enum State {
+ OPENING,
+ NEXT,
+ CLOSING
+ }
+
+ /**
+ * RPC sent out to fetch the next rows from the TabletServer.
+ */
+ private final class ScanRequest extends KuduRpc<Response> {
+
+ State state;
+
+ ScanRequest(KuduTable table, State state) {
+ super(table);
+ this.state = state;
+ this.setTimeoutMillis(scanRequestTimeout);
+ }
+
+ @Override
+ String serviceName() { return TABLET_SERVER_SERVICE_NAME; }
+
+ @Override
+ String method() {
+ return "Scan";
+ }
+
+ @Override
+ Collection<Integer> getRequiredFeatures() {
+ if (predicates.isEmpty()) {
+ return ImmutableList.of();
+ } else {
+ return ImmutableList.of(Tserver.TabletServerFeatures.COLUMN_PREDICATES_VALUE);
+ }
+ }
+
+ /** Serializes this request. */
+ ChannelBuffer serialize(Message header) {
+ final ScanRequestPB.Builder builder = ScanRequestPB.newBuilder();
+ switch (state) {
+ case OPENING:
+ // Save the tablet in the AsyncKuduScanner. This kind of a kludge but it really
+ // is the easiest way.
+ AsyncKuduScanner.this.tablet = super.getTablet();
+ NewScanRequestPB.Builder newBuilder = NewScanRequestPB.newBuilder();
+ newBuilder.setLimit(limit); // currently ignored
+ newBuilder.addAllProjectedColumns(ProtobufHelper.schemaToListPb(schema));
+ newBuilder.setTabletId(ZeroCopyLiteralByteString.wrap(tablet.getTabletIdAsBytes()));
+ newBuilder.setReadMode(AsyncKuduScanner.this.getReadMode().pbVersion());
+ newBuilder.setOrderMode(AsyncKuduScanner.this.getOrderMode());
+ newBuilder.setCacheBlocks(cacheBlocks);
+ // if the last propagated timestamp is set send it with the scan
+ if (table.getAsyncClient().getLastPropagatedTimestamp() != AsyncKuduClient.NO_TIMESTAMP) {
+ newBuilder.setPropagatedTimestamp(table.getAsyncClient().getLastPropagatedTimestamp());
+ }
+ newBuilder.setReadMode(AsyncKuduScanner.this.getReadMode().pbVersion());
+
+ // if the mode is set to read on snapshot sent the snapshot timestamp
+ if (AsyncKuduScanner.this.getReadMode() == ReadMode.READ_AT_SNAPSHOT &&
+ AsyncKuduScanner.this.getSnapshotTimestamp() != AsyncKuduClient.NO_TIMESTAMP) {
+ newBuilder.setSnapTimestamp(AsyncKuduScanner.this.getSnapshotTimestamp());
+ }
+
+ if (AsyncKuduScanner.this.startPrimaryKey != AsyncKuduClient.EMPTY_ARRAY &&
+ AsyncKuduScanner.this.startPrimaryKey.length > 0) {
+ newBuilder.setStartPrimaryKey(ZeroCopyLiteralByteString.copyFrom(startPrimaryKey));
+ }
+
+ if (AsyncKuduScanner.this.endPrimaryKey != AsyncKuduClient.EMPTY_ARRAY &&
+ AsyncKuduScanner.this.endPrimaryKey.length > 0) {
+ newBuilder.setStopPrimaryKey(ZeroCopyLiteralByteString.copyFrom(endPrimaryKey));
+ }
+
+ for (KuduPredicate pred : predicates.values()) {
+ newBuilder.addColumnPredicates(pred.toPB());
+ }
+ builder.setNewScanRequest(newBuilder.build())
+ .setBatchSizeBytes(batchSizeBytes);
+ break;
+ case NEXT:
+ setTablet(AsyncKuduScanner.this.tablet);
+ builder.setScannerId(ZeroCopyLiteralByteString.wrap(scannerId))
+ .setCallSeqId(AsyncKuduScanner.this.sequenceId)
+ .setBatchSizeBytes(batchSizeBytes);
+ break;
+ case CLOSING:
+ setTablet(AsyncKuduScanner.this.tablet);
+ builder.setScannerId(ZeroCopyLiteralByteString.wrap(scannerId))
+ .setBatchSizeBytes(0)
+ .setCloseScanner(true);
+ }
+
+ ScanRequestPB request = builder.build();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending scan req: " + request.toString());
+ }
+
+ return toChannelBuffer(header, request);
+ }
+
+ @Override
+ Pair<Response, Object> deserialize(final CallResponse callResponse,
+ String tsUUID) throws Exception {
+ ScanResponsePB.Builder builder = ScanResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), builder);
+ ScanResponsePB resp = builder.build();
+ final byte[] id = resp.getScannerId().toByteArray();
+ TabletServerErrorPB error = resp.hasError() ? resp.getError() : null;
+
+ if (error != null && error.getCode().equals(TabletServerErrorPB.Code.TABLET_NOT_FOUND)) {
+ if (state == State.OPENING) {
+ // Doing this will trigger finding the new location.
+ return new Pair<Response, Object>(null, error);
+ } else {
+ Status statusIncomplete = Status.Incomplete("Cannot continue scanning, " +
+ "the tablet has moved and this isn't a fault tolerant scan");
+ throw new NonRecoverableException(statusIncomplete);
+ }
+ }
+ RowResultIterator iterator = RowResultIterator.makeRowResultIterator(
+ deadlineTracker.getElapsedMillis(), tsUUID, schema, resp.getData(),
+ callResponse);
+
+ boolean hasMore = resp.getHasMoreResults();
+ if (id.length != 0 && scannerId != null && !Bytes.equals(scannerId, id)) {
+ Status statusIllegalState = Status.IllegalState("Scan RPC response was for scanner"
+ + " ID " + Bytes.pretty(id) + " but we expected "
+ + Bytes.pretty(scannerId));
+ throw new NonRecoverableException(statusIllegalState);
+ }
+ Response response = new Response(id, iterator, hasMore);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(response.toString());
+ }
+ return new Pair<Response, Object>(response, error);
+ }
+
+ public String toString() {
+ return "ScanRequest(scannerId=" + Bytes.pretty(scannerId)
+ + (tablet != null? ", tabletSlice=" + tablet.getTabletIdAsString() : "")
+ + ", attempt=" + attempt + ')';
+ }
+
+ @Override
+ public byte[] partitionKey() {
+ // This key is used to lookup where the request needs to go
+ return nextPartitionKey;
+ }
+ }
+
+ /**
+ * A Builder class to build {@link AsyncKuduScanner}.
+ * Use {@link AsyncKuduClient#newScannerBuilder} in order to get a builder instance.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public static class AsyncKuduScannerBuilder
+ extends AbstractKuduScannerBuilder<AsyncKuduScannerBuilder, AsyncKuduScanner> {
+
+ AsyncKuduScannerBuilder(AsyncKuduClient client, KuduTable table) {
+ super(client, table);
+ }
+
+ /**
+ * Builds an {@link AsyncKuduScanner} using the passed configurations.
+ * @return a new {@link AsyncKuduScanner}
+ */
+ public AsyncKuduScanner build() {
+ return new AsyncKuduScanner(
+ client, table, projectedColumnNames, projectedColumnIndexes, readMode, orderMode,
+ scanRequestTimeout, predicates, limit, cacheBlocks,
+ prefetching, lowerBoundPrimaryKey, upperBoundPrimaryKey,
+ lowerBoundPartitionKey, upperBoundPartitionKey,
+ htTimestamp, batchSizeBytes);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java
new file mode 100644
index 0000000..0290ee7
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java
@@ -0,0 +1,856 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Range;
+import com.stumbleupon.async.Callback;
+import com.stumbleupon.async.Deferred;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.annotations.InterfaceStability;
+import org.kududb.util.AsyncUtil;
+import org.kududb.util.Slice;
+import org.jboss.netty.util.Timeout;
+import org.jboss.netty.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.GuardedBy;
+import javax.annotation.concurrent.NotThreadSafe;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+
+/**
+ * A AsyncKuduSession belongs to a specific AsyncKuduClient, and represents a context in
+ * which all read/write data access should take place. Within a session,
+ * multiple operations may be accumulated and batched together for better
+ * efficiency. Settings like timeouts, priorities, and trace IDs are also set
+ * per session.<p>
+ *
+ * AsyncKuduSession is separate from KuduTable because a given batch or transaction
+ * may span multiple tables. This is particularly important in the future when
+ * we add ACID support, but even in the context of batching, we may be able to
+ * coalesce writes to different tables hosted on the same server into the same
+ * RPC.<p>
+ *
+ * AsyncKuduSession is separate from AsyncKuduClient because, in a multi-threaded
+ * application, different threads may need to concurrently execute
+ * transactions. Similar to a JDBC "session", transaction boundaries will be
+ * delineated on a per-session basis -- in between a "BeginTransaction" and
+ * "Commit" call on a given session, all operations will be part of the same
+ * transaction. Meanwhile another concurrent Session object can safely run
+ * non-transactional work or other transactions without interfering.<p>
+ *
+ * Therefore, this class is <b>not</b> thread-safe.<p>
+ *
+ * Additionally, there is a guarantee that writes from different sessions do not
+ * get batched together into the same RPCs -- this means that latency-sensitive
+ * clients can run through the same AsyncKuduClient object as throughput-oriented
+ * clients, perhaps by setting the latency-sensitive session's timeouts low and
+ * priorities high. Without the separation of batches, a latency-sensitive
+ * single-row insert might get batched along with 10MB worth of inserts from the
+ * batch writer, thus delaying the response significantly.<p>
+ *
+ * Though we currently do not have transactional support, users will be forced
+ * to use a AsyncKuduSession to instantiate reads as well as writes. This will make
+ * it more straight-forward to add RW transactions in the future without
+ * significant modifications to the API.<p>
+ *
+ * Timeouts are handled differently depending on the flush mode.
+ * With AUTO_FLUSH_SYNC, the timeout is set on each apply()'d operation.
+ * With AUTO_FLUSH_BACKGROUND and MANUAL_FLUSH, the timeout is assigned to a whole batch of
+ * operations upon flush()'ing. It means that in a situation with a timeout of 500ms and a flush
+ * interval of 1000ms, an operation can be outstanding for up to 1500ms before being timed out.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@NotThreadSafe
+public class AsyncKuduSession implements SessionConfiguration {
+
+ public static final Logger LOG = LoggerFactory.getLogger(AsyncKuduSession.class);
+ private static final Range<Float> PERCENTAGE_RANGE = Range.closed(0.0f, 1.0f);
+
+ private final AsyncKuduClient client;
+ private final Random randomizer = new Random();
+ private final ErrorCollector errorCollector;
+ private int interval = 1000;
+ private int mutationBufferSpace = 1000; // TODO express this in terms of data size.
+ private float mutationBufferLowWatermarkPercentage = 0.5f;
+ private int mutationBufferLowWatermark;
+ private FlushMode flushMode;
+ private ExternalConsistencyMode consistencyMode;
+ private long timeoutMs;
+
+ /**
+ * Protects internal state from concurrent access. {@code AsyncKuduSession} is not threadsafe
+ * from the application's perspective, but because internally async timers and async flushing
+ * tasks may access the session concurrently with the application, synchronization is still
+ * needed.
+ */
+ private final Object monitor = new Object();
+
+ /**
+ * Tracks the currently active buffer.
+ *
+ * When in mode {@link FlushMode#AUTO_FLUSH_BACKGROUND} or {@link FlushMode#AUTO_FLUSH_SYNC},
+ * {@code AsyncKuduSession} uses double buffering to improve write throughput. While the
+ * application is {@link #apply}ing operations to one buffer (the {@code activeBuffer}), the
+ * second buffer is either being flushed, or if it has already been flushed, it waits in the
+ * {@link #inactiveBuffers} queue. When the currently active buffer is flushed,
+ * {@code activeBuffer} is set to {@code null}. On the next call to {@code apply}, an inactive
+ * buffer is taken from {@code inactiveBuffers} and made the new active buffer. If both
+ * buffers are still flushing, then the {@code apply} call throws {@link PleaseThrottleException}.
+ */
+ @GuardedBy("monitor")
+ private Buffer activeBuffer;
+
+ /**
+ * The buffers. May either be active (pointed to by {@link #activeBuffer},
+ * inactive (in the {@link #inactiveBuffers}) queue, or flushing.
+ */
+ private final Buffer bufferA = new Buffer();
+ private final Buffer bufferB = new Buffer();
+
+ /**
+ * Queue containing flushed, inactive buffers. May be accessed from callbacks (I/O threads).
+ * We restrict the session to only two buffers, so {@link BlockingQueue#add} can
+ * be used without chance of failure.
+ */
+ private final BlockingQueue<Buffer> inactiveBuffers = new ArrayBlockingQueue<>(2, false);
+
+ /**
+ * Deferred used to notify on flush events. Atomically swapped and completed every time a buffer
+ * is flushed. This can be used to notify handlers of {@link PleaseThrottleException} that more
+ * capacity may be available in the active buffer.
+ */
+ private final AtomicReference<Deferred<Void>> flushNotification =
+ new AtomicReference<>(new Deferred<Void>());
+
+ /**
+ * Tracks whether the session has been closed.
+ */
+ private volatile boolean closed = false;
+
+ private boolean ignoreAllDuplicateRows = false;
+
+ /**
+ * Package-private constructor meant to be used via AsyncKuduClient
+ * @param client client that creates this session
+ */
+ AsyncKuduSession(AsyncKuduClient client) {
+ this.client = client;
+ flushMode = FlushMode.AUTO_FLUSH_SYNC;
+ consistencyMode = CLIENT_PROPAGATED;
+ timeoutMs = client.getDefaultOperationTimeoutMs();
+ inactiveBuffers.add(bufferA);
+ inactiveBuffers.add(bufferB);
+ errorCollector = new ErrorCollector(mutationBufferSpace);
+ setMutationBufferLowWatermark(this.mutationBufferLowWatermarkPercentage);
+ }
+
+ @Override
+ public FlushMode getFlushMode() {
+ return this.flushMode;
+ }
+
+ @Override
+ public void setFlushMode(FlushMode flushMode) {
+ if (hasPendingOperations()) {
+ throw new IllegalArgumentException("Cannot change flush mode when writes are buffered");
+ }
+ this.flushMode = flushMode;
+ }
+
+ @Override
+ public void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode) {
+ if (hasPendingOperations()) {
+ throw new IllegalArgumentException("Cannot change consistency mode "
+ + "when writes are buffered");
+ }
+ this.consistencyMode = consistencyMode;
+ }
+
+ @Override
+ public void setMutationBufferSpace(int size) {
+ if (hasPendingOperations()) {
+ throw new IllegalArgumentException("Cannot change the buffer" +
+ " size when operations are buffered");
+ }
+ this.mutationBufferSpace = size;
+ // Reset the low watermark, using the same percentage as before.
+ setMutationBufferLowWatermark(mutationBufferLowWatermarkPercentage);
+ }
+
+ @Override
+ public void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage) {
+ if (hasPendingOperations()) {
+ throw new IllegalArgumentException("Cannot change the buffer" +
+ " low watermark when operations are buffered");
+ } else if (!PERCENTAGE_RANGE.contains(mutationBufferLowWatermarkPercentage)) {
+ throw new IllegalArgumentException("The low watermark must be between 0 and 1 inclusively");
+ }
+ this.mutationBufferLowWatermarkPercentage = mutationBufferLowWatermarkPercentage;
+ this.mutationBufferLowWatermark =
+ (int)(this.mutationBufferLowWatermarkPercentage * mutationBufferSpace);
+ }
+
+ /**
+ * Lets us set a specific seed for tests
+ * @param seed
+ */
+ @VisibleForTesting
+ void setRandomSeed(long seed) {
+ this.randomizer.setSeed(seed);
+ }
+
+ @Override
+ public void setFlushInterval(int interval) {
+ this.interval = interval;
+ }
+
+ @Override
+ public void setTimeoutMillis(long timeout) {
+ this.timeoutMs = timeout;
+ }
+
+ @Override
+ public long getTimeoutMillis() {
+ return this.timeoutMs;
+ }
+
+ @Override
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public boolean isIgnoreAllDuplicateRows() {
+ return ignoreAllDuplicateRows;
+ }
+
+ @Override
+ public void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows) {
+ this.ignoreAllDuplicateRows = ignoreAllDuplicateRows;
+ }
+
+ @Override
+ public int countPendingErrors() {
+ return errorCollector.countErrors();
+ }
+
+ @Override
+ public RowErrorsAndOverflowStatus getPendingErrors() {
+ return errorCollector.getErrors();
+ }
+
+ /**
+ * Flushes the buffered operations and marks this session as closed.
+ * See the javadoc on {@link #flush()} on how to deal with exceptions coming out of this method.
+ * @return a Deferred whose callback chain will be invoked when.
+ * everything that was buffered at the time of the call has been flushed.
+ */
+ public Deferred<List<OperationResponse>> close() {
+ if (!closed) {
+ closed = true;
+ client.removeSession(this);
+ }
+ return flush();
+ }
+
+ /**
+ * Returns a buffer to the inactive queue after flushing.
+ * @param buffer the buffer to return to the inactive queue.
+ */
+ private void queueBuffer(Buffer buffer) {
+ buffer.callbackFlushNotification();
+ Deferred<Void> localFlushNotification = flushNotification.getAndSet(new Deferred<Void>());
+ inactiveBuffers.add(buffer);
+ localFlushNotification.callback(null);
+ }
+
+ /**
+ * Callback which waits for all tablet location lookups to complete, groups all operations into
+ * batches by tablet, and dispatches them. When all of the batches are complete, a deferred is
+ * fired and the buffer is added to the inactive queue.
+ */
+ private final class TabletLookupCB implements Callback<Void, Object> {
+ private final AtomicInteger lookupsOutstanding;
+ private final Buffer buffer;
+ private final Deferred<List<BatchResponse>> deferred;
+
+ public TabletLookupCB(Buffer buffer, Deferred<List<BatchResponse>> deferred) {
+ this.lookupsOutstanding = new AtomicInteger(buffer.getOperations().size());
+ this.buffer = buffer;
+ this.deferred = deferred;
+ }
+
+ @Override
+ public Void call(Object _void) throws Exception {
+ if (lookupsOutstanding.decrementAndGet() != 0) return null;
+
+ // The final tablet lookup is complete. Batch all of the buffered
+ // operations into their respective tablet, and then send the batches.
+
+ // Group the operations by tablet.
+ Map<Slice, Batch> batches = new HashMap<>();
+ List<OperationResponse> opsFailedInLookup = new ArrayList<>();
+
+ for (BufferedOperation bufferedOp : buffer.getOperations()) {
+ Operation operation = bufferedOp.getOperation();
+ if (bufferedOp.tabletLookupFailed()) {
+ Exception failure = bufferedOp.getTabletLookupFailure();
+ RowError error;
+ if (failure instanceof NonCoveredRangeException) {
+ // TODO: this should be something different than NotFound so that
+ // applications can distinguish from updates on missing rows.
+ error = new RowError(Status.NotFound(failure.getMessage()), operation);
+ } else {
+ LOG.warn("unexpected tablet lookup failure for operation {}", operation, failure);
+ error = new RowError(Status.RuntimeError(failure.getMessage()), operation);
+ }
+ OperationResponse response = new OperationResponse(0, null, 0, operation, error);
+ // Add the row error to the error collector if the session is in background flush mode,
+ // and complete the operation's deferred with the error response. The ordering between
+ // adding to the error collector and completing the deferred should not matter since
+ // applications should be using one or the other method for error handling, not both.
+ if (flushMode == FlushMode.AUTO_FLUSH_BACKGROUND) {
+ errorCollector.addError(error);
+ }
+ operation.callback(response);
+ opsFailedInLookup.add(response);
+ continue;
+ }
+ LocatedTablet tablet = bufferedOp.getTablet();
+ Slice tabletId = new Slice(tablet.getTabletId());
+
+ Batch batch = batches.get(tabletId);
+ if (batch == null) {
+ batch = new Batch(operation.getTable(), tablet, ignoreAllDuplicateRows);
+ batches.put(tabletId, batch);
+ }
+ batch.add(operation);
+ }
+
+ List<Deferred<BatchResponse>> batchResponses = new ArrayList<>(batches.size() + 1);
+ if (!opsFailedInLookup.isEmpty()) {
+ batchResponses.add(Deferred.fromResult(new BatchResponse(opsFailedInLookup)));
+ }
+
+ for (Batch batch : batches.values()) {
+ if (timeoutMs != 0) {
+ batch.deadlineTracker.reset();
+ batch.setTimeoutMillis(timeoutMs);
+ }
+ addBatchCallbacks(batch);
+ batchResponses.add(client.sendRpcToTablet(batch));
+ }
+
+ // On completion of all batches, fire the completion deferred, and add the buffer
+ // back to the inactive buffers queue. This frees it up for new inserts.
+ AsyncUtil.addBoth(
+ Deferred.group(batchResponses),
+ new Callback<Void, Object>() {
+ @Override
+ public Void call(Object responses) {
+ queueBuffer(buffer);
+ deferred.callback(responses);
+ return null;
+ }
+ });
+
+ return null;
+ }
+ }
+
+ /**
+ * Flush buffered writes.
+ * @return a {@link Deferred} whose callback chain will be invoked when all applied operations at
+ * the time of the call have been flushed.
+ */
+ public Deferred<List<OperationResponse>> flush() {
+ Buffer buffer;
+ Deferred<Void> nonActiveBufferFlush;
+ synchronized (monitor) {
+ nonActiveBufferFlush = getNonActiveFlushNotification();
+ buffer = activeBuffer;
+ activeBuffer = null;
+ }
+
+ final Deferred<List<OperationResponse>> activeBufferFlush = buffer == null ?
+ Deferred.<List<OperationResponse>>fromResult(ImmutableList.<OperationResponse>of()) :
+ doFlush(buffer);
+
+ return AsyncUtil.addBothDeferring(nonActiveBufferFlush,
+ new Callback<Deferred<List<OperationResponse>>, Object>() {
+ @Override
+ public Deferred<List<OperationResponse>> call(Object arg) {
+ return activeBufferFlush;
+ }
+ });
+ }
+
+ /**
+ * Flushes a write buffer. This method takes ownership of the buffer, no other concurrent access
+ * is allowed.
+ *
+ * @param buffer the buffer to flush, must not be modified once passed to this method
+ * @return the operation responses
+ */
+ private Deferred<List<OperationResponse>> doFlush(Buffer buffer) {
+ LOG.debug("flushing buffer: {}", buffer);
+ if (buffer.getOperations().isEmpty()) {
+ // no-op.
+ return Deferred.<List<OperationResponse>>fromResult(ImmutableList.<OperationResponse>of());
+ }
+
+ Deferred<List<BatchResponse>> batchResponses = new Deferred<>();
+ Callback<Void, Object> tabletLookupCB = new TabletLookupCB(buffer, batchResponses);
+
+ for (BufferedOperation bufferedOperation : buffer.getOperations()) {
+ AsyncUtil.addBoth(bufferedOperation.getTabletLookup(), tabletLookupCB);
+ }
+
+ return batchResponses.addCallback(ConvertBatchToListOfResponsesCB.getInstance());
+ }
+
+ /**
+ * Callback used to send a list of OperationResponse instead of BatchResponse since the
+ * latter is an implementation detail.
+ */
+ private static class ConvertBatchToListOfResponsesCB implements Callback<List<OperationResponse>,
+ List<BatchResponse>> {
+ private static final ConvertBatchToListOfResponsesCB INSTANCE =
+ new ConvertBatchToListOfResponsesCB();
+ @Override
+ public List<OperationResponse> call(List<BatchResponse> batchResponses) throws Exception {
+ // First compute the size of the union of all the lists so that we don't trigger expensive
+ // list growths while adding responses to it.
+ int size = 0;
+ for (BatchResponse batchResponse : batchResponses) {
+ size += batchResponse.getIndividualResponses().size();
+ }
+
+ ArrayList<OperationResponse> responses = new ArrayList<>(size);
+ for (BatchResponse batchResponse : batchResponses) {
+ responses.addAll(batchResponse.getIndividualResponses());
+ }
+
+ return responses;
+ }
+ @Override
+ public String toString() {
+ return "ConvertBatchToListOfResponsesCB";
+ }
+ public static ConvertBatchToListOfResponsesCB getInstance() {
+ return INSTANCE;
+ }
+ }
+
+ @Override
+ public boolean hasPendingOperations() {
+ synchronized (monitor) {
+ return activeBuffer == null ? inactiveBuffers.size() < 2 :
+ activeBuffer.getOperations().size() > 0 || !inactiveBufferAvailable();
+ }
+ }
+
+ /**
+ * Apply the given operation.
+ * The behavior of this function depends on the current flush mode. Regardless
+ * of flush mode, however, Apply may begin to perform processing in the background
+ * for the call (e.g looking up the tablet, etc).
+ * @param operation operation to apply
+ * @return a Deferred to track this operation
+ * @throws KuduException if an error happens or {@link PleaseThrottleException} is triggered
+ */
+ public Deferred<OperationResponse> apply(final Operation operation) throws KuduException {
+ Preconditions.checkNotNull(operation, "Can not apply a null operation");
+
+ // Freeze the row so that the client can not concurrently modify it while it is in flight.
+ operation.getRow().freeze();
+
+ // If immediate flush mode, send the operation directly.
+ if (flushMode == FlushMode.AUTO_FLUSH_SYNC) {
+ if (timeoutMs != 0) {
+ operation.setTimeoutMillis(timeoutMs);
+ }
+ operation.setExternalConsistencyMode(this.consistencyMode);
+ operation.setIgnoreAllDuplicateRows(ignoreAllDuplicateRows);
+ return client.sendRpcToTablet(operation);
+ }
+
+ // Kick off a location lookup.
+ Deferred<LocatedTablet> tablet = client.getTabletLocation(operation.getTable(),
+ operation.partitionKey(),
+ timeoutMs);
+
+ // Holds a buffer that should be flushed outside the synchronized block, if necessary.
+ Buffer fullBuffer = null;
+ try {
+ synchronized (monitor) {
+ if (activeBuffer == null) {
+ // If the active buffer is null then we recently flushed. Check if there
+ // is an inactive buffer available to replace as the active.
+ if (inactiveBufferAvailable()) {
+ refreshActiveBuffer();
+ } else {
+ Status statusServiceUnavailable =
+ Status.ServiceUnavailable("All buffers are currently flushing");
+ // This can happen if the user writes into a buffer, flushes it, writes
+ // into the second, flushes it, and immediately tries to write again.
+ throw new PleaseThrottleException(statusServiceUnavailable,
+ null, operation, flushNotification.get());
+ }
+ }
+
+ if (flushMode == FlushMode.MANUAL_FLUSH) {
+ if (activeBuffer.getOperations().size() < mutationBufferSpace) {
+ activeBuffer.getOperations().add(new BufferedOperation(tablet, operation));
+ } else {
+ Status statusIllegalState =
+ Status.IllegalState("MANUAL_FLUSH is enabled but the buffer is too big");
+ throw new NonRecoverableException(statusIllegalState);
+ }
+ } else {
+ assert flushMode == FlushMode.AUTO_FLUSH_BACKGROUND;
+ int activeBufferSize = activeBuffer.getOperations().size();
+
+ if (activeBufferSize >= mutationBufferSpace) {
+ // Save the active buffer into fullBuffer so that it gets flushed when we leave this
+ // synchronized block.
+ fullBuffer = activeBuffer;
+ activeBuffer = null;
+ activeBufferSize = 0;
+ if (inactiveBufferAvailable()) {
+ refreshActiveBuffer();
+ } else {
+ Status statusServiceUnavailable =
+ Status.ServiceUnavailable("All buffers are currently flushing");
+ throw new PleaseThrottleException(statusServiceUnavailable,
+ null, operation, flushNotification.get());
+ }
+ }
+
+ if (mutationBufferLowWatermark < mutationBufferSpace && // low watermark is enabled
+ activeBufferSize >= mutationBufferLowWatermark && // buffer is over low water mark
+ !inactiveBufferAvailable()) { // no inactive buffers
+
+ // Check if we are over the low water mark.
+ int randomWatermark = activeBufferSize + 1 +
+ randomizer.nextInt(mutationBufferSpace -
+ mutationBufferLowWatermark);
+
+ if (randomWatermark > mutationBufferSpace) {
+ Status statusServiceUnavailable =
+ Status.ServiceUnavailable("The previous buffer hasn't been flushed and the " +
+ "current buffer is over the low watermark, please retry later");
+ throw new PleaseThrottleException(statusServiceUnavailable,
+ null, operation, flushNotification.get());
+ }
+ }
+
+ activeBuffer.getOperations().add(new BufferedOperation(tablet, operation));
+
+ if (activeBufferSize + 1 >= mutationBufferSpace && inactiveBufferAvailable()) {
+ // If the operation filled the buffer, then flush it.
+ Preconditions.checkState(fullBuffer == null);
+ fullBuffer = activeBuffer;
+ activeBuffer = null;
+ activeBufferSize = 0;
+ } else if (activeBufferSize == 0) {
+ // If this is the first operation in the buffer, start a background flush timer.
+ client.newTimeout(activeBuffer.getFlusherTask(), interval);
+ }
+ }
+ }
+ } finally {
+ // Flush the buffer outside of the synchronized block, if required.
+ if (fullBuffer != null) {
+ doFlush(fullBuffer);
+ }
+ }
+ return operation.getDeferred();
+ }
+
+ /**
+ * Returns {@code true} if there is an inactive buffer available.
+ * @return true if there is currently an inactive buffer available
+ */
+ private boolean inactiveBufferAvailable() {
+ return inactiveBuffers.peek() != null;
+ }
+
+ /**
+ * Refreshes the active buffer. This should only be called after a
+ * {@link #flush()} when the active buffer is {@code null}, there is an
+ * inactive buffer available (see {@link #inactiveBufferAvailable()}, and
+ * {@link #monitor} is locked.
+ */
+ @GuardedBy("monitor")
+ private void refreshActiveBuffer() {
+ Preconditions.checkState(activeBuffer == null);
+ activeBuffer = inactiveBuffers.remove();
+ activeBuffer.reset();
+ }
+
+ /**
+ * Returns a flush notification for the currently non-active buffers.
+ * This is used during manual {@link #flush} calls to ensure that all buffers (not just the active
+ * buffer) are fully flushed before completing.
+ */
+ @GuardedBy("monitor")
+ private Deferred<Void> getNonActiveFlushNotification() {
+ final Deferred<Void> notificationA = bufferA.getFlushNotification();
+ final Deferred<Void> notificationB = bufferB.getFlushNotification();
+ if (activeBuffer == null) {
+ // Both buffers are either flushing or inactive.
+ return AsyncUtil.addBothDeferring(notificationA, new Callback<Deferred<Void>, Object>() {
+ @Override
+ public Deferred<Void> call(Object _obj) throws Exception {
+ return notificationB;
+ }
+ });
+ } else if (activeBuffer == bufferA) {
+ return notificationB;
+ } else {
+ return notificationA;
+ }
+ }
+
+ /**
+ * Creates callbacks to handle a multi-put and adds them to the request.
+ * @param request the request for which we must handle the response
+ */
+ private void addBatchCallbacks(final Batch request) {
+ final class BatchCallback implements Callback<BatchResponse, BatchResponse> {
+ public BatchResponse call(final BatchResponse response) {
+ LOG.trace("Got a Batch response for {} rows", request.operations.size());
+ if (response.getWriteTimestamp() != 0) {
+ AsyncKuduSession.this.client.updateLastPropagatedTimestamp(response.getWriteTimestamp());
+ }
+
+ // Send individualized responses to all the operations in this batch.
+ for (OperationResponse operationResponse : response.getIndividualResponses()) {
+ operationResponse.getOperation().callback(operationResponse);
+ if (flushMode == FlushMode.AUTO_FLUSH_BACKGROUND && operationResponse.hasRowError()) {
+ errorCollector.addError(operationResponse.getRowError());
+ }
+ }
+
+ return response;
+ }
+
+ @Override
+ public String toString() {
+ return "apply batch response";
+ }
+ }
+
+ final class BatchErrCallback implements Callback<Exception, Exception> {
+ @Override
+ public Exception call(Exception e) {
+ // Send the same exception to all the operations.
+ for (Operation operation : request.operations) {
+ operation.errback(e);
+ }
+ return e;
+ }
+ @Override
+ public String toString() {
+ return "apply batch error response";
+ }
+ }
+
+ request.getDeferred().addCallbacks(new BatchCallback(), new BatchErrCallback());
+ }
+
+ /**
+ * A FlusherTask is created for each active buffer in mode
+ * {@link FlushMode#AUTO_FLUSH_BACKGROUND}.
+ */
+ private final class FlusherTask implements TimerTask {
+ public void run(final Timeout timeout) {
+ Buffer buffer = null;
+ synchronized (monitor) {
+ if (activeBuffer == null) {
+ return;
+ }
+ if (activeBuffer.getFlusherTask() == this) {
+ buffer = activeBuffer;
+ activeBuffer = null;
+ }
+ }
+
+ if (buffer != null) {
+ doFlush(buffer);
+ }
+ }
+ }
+
+ /**
+ * The {@code Buffer} consists of a list of operations, an optional pointer to a flush task,
+ * and a flush notification.
+ *
+ * The {@link #flusherTask} is used in mode {@link FlushMode#AUTO_FLUSH_BACKGROUND} to point to
+ * the background flusher task assigned to the buffer when it becomes active and the first
+ * operation is applied to it. When the flusher task executes after the timeout, it checks
+ * that the currently active buffer's flusher task points to itself before executing the flush.
+ * This protects against the background task waking up after one or more manual flushes and
+ * attempting to flush the active buffer.
+ *
+ * The {@link #flushNotification} deferred is used when executing manual {@link #flush}es to
+ * ensure that non-active buffers are fully flushed. {@code flushNotification} is completed
+ * when this buffer is successfully flushed. When the buffer is promoted from inactive to active,
+ * the deferred is replaced with a new one to indicate that the buffer is not yet flushed.
+ *
+ * Buffer is externally synchronized. When the active buffer, {@link #monitor}
+ * synchronizes access to it.
+ */
+ private final class Buffer {
+ private final List<BufferedOperation> operations = new ArrayList<>();
+
+ private FlusherTask flusherTask = null;
+
+ private Deferred<Void> flushNotification = Deferred.fromResult(null);
+
+ public List<BufferedOperation> getOperations() {
+ return operations;
+ }
+
+ @GuardedBy("monitor")
+ public FlusherTask getFlusherTask() {
+ if (flusherTask == null) {
+ flusherTask = new FlusherTask();
+ }
+ return flusherTask;
+ }
+
+ /**
+ * Returns a {@link Deferred} which will be completed when this buffer is flushed. If the buffer
+ * is inactive (its flush is complete and it has been enqueued into {@link #inactiveBuffers}),
+ * then the deferred will already be complete.
+ */
+ public Deferred<Void> getFlushNotification() {
+ return flushNotification;
+ }
+
+ /**
+ * Completes the buffer's flush notification. Should be called when the buffer has been
+ * successfully flushed.
+ */
+ public void callbackFlushNotification() {
+ LOG.trace("buffer flush notification fired: {}", this);
+ flushNotification.callback(null);
+ }
+
+ /**
+ * Resets the buffer's internal state. Should be called when the buffer is promoted from
+ * inactive to active.
+ */
+ @GuardedBy("monitor")
+ public void reset() {
+ LOG.trace("buffer reset: {}", this);
+ operations.clear();
+ flushNotification = new Deferred<>();
+ flusherTask = null;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("operations", operations.size())
+ .add("flusherTask", flusherTask)
+ .add("flushNotification", flushNotification)
+ .toString();
+ }
+ }
+
+ /**
+ * Container class holding all the state associated with a buffered operation.
+ */
+ private static final class BufferedOperation {
+ /** Holds either a {@link LocatedTablet} or the failure exception if the lookup failed. */
+ private Object tablet = null;
+ private final Deferred<Void> tabletLookup;
+ private final Operation operation;
+
+ public BufferedOperation(Deferred<LocatedTablet> tablet,
+ Operation operation) {
+ tabletLookup = AsyncUtil.addBoth(tablet, new Callback<Void, Object>() {
+ @Override
+ public Void call(final Object tablet) {
+ BufferedOperation.this.tablet = tablet;
+ return null;
+ }
+ });
+ this.operation = Preconditions.checkNotNull(operation);
+ }
+
+ /**
+ * @return {@code true} if the tablet lookup failed.
+ */
+ public boolean tabletLookupFailed() {
+ return !(tablet instanceof LocatedTablet);
+ }
+
+ /**
+ * @return the located tablet
+ * @throws ClassCastException if the tablet lookup failed,
+ * check with {@link #tabletLookupFailed} before calling
+ */
+ public LocatedTablet getTablet() {
+ return (LocatedTablet) tablet;
+ }
+
+ /**
+ * @return the cause of the failed lookup
+ * @throws ClassCastException if the tablet lookup succeeded,
+ * check with {@link #tabletLookupFailed} before calling
+ */
+ public Exception getTabletLookupFailure() {
+ return (Exception) tablet;
+ }
+
+ public Deferred<Void> getTabletLookup() {
+ return tabletLookup;
+ }
+
+ public Operation getOperation() {
+ return operation;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("tablet", tablet)
+ .add("operation", operation)
+ .toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java
new file mode 100644
index 0000000..ed1d870
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.protobuf.Message;
+import com.google.protobuf.ZeroCopyLiteralByteString;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.kududb.WireProtocol;
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.client.Statistics.Statistic;
+import org.kududb.client.Statistics.TabletStatistics;
+import org.kududb.tserver.Tserver;
+import org.kududb.tserver.Tserver.TabletServerErrorPB;
+import org.kududb.util.Pair;
+import org.kududb.util.Slice;
+
+/**
+ * Used internally to group Operations for a single tablet together before sending to the tablet
+ * server.
+ */
+@InterfaceAudience.Private
+class Batch extends KuduRpc<BatchResponse> {
+
+ /** Holds batched operations. */
+ final List<Operation> operations = new ArrayList<>();
+
+ /** The tablet this batch will be routed to. */
+ private final LocatedTablet tablet;
+
+ /**
+ * This size will be set when serialize is called. It stands for the size of rows in all
+ * operations in this batch.
+ */
+ private long rowOperationsSizeBytes = 0;
+
+ /** See {@link SessionConfiguration#setIgnoreAllDuplicateRows(boolean)} */
+ private final boolean ignoreAllDuplicateRows;
+
+
+ Batch(KuduTable table, LocatedTablet tablet, boolean ignoreAllDuplicateRows) {
+ super(table);
+ this.ignoreAllDuplicateRows = ignoreAllDuplicateRows;
+ this.tablet = tablet;
+ }
+
+ /**
+ * Returns the bytes size of this batch's row operations after serialization.
+ * @return size in bytes
+ * @throws IllegalStateException thrown if this RPC hasn't been serialized eg sent to a TS
+ */
+ long getRowOperationsSizeBytes() {
+ if (this.rowOperationsSizeBytes == 0) {
+ throw new IllegalStateException("This row hasn't been serialized yet");
+ }
+ return this.rowOperationsSizeBytes;
+ }
+
+ public void add(Operation operation) {
+ assert Bytes.memcmp(operation.partitionKey(),
+ tablet.getPartition().getPartitionKeyStart()) >= 0 &&
+ (tablet.getPartition().getPartitionKeyEnd().length == 0 ||
+ Bytes.memcmp(operation.partitionKey(),
+ tablet.getPartition().getPartitionKeyEnd()) < 0);
+
+ operations.add(operation);
+ }
+
+ @Override
+ ChannelBuffer serialize(Message header) {
+ final Tserver.WriteRequestPB.Builder builder = Operation.createAndFillWriteRequestPB(operations);
+ rowOperationsSizeBytes = builder.getRowOperations().getRows().size() +
+ builder.getRowOperations().getIndirectData().size();
+ builder.setTabletId(ZeroCopyLiteralByteString.wrap(getTablet().getTabletIdAsBytes()));
+ builder.setExternalConsistencyMode(externalConsistencyMode.pbVersion());
+ return toChannelBuffer(header, builder.build());
+ }
+
+ @Override
+ String serviceName() {
+ return TABLET_SERVER_SERVICE_NAME;
+ }
+
+ @Override
+ String method() {
+ return Operation.METHOD;
+ }
+
+ @Override
+ Pair<BatchResponse, Object> deserialize(CallResponse callResponse,
+ String tsUUID) throws Exception {
+ Tserver.WriteResponsePB.Builder builder = Tserver.WriteResponsePB.newBuilder();
+ readProtobuf(callResponse.getPBMessage(), builder);
+
+ List<Tserver.WriteResponsePB.PerRowErrorPB> errorsPB = builder.getPerRowErrorsList();
+ if (ignoreAllDuplicateRows) {
+ boolean allAlreadyPresent = true;
+ for (Tserver.WriteResponsePB.PerRowErrorPB errorPB : errorsPB) {
+ if (errorPB.getError().getCode() != WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT) {
+ allAlreadyPresent = false;
+ break;
+ }
+ }
+ if (allAlreadyPresent) {
+ errorsPB = Collections.emptyList();
+ }
+ }
+
+ BatchResponse response = new BatchResponse(deadlineTracker.getElapsedMillis(), tsUUID,
+ builder.getTimestamp(), errorsPB, operations);
+
+ if (injectedError != null) {
+ if (injectedlatencyMs > 0) {
+ try {
+ Thread.sleep(injectedlatencyMs);
+ } catch (InterruptedException e) {
+ }
+ }
+ return new Pair<BatchResponse, Object>(response, injectedError);
+ }
+
+ return new Pair<BatchResponse, Object>(response, builder.hasError() ? builder.getError() : null);
+ }
+
+ @Override
+ public byte[] partitionKey() {
+ return tablet.getPartition().getPartitionKeyStart();
+ }
+
+ @Override
+ boolean isRequestTracked() {
+ return true;
+ }
+
+ @Override
+ void updateStatistics(Statistics statistics, BatchResponse response) {
+ Slice tabletId = this.getTablet().getTabletId();
+ String tableName = this.getTable().getName();
+ TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId);
+ if (response == null) {
+ tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, operations.size());
+ tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1);
+ return;
+ }
+ tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1);
+ for (OperationResponse opResponse : response.getIndividualResponses()) {
+ if (opResponse.hasRowError()) {
+ tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1);
+ } else {
+ tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1);
+ }
+ }
+ tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationsSizeBytes());
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("operations", operations.size())
+ .add("tablet", tablet)
+ .add("ignoreAllDuplicateRows", ignoreAllDuplicateRows)
+ .toString();
+ }
+
+ private static TabletServerErrorPB injectedError;
+ private static int injectedlatencyMs;
+
+ /**
+ * Inject tablet server side error for Batch rpc related tests.
+ * @param error error response from tablet server
+ * @param latencyMs blocks response handling thread for some time to simulate
+ * write latency
+ */
+ @VisibleForTesting
+ static void injectTabletServerErrorAndLatency(TabletServerErrorPB error, int latencyMs) {
+ injectedError = error;
+ injectedlatencyMs = latencyMs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java
new file mode 100644
index 0000000..f67153b
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.kududb.client;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.kududb.annotations.InterfaceAudience;
+import org.kududb.tserver.Tserver;
+
+/**
+ * Response type for Batch (which is used internally by AsyncKuduSession).
+ * Provides the Hybrid Time write timestamp returned by the Tablet Server.
+ */
+@InterfaceAudience.Private
+public class BatchResponse extends KuduRpcResponse {
+
+ private final long writeTimestamp;
+ private final List<RowError> rowErrors;
+ private final List<OperationResponse> individualResponses;
+
+ /**
+ * Package-private constructor to be used by the RPCs.
+ * @param elapsedMillis time in milliseconds since RPC creation to now
+ * @param writeTimestamp HT's write timestamp
+ * @param errorsPB a list of row errors, can be empty
+ * @param operations the list of operations which created this response
+ */
+ BatchResponse(long elapsedMillis, String tsUUID, long writeTimestamp,
+ List<Tserver.WriteResponsePB.PerRowErrorPB> errorsPB,
+ List<Operation> operations) {
+ super(elapsedMillis, tsUUID);
+ this.writeTimestamp = writeTimestamp;
+ individualResponses = new ArrayList<>(operations.size());
+ if (errorsPB.isEmpty()) {
+ rowErrors = Collections.emptyList();
+ } else {
+ rowErrors = new ArrayList<>(errorsPB.size());
+ }
+
+ // Populate the list of individual row responses and the list of row errors. Not all the rows
+ // maybe have errors, but 'errorsPB' contains them in the same order as the operations that
+ // were sent.
+ int currentErrorIndex = 0;
+ Operation currentOperation;
+ for (int i = 0; i < operations.size(); i++) {
+ RowError rowError = null;
+ currentOperation = operations.get(i);
+ if (currentErrorIndex < errorsPB.size() &&
+ errorsPB.get(currentErrorIndex).getRowIndex() == i) {
+ rowError = RowError.fromRowErrorPb(errorsPB.get(currentErrorIndex),
+ currentOperation, tsUUID);
+ rowErrors.add(rowError);
+ currentErrorIndex++;
+ }
+ individualResponses.add(
+ new OperationResponse(currentOperation.deadlineTracker.getElapsedMillis(), tsUUID,
+ writeTimestamp, currentOperation, rowError));
+ }
+ assert (rowErrors.size() == errorsPB.size());
+ assert (individualResponses.size() == operations.size());
+ }
+
+ BatchResponse(List<OperationResponse> individualResponses) {
+ super(0, null);
+ writeTimestamp = 0;
+ rowErrors = ImmutableList.of();
+ this.individualResponses = individualResponses;
+ }
+
+ /**
+ * Gives the write timestamp that was returned by the Tablet Server.
+ * @return a timestamp in milliseconds, 0 if the external consistency mode set in AsyncKuduSession
+ * wasn't CLIENT_PROPAGATED
+ */
+ public long getWriteTimestamp() {
+ return writeTimestamp;
+ }
+
+ /**
+ * Package-private method to get the individual responses.
+ * @return a list of OperationResponses
+ */
+ List<OperationResponse> getIndividualResponses() {
+ return individualResponses;
+ }
+
+}
[19/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
deleted file mode 100644
index 7699536..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
+++ /dev/null
@@ -1,894 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import com.google.protobuf.Message;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static org.kududb.tserver.Tserver.NewScanRequestPB;
-import static org.kududb.tserver.Tserver.ScanRequestPB;
-import static org.kududb.tserver.Tserver.ScanResponsePB;
-import static org.kududb.tserver.Tserver.TabletServerErrorPB;
-
-/**
- * Creates a scanner to read data from Kudu.
- * <p>
- * This class is <strong>not synchronized</strong> as it's expected to be
- * used from a single thread at a time. It's rarely (if ever?) useful to
- * scan concurrently from a shared scanner using multiple threads. If you
- * want to optimize large table scans using extra parallelism, create a few
- * scanners and give each of them a partition of the table to scan. Or use
- * MapReduce.
- * <p>
- * There's no method in this class to explicitly open the scanner. It will open
- * itself automatically when you start scanning by calling {@link #nextRows()}.
- * Also, the scanner will automatically call {@link #close} when it reaches the
- * end key. If, however, you would like to stop scanning <i>before reaching the
- * end key</i>, you <b>must</b> call {@link #close} before disposing of the scanner.
- * Note that it's always safe to call {@link #close} on a scanner.
- * <p>
- * A {@code AsyncKuduScanner} is not re-usable. Should you want to scan the same rows
- * or the same table again, you must create a new one.
- *
- * <h1>A note on passing {@code byte} arrays in argument</h1>
- * None of the method that receive a {@code byte[]} in argument will copy it.
- * For more info, please refer to the documentation of {@link KuduRpc}.
- * <h1>A note on passing {@code String}s in argument</h1>
- * All strings are assumed to use the platform's default charset.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class AsyncKuduScanner {
-
- private static final Logger LOG = LoggerFactory.getLogger(AsyncKuduScanner.class);
-
- /**
- * The possible read modes for scanners.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public enum ReadMode {
- /**
- * When READ_LATEST is specified the server will always return committed writes at
- * the time the request was received. This type of read does not return a snapshot
- * timestamp and is not repeatable.
- *
- * In ACID terms this corresponds to Isolation mode: "Read Committed"
- *
- * This is the default mode.
- */
- READ_LATEST(Common.ReadMode.READ_LATEST),
-
- /**
- * When READ_AT_SNAPSHOT is specified the server will attempt to perform a read
- * at the provided timestamp. If no timestamp is provided the server will take the
- * current time as the snapshot timestamp. In this mode reads are repeatable, i.e.
- * all future reads at the same timestamp will yield the same data. This is
- * performed at the expense of waiting for in-flight transactions whose timestamp
- * is lower than the snapshot's timestamp to complete, so it might incur a latency
- * penalty.
- *
- * In ACID terms this, by itself, corresponds to Isolation mode "Repeatable
- * Read". If all writes to the scanned tablet are made externally consistent,
- * then this corresponds to Isolation mode "Strict-Serializable".
- *
- * Note: there currently "holes", which happen in rare edge conditions, by which writes
- * are sometimes not externally consistent even when action was taken to make them so.
- * In these cases Isolation may degenerate to mode "Read Committed". See KUDU-430.
- */
- READ_AT_SNAPSHOT(Common.ReadMode.READ_AT_SNAPSHOT);
-
- private Common.ReadMode pbVersion;
- ReadMode(Common.ReadMode pbVersion) {
- this.pbVersion = pbVersion;
- }
-
- @InterfaceAudience.Private
- public Common.ReadMode pbVersion() {
- return this.pbVersion;
- }
- }
-
- //////////////////////////
- // Initial configurations.
- //////////////////////////
-
- private final AsyncKuduClient client;
- private final KuduTable table;
- private final Schema schema;
-
- /**
- * Map of column name to predicate.
- */
- private final Map<String, KuduPredicate> predicates;
-
- /**
- * Maximum number of bytes returned by the scanner, on each batch.
- */
- private final int batchSizeBytes;
-
- /**
- * The maximum number of rows to scan.
- */
- private final long limit;
-
- /**
- * The start partition key of the next tablet to scan.
- *
- * Each time the scan exhausts a tablet, this is updated to that tablet's end partition key.
- */
- private byte[] nextPartitionKey;
-
- /**
- * The end partition key of the last tablet to scan.
- */
- private final byte[] endPartitionKey;
-
- /**
- * Set in the builder. If it's not set by the user, it will default to EMPTY_ARRAY.
- * It is then reset to the new start primary key of each tablet we open a scanner on as the scan
- * moves from one tablet to the next.
- */
- private final byte[] startPrimaryKey;
-
- /**
- * Set in the builder. If it's not set by the user, it will default to EMPTY_ARRAY.
- * It's never modified after that.
- */
- private final byte[] endPrimaryKey;
-
- private final boolean prefetching;
-
- private final boolean cacheBlocks;
-
- private final ReadMode readMode;
-
- private final Common.OrderMode orderMode;
-
- private final long htTimestamp;
-
- /////////////////////
- // Runtime variables.
- /////////////////////
-
- private boolean closed = false;
-
- private boolean hasMore = true;
-
- /**
- * The tabletSlice currently being scanned.
- * If null, we haven't started scanning.
- * If == DONE, then we're done scanning.
- * Otherwise it contains a proper tabletSlice name, and we're currently scanning.
- */
- private AsyncKuduClient.RemoteTablet tablet;
-
- /**
- * This is the scanner ID we got from the TabletServer.
- * It's generated randomly so any value is possible.
- */
- private byte[] scannerId;
-
- /**
- * The sequence ID of this call. The sequence ID should start at 0
- * with the request for a new scanner, and after each successful request,
- * the client should increment it by 1. When retrying a request, the client
- * should _not_ increment this value. If the server detects that the client
- * missed a chunk of rows from the middle of a scan, it will respond with an
- * error.
- */
- private int sequenceId;
-
- private Deferred<RowResultIterator> prefetcherDeferred;
-
- private boolean inFirstTablet = true;
-
- final long scanRequestTimeout;
-
- private static final AtomicBoolean PARTITION_PRUNE_WARN = new AtomicBoolean(true);
-
- AsyncKuduScanner(AsyncKuduClient client, KuduTable table, List<String> projectedNames,
- List<Integer> projectedIndexes, ReadMode readMode, Common.OrderMode orderMode,
- long scanRequestTimeout,
- Map<String, KuduPredicate> predicates, long limit,
- boolean cacheBlocks, boolean prefetching,
- byte[] startPrimaryKey, byte[] endPrimaryKey,
- byte[] startPartitionKey, byte[] endPartitionKey,
- long htTimestamp, int batchSizeBytes) {
- checkArgument(batchSizeBytes > 0, "Need a strictly positive number of bytes, " +
- "got %s", batchSizeBytes);
- checkArgument(limit > 0, "Need a strictly positive number for the limit, " +
- "got %s", limit);
- if (htTimestamp != AsyncKuduClient.NO_TIMESTAMP) {
- checkArgument(htTimestamp >= 0, "Need non-negative number for the scan, " +
- " timestamp got %s", htTimestamp);
- checkArgument(readMode == ReadMode.READ_AT_SNAPSHOT, "When specifying a " +
- "HybridClock timestamp, the read mode needs to be set to READ_AT_SNAPSHOT");
- }
- if (orderMode == Common.OrderMode.ORDERED) {
- checkArgument(readMode == ReadMode.READ_AT_SNAPSHOT, "Returning rows in primary key order " +
- "requires the read mode to be set to READ_AT_SNAPSHOT");
- }
-
- this.client = client;
- this.table = table;
- this.readMode = readMode;
- this.orderMode = orderMode;
- this.scanRequestTimeout = scanRequestTimeout;
- this.predicates = predicates;
- this.limit = limit;
- this.cacheBlocks = cacheBlocks;
- this.prefetching = prefetching;
- this.startPrimaryKey = startPrimaryKey;
- this.endPrimaryKey = endPrimaryKey;
- this.htTimestamp = htTimestamp;
- this.batchSizeBytes = batchSizeBytes;
-
- if (!table.getPartitionSchema().isSimpleRangePartitioning() &&
- (startPrimaryKey != AsyncKuduClient.EMPTY_ARRAY ||
- endPrimaryKey != AsyncKuduClient.EMPTY_ARRAY) &&
- PARTITION_PRUNE_WARN.getAndSet(false)) {
- LOG.warn("Starting full table scan. " +
- "In the future this scan may be automatically optimized with partition pruning.");
- }
-
- if (table.getPartitionSchema().isSimpleRangePartitioning()) {
- // If the table is simple range partitioned, then the partition key space
- // is isomorphic to the primary key space. We can potentially reduce the
- // scan length by only scanning the intersection of the primary key range
- // and the partition key range. This is a stop-gap until real partition
- // pruning is in place that can work across any partitioning type.
-
- if ((endPartitionKey.length != 0 && Bytes.memcmp(startPrimaryKey, endPartitionKey) >= 0) ||
- (endPrimaryKey.length != 0 && Bytes.memcmp(startPartitionKey, endPrimaryKey) >= 0)) {
- // The primary key range and the partition key range do not intersect;
- // the scan will be empty.
- this.nextPartitionKey = startPartitionKey;
- this.endPartitionKey = endPartitionKey;
- } else {
- // Assign the scan's partition key range to the intersection of the
- // primary key and partition key ranges.
- if (Bytes.memcmp(startPartitionKey, startPrimaryKey) < 0) {
- this.nextPartitionKey = startPrimaryKey;
- } else {
- this.nextPartitionKey = startPartitionKey;
- }
- if (endPrimaryKey.length != 0 && Bytes.memcmp(endPartitionKey, endPrimaryKey) > 0) {
- this.endPartitionKey = endPrimaryKey;
- } else {
- this.endPartitionKey = endPartitionKey;
- }
- }
- } else {
- this.nextPartitionKey = startPartitionKey;
- this.endPartitionKey = endPartitionKey;
- }
-
- // Map the column names to actual columns in the table schema.
- // If the user set this to 'null', we scan all columns.
- if (projectedNames != null) {
- List<ColumnSchema> columns = new ArrayList<ColumnSchema>();
- for (String columnName : projectedNames) {
- ColumnSchema originalColumn = table.getSchema().getColumn(columnName);
- columns.add(getStrippedColumnSchema(originalColumn));
- }
- this.schema = new Schema(columns);
- } else if (projectedIndexes != null) {
- List<ColumnSchema> columns = new ArrayList<ColumnSchema>();
- for (Integer columnIndex : projectedIndexes) {
- ColumnSchema originalColumn = table.getSchema().getColumnByIndex(columnIndex);
- columns.add(getStrippedColumnSchema(originalColumn));
- }
- this.schema = new Schema(columns);
- } else {
- this.schema = table.getSchema();
- }
-
- // If any of the column predicates are of type None (the predicate is known
- // to match no rows), then the scan can be short circuited without
- // contacting any tablet servers.
- boolean shortCircuit = false;
- for (KuduPredicate predicate : this.predicates.values()) {
- if (predicate.getType() == KuduPredicate.PredicateType.NONE) {
- shortCircuit = true;
- break;
- }
- }
- if (shortCircuit) {
- LOG.debug("Short circuiting scan with predicates: {}", predicates.values());
- this.hasMore = false;
- this.closed = true;
- }
- }
-
- /**
- * Clone the given column schema instance. The new instance will include only the name, type, and
- * nullability of the passed one.
- * @return a new column schema
- */
- private static ColumnSchema getStrippedColumnSchema(ColumnSchema columnToClone) {
- return new ColumnSchema.ColumnSchemaBuilder(columnToClone.getName(), columnToClone.getType())
- .nullable(columnToClone.isNullable())
- .build();
- }
-
- /**
- * Returns the maximum number of rows that this scanner was configured to return.
- * @return a long representing the maximum number of rows that can be returned
- */
- public long getLimit() {
- return this.limit;
- }
-
- /**
- * Tells if the last rpc returned that there might be more rows to scan.
- * @return true if there might be more data to scan, else false
- */
- public boolean hasMoreRows() {
- return this.hasMore;
- }
-
- /**
- * Returns if this scanner was configured to cache data blocks or not.
- * @return true if this scanner will cache blocks, else else.
- */
- public boolean getCacheBlocks() {
- return this.cacheBlocks;
- }
-
- /**
- * Returns the maximum number of bytes returned by the scanner, on each batch.
- * @return a long representing the maximum number of bytes that a scanner can receive at once
- * from a tablet server
- */
- public long getBatchSizeBytes() {
- return this.batchSizeBytes;
- }
-
- /**
- * Returns the ReadMode for this scanner.
- * @return the configured read mode for this scanner
- */
- public ReadMode getReadMode() {
- return this.readMode;
- }
-
- private Common.OrderMode getOrderMode() {
- return this.orderMode;
- }
-
- /**
- * Returns the projection schema of this scanner. If specific columns were
- * not specified during scanner creation, the table schema is returned.
- * @return the projection schema for this scanner
- */
- public Schema getProjectionSchema() {
- return this.schema;
- }
-
- long getSnapshotTimestamp() {
- return this.htTimestamp;
- }
-
- /**
- * Scans a number of rows.
- * <p>
- * Once this method returns {@code null} once (which indicates that this
- * {@code Scanner} is done scanning), calling it again leads to an undefined
- * behavior.
- * @return a deferred list of rows.
- */
- public Deferred<RowResultIterator> nextRows() {
- if (closed) { // We're already done scanning.
- return Deferred.fromResult(null);
- } else if (tablet == null) {
-
- Callback<Deferred<RowResultIterator>, AsyncKuduScanner.Response> cb =
- new Callback<Deferred<RowResultIterator>, Response>() {
- @Override
- public Deferred<RowResultIterator> call(Response resp) throws Exception {
- if (!resp.more || resp.scanner_id == null) {
- scanFinished();
- return Deferred.fromResult(resp.data); // there might be data to return
- }
- scannerId = resp.scanner_id;
- sequenceId++;
- hasMore = resp.more;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Scanner " + Bytes.pretty(scannerId) + " opened on " + tablet);
- }
- return Deferred.fromResult(resp.data);
- }
- public String toString() {
- return "scanner opened";
- }
- };
-
- Callback<Deferred<RowResultIterator>, Exception> eb =
- new Callback<Deferred<RowResultIterator>, Exception>() {
- @Override
- public Deferred<RowResultIterator> call(Exception e) throws Exception {
- invalidate();
- if (e instanceof NonCoveredRangeException) {
- NonCoveredRangeException ncre = (NonCoveredRangeException) e;
- nextPartitionKey = ncre.getNonCoveredRangeEnd();
-
- // Stop scanning if the non-covered range is past the end partition key.
- if (ncre.getNonCoveredRangeEnd().length == 0
- || (endPartitionKey != AsyncKuduClient.EMPTY_ARRAY
- && Bytes.memcmp(endPartitionKey, ncre.getNonCoveredRangeEnd()) <= 0)) {
- hasMore = false;
- closed = true; // the scanner is closed on the other side at this point
- return Deferred.fromResult(RowResultIterator.empty());
- }
- nextPartitionKey = ncre.getNonCoveredRangeEnd();
- scannerId = null;
- sequenceId = 0;
- return nextRows();
- } else {
- LOG.warn("Can not open scanner", e);
- // Don't let the scanner think it's opened on this tablet.
- return Deferred.fromError(e); // Let the error propogate.
- }
- }
- public String toString() {
- return "open scanner errback";
- }
- };
-
- // We need to open the scanner first.
- return client.sendRpcToTablet(getOpenRequest()).addCallbackDeferring(cb).addErrback(eb);
- } else if (prefetching && prefetcherDeferred != null) {
- // TODO KUDU-1260 - Check if this works and add a test
- prefetcherDeferred.chain(new Deferred<RowResultIterator>().addCallback(prefetch));
- return prefetcherDeferred;
- }
- final Deferred<RowResultIterator> d =
- client.scanNextRows(this).addCallbacks(got_next_row, nextRowErrback());
- if (prefetching) {
- d.chain(new Deferred<RowResultIterator>().addCallback(prefetch));
- }
- return d;
- }
-
- private final Callback<RowResultIterator, RowResultIterator> prefetch =
- new Callback<RowResultIterator, RowResultIterator>() {
- @Override
- public RowResultIterator call(RowResultIterator arg) throws Exception {
- if (hasMoreRows()) {
- prefetcherDeferred = client.scanNextRows(AsyncKuduScanner.this).addCallbacks
- (got_next_row, nextRowErrback());
- }
- return null;
- }
- };
-
- /**
- * Singleton callback to handle responses of "next" RPCs.
- * This returns an {@code ArrayList<ArrayList<KeyValue>>} (possibly inside a
- * deferred one).
- */
- private final Callback<RowResultIterator, Response> got_next_row =
- new Callback<RowResultIterator, Response>() {
- public RowResultIterator call(final Response resp) {
- if (!resp.more) { // We're done scanning this tablet.
- scanFinished();
- return resp.data;
- }
- sequenceId++;
- hasMore = resp.more;
- //LOG.info("Scan.next is returning rows: " + resp.data.getNumRows());
- return resp.data;
- }
- public String toString() {
- return "get nextRows response";
- }
- };
-
- /**
- * Creates a new errback to handle errors while trying to get more rows.
- */
- private final Callback<Exception, Exception> nextRowErrback() {
- return new Callback<Exception, Exception>() {
- public Exception call(final Exception error) {
- final AsyncKuduClient.RemoteTablet old_tablet = tablet; // Save before invalidate().
- String message = old_tablet + " pretends to not know " + AsyncKuduScanner.this;
- LOG.warn(message, error);
- invalidate(); // If there was an error, don't assume we're still OK.
- return error; // Let the error propagate.
- }
- public String toString() {
- return "NextRow errback";
- }
- };
- }
-
- void scanFinished() {
- Partition partition = tablet.getPartition();
- // Stop scanning if we have scanned until or past the end partition key.
- if (partition.isEndPartition()
- || (this.endPartitionKey != AsyncKuduClient.EMPTY_ARRAY
- && Bytes.memcmp(this.endPartitionKey, partition.getPartitionKeyEnd()) <= 0)) {
- hasMore = false;
- closed = true; // the scanner is closed on the other side at this point
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Done scanning tablet {} for partition {} with scanner id {}",
- tablet.getTabletIdAsString(), tablet.getPartition(), Bytes.pretty(scannerId));
- }
- nextPartitionKey = partition.getPartitionKeyEnd();
- scannerId = null;
- sequenceId = 0;
- invalidate();
- }
-
- /**
- * Closes this scanner (don't forget to call this when you're done with it!).
- * <p>
- * Closing a scanner already closed has no effect. The deferred returned
- * will be called back immediately.
- * @return A deferred object that indicates the completion of the request.
- * The {@link Object} can be null, a RowResultIterator if there was data left
- * in the scanner, or an Exception.
- */
- public Deferred<RowResultIterator> close() {
- if (closed) {
- return Deferred.fromResult(null);
- }
- final Deferred<RowResultIterator> d =
- client.closeScanner(this).addCallback(closedCallback()); // TODO errBack ?
- return d;
- }
-
- /** Callback+Errback invoked when the TabletServer closed our scanner. */
- private Callback<RowResultIterator, Response> closedCallback() {
- return new Callback<RowResultIterator, Response>() {
- public RowResultIterator call(Response response) {
- closed = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Scanner " + Bytes.pretty(scannerId) + " closed on "
- + tablet);
- }
- tablet = null;
- scannerId = "client debug closed".getBytes(); // Make debugging easier.
- return response == null ? null : response.data;
- }
- public String toString() {
- return "scanner closed";
- }
- };
- }
-
- public String toString() {
- final String tablet = this.tablet == null ? "null" : this.tablet.getTabletIdAsString();
- final StringBuilder buf = new StringBuilder();
- buf.append("KuduScanner(table=");
- buf.append(table.getName());
- buf.append(", tablet=").append(tablet);
- buf.append(", scannerId=").append(Bytes.pretty(scannerId));
- buf.append(", scanRequestTimeout=").append(scanRequestTimeout);
- buf.append(')');
- return buf.toString();
- }
-
- // ---------------------- //
- // Package private stuff. //
- // ---------------------- //
-
- KuduTable table() {
- return table;
- }
-
- /**
- * Sets the name of the tabletSlice that's hosting {@code this.start_key}.
- * @param tablet The tabletSlice we're currently supposed to be scanning.
- */
- void setTablet(final AsyncKuduClient.RemoteTablet tablet) {
- this.tablet = tablet;
- }
-
- /**
- * Invalidates this scanner and makes it assume it's no longer opened.
- * When a TabletServer goes away while we're scanning it, or some other type
- * of access problem happens, this method should be called so that the
- * scanner will have to re-locate the TabletServer and re-open itself.
- */
- void invalidate() {
- tablet = null;
- }
-
- /**
- * Returns the tabletSlice currently being scanned, if any.
- */
- AsyncKuduClient.RemoteTablet currentTablet() {
- return tablet;
- }
-
- /**
- * Returns an RPC to open this scanner.
- */
- KuduRpc<Response> getOpenRequest() {
- checkScanningNotStarted();
- // This is the only point where we know we haven't started scanning and where the scanner
- // should be fully configured
- if (this.inFirstTablet) {
- this.inFirstTablet = false;
- }
- return new ScanRequest(table, State.OPENING);
- }
-
- /**
- * Returns an RPC to fetch the next rows.
- */
- KuduRpc<Response> getNextRowsRequest() {
- return new ScanRequest(table, State.NEXT);
- }
-
- /**
- * Returns an RPC to close this scanner.
- */
- KuduRpc<Response> getCloseRequest() {
- return new ScanRequest(table, State.CLOSING);
- }
-
- /**
- * Throws an exception if scanning already started.
- * @throws IllegalStateException if scanning already started.
- */
- private void checkScanningNotStarted() {
- if (tablet != null) {
- throw new IllegalStateException("scanning already started");
- }
- }
-
- /**
- * Helper object that contains all the info sent by a TS after a Scan request.
- */
- static final class Response {
- /** The ID associated with the scanner that issued the request. */
- private final byte[] scanner_id;
- /** The actual payload of the response. */
- private final RowResultIterator data;
-
- /**
- * If false, the filter we use decided there was no more data to scan.
- * In this case, the server has automatically closed the scanner for us,
- * so we don't need to explicitly close it.
- */
- private final boolean more;
-
- Response(final byte[] scanner_id,
- final RowResultIterator data,
- final boolean more) {
- this.scanner_id = scanner_id;
- this.data = data;
- this.more = more;
- }
-
- public String toString() {
- return "AsyncKuduScanner$Response(scannerId=" + Bytes.pretty(scanner_id)
- + ", data=" + data + ", more=" + more + ") ";
- }
- }
-
- private enum State {
- OPENING,
- NEXT,
- CLOSING
- }
-
- /**
- * RPC sent out to fetch the next rows from the TabletServer.
- */
- private final class ScanRequest extends KuduRpc<Response> {
-
- State state;
-
- ScanRequest(KuduTable table, State state) {
- super(table);
- this.state = state;
- this.setTimeoutMillis(scanRequestTimeout);
- }
-
- @Override
- String serviceName() { return TABLET_SERVER_SERVICE_NAME; }
-
- @Override
- String method() {
- return "Scan";
- }
-
- @Override
- Collection<Integer> getRequiredFeatures() {
- if (predicates.isEmpty()) {
- return ImmutableList.of();
- } else {
- return ImmutableList.of(Tserver.TabletServerFeatures.COLUMN_PREDICATES_VALUE);
- }
- }
-
- /** Serializes this request. */
- ChannelBuffer serialize(Message header) {
- final ScanRequestPB.Builder builder = ScanRequestPB.newBuilder();
- switch (state) {
- case OPENING:
- // Save the tablet in the AsyncKuduScanner. This kind of a kludge but it really
- // is the easiest way.
- AsyncKuduScanner.this.tablet = super.getTablet();
- NewScanRequestPB.Builder newBuilder = NewScanRequestPB.newBuilder();
- newBuilder.setLimit(limit); // currently ignored
- newBuilder.addAllProjectedColumns(ProtobufHelper.schemaToListPb(schema));
- newBuilder.setTabletId(ZeroCopyLiteralByteString.wrap(tablet.getTabletIdAsBytes()));
- newBuilder.setReadMode(AsyncKuduScanner.this.getReadMode().pbVersion());
- newBuilder.setOrderMode(AsyncKuduScanner.this.getOrderMode());
- newBuilder.setCacheBlocks(cacheBlocks);
- // if the last propagated timestamp is set send it with the scan
- if (table.getAsyncClient().getLastPropagatedTimestamp() != AsyncKuduClient.NO_TIMESTAMP) {
- newBuilder.setPropagatedTimestamp(table.getAsyncClient().getLastPropagatedTimestamp());
- }
- newBuilder.setReadMode(AsyncKuduScanner.this.getReadMode().pbVersion());
-
- // if the mode is set to read on snapshot sent the snapshot timestamp
- if (AsyncKuduScanner.this.getReadMode() == ReadMode.READ_AT_SNAPSHOT &&
- AsyncKuduScanner.this.getSnapshotTimestamp() != AsyncKuduClient.NO_TIMESTAMP) {
- newBuilder.setSnapTimestamp(AsyncKuduScanner.this.getSnapshotTimestamp());
- }
-
- if (AsyncKuduScanner.this.startPrimaryKey != AsyncKuduClient.EMPTY_ARRAY &&
- AsyncKuduScanner.this.startPrimaryKey.length > 0) {
- newBuilder.setStartPrimaryKey(ZeroCopyLiteralByteString.copyFrom(startPrimaryKey));
- }
-
- if (AsyncKuduScanner.this.endPrimaryKey != AsyncKuduClient.EMPTY_ARRAY &&
- AsyncKuduScanner.this.endPrimaryKey.length > 0) {
- newBuilder.setStopPrimaryKey(ZeroCopyLiteralByteString.copyFrom(endPrimaryKey));
- }
-
- for (KuduPredicate pred : predicates.values()) {
- newBuilder.addColumnPredicates(pred.toPB());
- }
- builder.setNewScanRequest(newBuilder.build())
- .setBatchSizeBytes(batchSizeBytes);
- break;
- case NEXT:
- setTablet(AsyncKuduScanner.this.tablet);
- builder.setScannerId(ZeroCopyLiteralByteString.wrap(scannerId))
- .setCallSeqId(AsyncKuduScanner.this.sequenceId)
- .setBatchSizeBytes(batchSizeBytes);
- break;
- case CLOSING:
- setTablet(AsyncKuduScanner.this.tablet);
- builder.setScannerId(ZeroCopyLiteralByteString.wrap(scannerId))
- .setBatchSizeBytes(0)
- .setCloseScanner(true);
- }
-
- ScanRequestPB request = builder.build();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sending scan req: " + request.toString());
- }
-
- return toChannelBuffer(header, request);
- }
-
- @Override
- Pair<Response, Object> deserialize(final CallResponse callResponse,
- String tsUUID) throws Exception {
- ScanResponsePB.Builder builder = ScanResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), builder);
- ScanResponsePB resp = builder.build();
- final byte[] id = resp.getScannerId().toByteArray();
- TabletServerErrorPB error = resp.hasError() ? resp.getError() : null;
-
- if (error != null && error.getCode().equals(TabletServerErrorPB.Code.TABLET_NOT_FOUND)) {
- if (state == State.OPENING) {
- // Doing this will trigger finding the new location.
- return new Pair<Response, Object>(null, error);
- } else {
- Status statusIncomplete = Status.Incomplete("Cannot continue scanning, " +
- "the tablet has moved and this isn't a fault tolerant scan");
- throw new NonRecoverableException(statusIncomplete);
- }
- }
- RowResultIterator iterator = RowResultIterator.makeRowResultIterator(
- deadlineTracker.getElapsedMillis(), tsUUID, schema, resp.getData(),
- callResponse);
-
- boolean hasMore = resp.getHasMoreResults();
- if (id.length != 0 && scannerId != null && !Bytes.equals(scannerId, id)) {
- Status statusIllegalState = Status.IllegalState("Scan RPC response was for scanner"
- + " ID " + Bytes.pretty(id) + " but we expected "
- + Bytes.pretty(scannerId));
- throw new NonRecoverableException(statusIllegalState);
- }
- Response response = new Response(id, iterator, hasMore);
- if (LOG.isDebugEnabled()) {
- LOG.debug(response.toString());
- }
- return new Pair<Response, Object>(response, error);
- }
-
- public String toString() {
- return "ScanRequest(scannerId=" + Bytes.pretty(scannerId)
- + (tablet != null? ", tabletSlice=" + tablet.getTabletIdAsString() : "")
- + ", attempt=" + attempt + ')';
- }
-
- @Override
- public byte[] partitionKey() {
- // This key is used to lookup where the request needs to go
- return nextPartitionKey;
- }
- }
-
- /**
- * A Builder class to build {@link AsyncKuduScanner}.
- * Use {@link AsyncKuduClient#newScannerBuilder} in order to get a builder instance.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public static class AsyncKuduScannerBuilder
- extends AbstractKuduScannerBuilder<AsyncKuduScannerBuilder, AsyncKuduScanner> {
-
- AsyncKuduScannerBuilder(AsyncKuduClient client, KuduTable table) {
- super(client, table);
- }
-
- /**
- * Builds an {@link AsyncKuduScanner} using the passed configurations.
- * @return a new {@link AsyncKuduScanner}
- */
- public AsyncKuduScanner build() {
- return new AsyncKuduScanner(
- client, table, projectedColumnNames, projectedColumnIndexes, readMode, orderMode,
- scanRequestTimeout, predicates, limit, cacheBlocks,
- prefetching, lowerBoundPrimaryKey, upperBoundPrimaryKey,
- lowerBoundPartitionKey, upperBoundPartitionKey,
- htTimestamp, batchSizeBytes);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java b/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java
deleted file mode 100644
index 0290ee7..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/AsyncKuduSession.java
+++ /dev/null
@@ -1,856 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Range;
-import com.stumbleupon.async.Callback;
-import com.stumbleupon.async.Deferred;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.AsyncUtil;
-import org.kududb.util.Slice;
-import org.jboss.netty.util.Timeout;
-import org.jboss.netty.util.TimerTask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.concurrent.GuardedBy;
-import javax.annotation.concurrent.NotThreadSafe;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
-
-/**
- * A AsyncKuduSession belongs to a specific AsyncKuduClient, and represents a context in
- * which all read/write data access should take place. Within a session,
- * multiple operations may be accumulated and batched together for better
- * efficiency. Settings like timeouts, priorities, and trace IDs are also set
- * per session.<p>
- *
- * AsyncKuduSession is separate from KuduTable because a given batch or transaction
- * may span multiple tables. This is particularly important in the future when
- * we add ACID support, but even in the context of batching, we may be able to
- * coalesce writes to different tables hosted on the same server into the same
- * RPC.<p>
- *
- * AsyncKuduSession is separate from AsyncKuduClient because, in a multi-threaded
- * application, different threads may need to concurrently execute
- * transactions. Similar to a JDBC "session", transaction boundaries will be
- * delineated on a per-session basis -- in between a "BeginTransaction" and
- * "Commit" call on a given session, all operations will be part of the same
- * transaction. Meanwhile another concurrent Session object can safely run
- * non-transactional work or other transactions without interfering.<p>
- *
- * Therefore, this class is <b>not</b> thread-safe.<p>
- *
- * Additionally, there is a guarantee that writes from different sessions do not
- * get batched together into the same RPCs -- this means that latency-sensitive
- * clients can run through the same AsyncKuduClient object as throughput-oriented
- * clients, perhaps by setting the latency-sensitive session's timeouts low and
- * priorities high. Without the separation of batches, a latency-sensitive
- * single-row insert might get batched along with 10MB worth of inserts from the
- * batch writer, thus delaying the response significantly.<p>
- *
- * Though we currently do not have transactional support, users will be forced
- * to use a AsyncKuduSession to instantiate reads as well as writes. This will make
- * it more straight-forward to add RW transactions in the future without
- * significant modifications to the API.<p>
- *
- * Timeouts are handled differently depending on the flush mode.
- * With AUTO_FLUSH_SYNC, the timeout is set on each apply()'d operation.
- * With AUTO_FLUSH_BACKGROUND and MANUAL_FLUSH, the timeout is assigned to a whole batch of
- * operations upon flush()'ing. It means that in a situation with a timeout of 500ms and a flush
- * interval of 1000ms, an operation can be outstanding for up to 1500ms before being timed out.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-@NotThreadSafe
-public class AsyncKuduSession implements SessionConfiguration {
-
- public static final Logger LOG = LoggerFactory.getLogger(AsyncKuduSession.class);
- private static final Range<Float> PERCENTAGE_RANGE = Range.closed(0.0f, 1.0f);
-
- private final AsyncKuduClient client;
- private final Random randomizer = new Random();
- private final ErrorCollector errorCollector;
- private int interval = 1000;
- private int mutationBufferSpace = 1000; // TODO express this in terms of data size.
- private float mutationBufferLowWatermarkPercentage = 0.5f;
- private int mutationBufferLowWatermark;
- private FlushMode flushMode;
- private ExternalConsistencyMode consistencyMode;
- private long timeoutMs;
-
- /**
- * Protects internal state from concurrent access. {@code AsyncKuduSession} is not threadsafe
- * from the application's perspective, but because internally async timers and async flushing
- * tasks may access the session concurrently with the application, synchronization is still
- * needed.
- */
- private final Object monitor = new Object();
-
- /**
- * Tracks the currently active buffer.
- *
- * When in mode {@link FlushMode#AUTO_FLUSH_BACKGROUND} or {@link FlushMode#AUTO_FLUSH_SYNC},
- * {@code AsyncKuduSession} uses double buffering to improve write throughput. While the
- * application is {@link #apply}ing operations to one buffer (the {@code activeBuffer}), the
- * second buffer is either being flushed, or if it has already been flushed, it waits in the
- * {@link #inactiveBuffers} queue. When the currently active buffer is flushed,
- * {@code activeBuffer} is set to {@code null}. On the next call to {@code apply}, an inactive
- * buffer is taken from {@code inactiveBuffers} and made the new active buffer. If both
- * buffers are still flushing, then the {@code apply} call throws {@link PleaseThrottleException}.
- */
- @GuardedBy("monitor")
- private Buffer activeBuffer;
-
- /**
- * The buffers. May either be active (pointed to by {@link #activeBuffer},
- * inactive (in the {@link #inactiveBuffers}) queue, or flushing.
- */
- private final Buffer bufferA = new Buffer();
- private final Buffer bufferB = new Buffer();
-
- /**
- * Queue containing flushed, inactive buffers. May be accessed from callbacks (I/O threads).
- * We restrict the session to only two buffers, so {@link BlockingQueue#add} can
- * be used without chance of failure.
- */
- private final BlockingQueue<Buffer> inactiveBuffers = new ArrayBlockingQueue<>(2, false);
-
- /**
- * Deferred used to notify on flush events. Atomically swapped and completed every time a buffer
- * is flushed. This can be used to notify handlers of {@link PleaseThrottleException} that more
- * capacity may be available in the active buffer.
- */
- private final AtomicReference<Deferred<Void>> flushNotification =
- new AtomicReference<>(new Deferred<Void>());
-
- /**
- * Tracks whether the session has been closed.
- */
- private volatile boolean closed = false;
-
- private boolean ignoreAllDuplicateRows = false;
-
- /**
- * Package-private constructor meant to be used via AsyncKuduClient
- * @param client client that creates this session
- */
- AsyncKuduSession(AsyncKuduClient client) {
- this.client = client;
- flushMode = FlushMode.AUTO_FLUSH_SYNC;
- consistencyMode = CLIENT_PROPAGATED;
- timeoutMs = client.getDefaultOperationTimeoutMs();
- inactiveBuffers.add(bufferA);
- inactiveBuffers.add(bufferB);
- errorCollector = new ErrorCollector(mutationBufferSpace);
- setMutationBufferLowWatermark(this.mutationBufferLowWatermarkPercentage);
- }
-
- @Override
- public FlushMode getFlushMode() {
- return this.flushMode;
- }
-
- @Override
- public void setFlushMode(FlushMode flushMode) {
- if (hasPendingOperations()) {
- throw new IllegalArgumentException("Cannot change flush mode when writes are buffered");
- }
- this.flushMode = flushMode;
- }
-
- @Override
- public void setExternalConsistencyMode(ExternalConsistencyMode consistencyMode) {
- if (hasPendingOperations()) {
- throw new IllegalArgumentException("Cannot change consistency mode "
- + "when writes are buffered");
- }
- this.consistencyMode = consistencyMode;
- }
-
- @Override
- public void setMutationBufferSpace(int size) {
- if (hasPendingOperations()) {
- throw new IllegalArgumentException("Cannot change the buffer" +
- " size when operations are buffered");
- }
- this.mutationBufferSpace = size;
- // Reset the low watermark, using the same percentage as before.
- setMutationBufferLowWatermark(mutationBufferLowWatermarkPercentage);
- }
-
- @Override
- public void setMutationBufferLowWatermark(float mutationBufferLowWatermarkPercentage) {
- if (hasPendingOperations()) {
- throw new IllegalArgumentException("Cannot change the buffer" +
- " low watermark when operations are buffered");
- } else if (!PERCENTAGE_RANGE.contains(mutationBufferLowWatermarkPercentage)) {
- throw new IllegalArgumentException("The low watermark must be between 0 and 1 inclusively");
- }
- this.mutationBufferLowWatermarkPercentage = mutationBufferLowWatermarkPercentage;
- this.mutationBufferLowWatermark =
- (int)(this.mutationBufferLowWatermarkPercentage * mutationBufferSpace);
- }
-
- /**
- * Lets us set a specific seed for tests
- * @param seed
- */
- @VisibleForTesting
- void setRandomSeed(long seed) {
- this.randomizer.setSeed(seed);
- }
-
- @Override
- public void setFlushInterval(int interval) {
- this.interval = interval;
- }
-
- @Override
- public void setTimeoutMillis(long timeout) {
- this.timeoutMs = timeout;
- }
-
- @Override
- public long getTimeoutMillis() {
- return this.timeoutMs;
- }
-
- @Override
- public boolean isClosed() {
- return closed;
- }
-
- @Override
- public boolean isIgnoreAllDuplicateRows() {
- return ignoreAllDuplicateRows;
- }
-
- @Override
- public void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows) {
- this.ignoreAllDuplicateRows = ignoreAllDuplicateRows;
- }
-
- @Override
- public int countPendingErrors() {
- return errorCollector.countErrors();
- }
-
- @Override
- public RowErrorsAndOverflowStatus getPendingErrors() {
- return errorCollector.getErrors();
- }
-
- /**
- * Flushes the buffered operations and marks this session as closed.
- * See the javadoc on {@link #flush()} on how to deal with exceptions coming out of this method.
- * @return a Deferred whose callback chain will be invoked when.
- * everything that was buffered at the time of the call has been flushed.
- */
- public Deferred<List<OperationResponse>> close() {
- if (!closed) {
- closed = true;
- client.removeSession(this);
- }
- return flush();
- }
-
- /**
- * Returns a buffer to the inactive queue after flushing.
- * @param buffer the buffer to return to the inactive queue.
- */
- private void queueBuffer(Buffer buffer) {
- buffer.callbackFlushNotification();
- Deferred<Void> localFlushNotification = flushNotification.getAndSet(new Deferred<Void>());
- inactiveBuffers.add(buffer);
- localFlushNotification.callback(null);
- }
-
- /**
- * Callback which waits for all tablet location lookups to complete, groups all operations into
- * batches by tablet, and dispatches them. When all of the batches are complete, a deferred is
- * fired and the buffer is added to the inactive queue.
- */
- private final class TabletLookupCB implements Callback<Void, Object> {
- private final AtomicInteger lookupsOutstanding;
- private final Buffer buffer;
- private final Deferred<List<BatchResponse>> deferred;
-
- public TabletLookupCB(Buffer buffer, Deferred<List<BatchResponse>> deferred) {
- this.lookupsOutstanding = new AtomicInteger(buffer.getOperations().size());
- this.buffer = buffer;
- this.deferred = deferred;
- }
-
- @Override
- public Void call(Object _void) throws Exception {
- if (lookupsOutstanding.decrementAndGet() != 0) return null;
-
- // The final tablet lookup is complete. Batch all of the buffered
- // operations into their respective tablet, and then send the batches.
-
- // Group the operations by tablet.
- Map<Slice, Batch> batches = new HashMap<>();
- List<OperationResponse> opsFailedInLookup = new ArrayList<>();
-
- for (BufferedOperation bufferedOp : buffer.getOperations()) {
- Operation operation = bufferedOp.getOperation();
- if (bufferedOp.tabletLookupFailed()) {
- Exception failure = bufferedOp.getTabletLookupFailure();
- RowError error;
- if (failure instanceof NonCoveredRangeException) {
- // TODO: this should be something different than NotFound so that
- // applications can distinguish from updates on missing rows.
- error = new RowError(Status.NotFound(failure.getMessage()), operation);
- } else {
- LOG.warn("unexpected tablet lookup failure for operation {}", operation, failure);
- error = new RowError(Status.RuntimeError(failure.getMessage()), operation);
- }
- OperationResponse response = new OperationResponse(0, null, 0, operation, error);
- // Add the row error to the error collector if the session is in background flush mode,
- // and complete the operation's deferred with the error response. The ordering between
- // adding to the error collector and completing the deferred should not matter since
- // applications should be using one or the other method for error handling, not both.
- if (flushMode == FlushMode.AUTO_FLUSH_BACKGROUND) {
- errorCollector.addError(error);
- }
- operation.callback(response);
- opsFailedInLookup.add(response);
- continue;
- }
- LocatedTablet tablet = bufferedOp.getTablet();
- Slice tabletId = new Slice(tablet.getTabletId());
-
- Batch batch = batches.get(tabletId);
- if (batch == null) {
- batch = new Batch(operation.getTable(), tablet, ignoreAllDuplicateRows);
- batches.put(tabletId, batch);
- }
- batch.add(operation);
- }
-
- List<Deferred<BatchResponse>> batchResponses = new ArrayList<>(batches.size() + 1);
- if (!opsFailedInLookup.isEmpty()) {
- batchResponses.add(Deferred.fromResult(new BatchResponse(opsFailedInLookup)));
- }
-
- for (Batch batch : batches.values()) {
- if (timeoutMs != 0) {
- batch.deadlineTracker.reset();
- batch.setTimeoutMillis(timeoutMs);
- }
- addBatchCallbacks(batch);
- batchResponses.add(client.sendRpcToTablet(batch));
- }
-
- // On completion of all batches, fire the completion deferred, and add the buffer
- // back to the inactive buffers queue. This frees it up for new inserts.
- AsyncUtil.addBoth(
- Deferred.group(batchResponses),
- new Callback<Void, Object>() {
- @Override
- public Void call(Object responses) {
- queueBuffer(buffer);
- deferred.callback(responses);
- return null;
- }
- });
-
- return null;
- }
- }
-
- /**
- * Flush buffered writes.
- * @return a {@link Deferred} whose callback chain will be invoked when all applied operations at
- * the time of the call have been flushed.
- */
- public Deferred<List<OperationResponse>> flush() {
- Buffer buffer;
- Deferred<Void> nonActiveBufferFlush;
- synchronized (monitor) {
- nonActiveBufferFlush = getNonActiveFlushNotification();
- buffer = activeBuffer;
- activeBuffer = null;
- }
-
- final Deferred<List<OperationResponse>> activeBufferFlush = buffer == null ?
- Deferred.<List<OperationResponse>>fromResult(ImmutableList.<OperationResponse>of()) :
- doFlush(buffer);
-
- return AsyncUtil.addBothDeferring(nonActiveBufferFlush,
- new Callback<Deferred<List<OperationResponse>>, Object>() {
- @Override
- public Deferred<List<OperationResponse>> call(Object arg) {
- return activeBufferFlush;
- }
- });
- }
-
- /**
- * Flushes a write buffer. This method takes ownership of the buffer, no other concurrent access
- * is allowed.
- *
- * @param buffer the buffer to flush, must not be modified once passed to this method
- * @return the operation responses
- */
- private Deferred<List<OperationResponse>> doFlush(Buffer buffer) {
- LOG.debug("flushing buffer: {}", buffer);
- if (buffer.getOperations().isEmpty()) {
- // no-op.
- return Deferred.<List<OperationResponse>>fromResult(ImmutableList.<OperationResponse>of());
- }
-
- Deferred<List<BatchResponse>> batchResponses = new Deferred<>();
- Callback<Void, Object> tabletLookupCB = new TabletLookupCB(buffer, batchResponses);
-
- for (BufferedOperation bufferedOperation : buffer.getOperations()) {
- AsyncUtil.addBoth(bufferedOperation.getTabletLookup(), tabletLookupCB);
- }
-
- return batchResponses.addCallback(ConvertBatchToListOfResponsesCB.getInstance());
- }
-
- /**
- * Callback used to send a list of OperationResponse instead of BatchResponse since the
- * latter is an implementation detail.
- */
- private static class ConvertBatchToListOfResponsesCB implements Callback<List<OperationResponse>,
- List<BatchResponse>> {
- private static final ConvertBatchToListOfResponsesCB INSTANCE =
- new ConvertBatchToListOfResponsesCB();
- @Override
- public List<OperationResponse> call(List<BatchResponse> batchResponses) throws Exception {
- // First compute the size of the union of all the lists so that we don't trigger expensive
- // list growths while adding responses to it.
- int size = 0;
- for (BatchResponse batchResponse : batchResponses) {
- size += batchResponse.getIndividualResponses().size();
- }
-
- ArrayList<OperationResponse> responses = new ArrayList<>(size);
- for (BatchResponse batchResponse : batchResponses) {
- responses.addAll(batchResponse.getIndividualResponses());
- }
-
- return responses;
- }
- @Override
- public String toString() {
- return "ConvertBatchToListOfResponsesCB";
- }
- public static ConvertBatchToListOfResponsesCB getInstance() {
- return INSTANCE;
- }
- }
-
- @Override
- public boolean hasPendingOperations() {
- synchronized (monitor) {
- return activeBuffer == null ? inactiveBuffers.size() < 2 :
- activeBuffer.getOperations().size() > 0 || !inactiveBufferAvailable();
- }
- }
-
- /**
- * Apply the given operation.
- * The behavior of this function depends on the current flush mode. Regardless
- * of flush mode, however, Apply may begin to perform processing in the background
- * for the call (e.g looking up the tablet, etc).
- * @param operation operation to apply
- * @return a Deferred to track this operation
- * @throws KuduException if an error happens or {@link PleaseThrottleException} is triggered
- */
- public Deferred<OperationResponse> apply(final Operation operation) throws KuduException {
- Preconditions.checkNotNull(operation, "Can not apply a null operation");
-
- // Freeze the row so that the client can not concurrently modify it while it is in flight.
- operation.getRow().freeze();
-
- // If immediate flush mode, send the operation directly.
- if (flushMode == FlushMode.AUTO_FLUSH_SYNC) {
- if (timeoutMs != 0) {
- operation.setTimeoutMillis(timeoutMs);
- }
- operation.setExternalConsistencyMode(this.consistencyMode);
- operation.setIgnoreAllDuplicateRows(ignoreAllDuplicateRows);
- return client.sendRpcToTablet(operation);
- }
-
- // Kick off a location lookup.
- Deferred<LocatedTablet> tablet = client.getTabletLocation(operation.getTable(),
- operation.partitionKey(),
- timeoutMs);
-
- // Holds a buffer that should be flushed outside the synchronized block, if necessary.
- Buffer fullBuffer = null;
- try {
- synchronized (monitor) {
- if (activeBuffer == null) {
- // If the active buffer is null then we recently flushed. Check if there
- // is an inactive buffer available to replace as the active.
- if (inactiveBufferAvailable()) {
- refreshActiveBuffer();
- } else {
- Status statusServiceUnavailable =
- Status.ServiceUnavailable("All buffers are currently flushing");
- // This can happen if the user writes into a buffer, flushes it, writes
- // into the second, flushes it, and immediately tries to write again.
- throw new PleaseThrottleException(statusServiceUnavailable,
- null, operation, flushNotification.get());
- }
- }
-
- if (flushMode == FlushMode.MANUAL_FLUSH) {
- if (activeBuffer.getOperations().size() < mutationBufferSpace) {
- activeBuffer.getOperations().add(new BufferedOperation(tablet, operation));
- } else {
- Status statusIllegalState =
- Status.IllegalState("MANUAL_FLUSH is enabled but the buffer is too big");
- throw new NonRecoverableException(statusIllegalState);
- }
- } else {
- assert flushMode == FlushMode.AUTO_FLUSH_BACKGROUND;
- int activeBufferSize = activeBuffer.getOperations().size();
-
- if (activeBufferSize >= mutationBufferSpace) {
- // Save the active buffer into fullBuffer so that it gets flushed when we leave this
- // synchronized block.
- fullBuffer = activeBuffer;
- activeBuffer = null;
- activeBufferSize = 0;
- if (inactiveBufferAvailable()) {
- refreshActiveBuffer();
- } else {
- Status statusServiceUnavailable =
- Status.ServiceUnavailable("All buffers are currently flushing");
- throw new PleaseThrottleException(statusServiceUnavailable,
- null, operation, flushNotification.get());
- }
- }
-
- if (mutationBufferLowWatermark < mutationBufferSpace && // low watermark is enabled
- activeBufferSize >= mutationBufferLowWatermark && // buffer is over low water mark
- !inactiveBufferAvailable()) { // no inactive buffers
-
- // Check if we are over the low water mark.
- int randomWatermark = activeBufferSize + 1 +
- randomizer.nextInt(mutationBufferSpace -
- mutationBufferLowWatermark);
-
- if (randomWatermark > mutationBufferSpace) {
- Status statusServiceUnavailable =
- Status.ServiceUnavailable("The previous buffer hasn't been flushed and the " +
- "current buffer is over the low watermark, please retry later");
- throw new PleaseThrottleException(statusServiceUnavailable,
- null, operation, flushNotification.get());
- }
- }
-
- activeBuffer.getOperations().add(new BufferedOperation(tablet, operation));
-
- if (activeBufferSize + 1 >= mutationBufferSpace && inactiveBufferAvailable()) {
- // If the operation filled the buffer, then flush it.
- Preconditions.checkState(fullBuffer == null);
- fullBuffer = activeBuffer;
- activeBuffer = null;
- activeBufferSize = 0;
- } else if (activeBufferSize == 0) {
- // If this is the first operation in the buffer, start a background flush timer.
- client.newTimeout(activeBuffer.getFlusherTask(), interval);
- }
- }
- }
- } finally {
- // Flush the buffer outside of the synchronized block, if required.
- if (fullBuffer != null) {
- doFlush(fullBuffer);
- }
- }
- return operation.getDeferred();
- }
-
- /**
- * Returns {@code true} if there is an inactive buffer available.
- * @return true if there is currently an inactive buffer available
- */
- private boolean inactiveBufferAvailable() {
- return inactiveBuffers.peek() != null;
- }
-
- /**
- * Refreshes the active buffer. This should only be called after a
- * {@link #flush()} when the active buffer is {@code null}, there is an
- * inactive buffer available (see {@link #inactiveBufferAvailable()}, and
- * {@link #monitor} is locked.
- */
- @GuardedBy("monitor")
- private void refreshActiveBuffer() {
- Preconditions.checkState(activeBuffer == null);
- activeBuffer = inactiveBuffers.remove();
- activeBuffer.reset();
- }
-
- /**
- * Returns a flush notification for the currently non-active buffers.
- * This is used during manual {@link #flush} calls to ensure that all buffers (not just the active
- * buffer) are fully flushed before completing.
- */
- @GuardedBy("monitor")
- private Deferred<Void> getNonActiveFlushNotification() {
- final Deferred<Void> notificationA = bufferA.getFlushNotification();
- final Deferred<Void> notificationB = bufferB.getFlushNotification();
- if (activeBuffer == null) {
- // Both buffers are either flushing or inactive.
- return AsyncUtil.addBothDeferring(notificationA, new Callback<Deferred<Void>, Object>() {
- @Override
- public Deferred<Void> call(Object _obj) throws Exception {
- return notificationB;
- }
- });
- } else if (activeBuffer == bufferA) {
- return notificationB;
- } else {
- return notificationA;
- }
- }
-
- /**
- * Creates callbacks to handle a multi-put and adds them to the request.
- * @param request the request for which we must handle the response
- */
- private void addBatchCallbacks(final Batch request) {
- final class BatchCallback implements Callback<BatchResponse, BatchResponse> {
- public BatchResponse call(final BatchResponse response) {
- LOG.trace("Got a Batch response for {} rows", request.operations.size());
- if (response.getWriteTimestamp() != 0) {
- AsyncKuduSession.this.client.updateLastPropagatedTimestamp(response.getWriteTimestamp());
- }
-
- // Send individualized responses to all the operations in this batch.
- for (OperationResponse operationResponse : response.getIndividualResponses()) {
- operationResponse.getOperation().callback(operationResponse);
- if (flushMode == FlushMode.AUTO_FLUSH_BACKGROUND && operationResponse.hasRowError()) {
- errorCollector.addError(operationResponse.getRowError());
- }
- }
-
- return response;
- }
-
- @Override
- public String toString() {
- return "apply batch response";
- }
- }
-
- final class BatchErrCallback implements Callback<Exception, Exception> {
- @Override
- public Exception call(Exception e) {
- // Send the same exception to all the operations.
- for (Operation operation : request.operations) {
- operation.errback(e);
- }
- return e;
- }
- @Override
- public String toString() {
- return "apply batch error response";
- }
- }
-
- request.getDeferred().addCallbacks(new BatchCallback(), new BatchErrCallback());
- }
-
- /**
- * A FlusherTask is created for each active buffer in mode
- * {@link FlushMode#AUTO_FLUSH_BACKGROUND}.
- */
- private final class FlusherTask implements TimerTask {
- public void run(final Timeout timeout) {
- Buffer buffer = null;
- synchronized (monitor) {
- if (activeBuffer == null) {
- return;
- }
- if (activeBuffer.getFlusherTask() == this) {
- buffer = activeBuffer;
- activeBuffer = null;
- }
- }
-
- if (buffer != null) {
- doFlush(buffer);
- }
- }
- }
-
- /**
- * The {@code Buffer} consists of a list of operations, an optional pointer to a flush task,
- * and a flush notification.
- *
- * The {@link #flusherTask} is used in mode {@link FlushMode#AUTO_FLUSH_BACKGROUND} to point to
- * the background flusher task assigned to the buffer when it becomes active and the first
- * operation is applied to it. When the flusher task executes after the timeout, it checks
- * that the currently active buffer's flusher task points to itself before executing the flush.
- * This protects against the background task waking up after one or more manual flushes and
- * attempting to flush the active buffer.
- *
- * The {@link #flushNotification} deferred is used when executing manual {@link #flush}es to
- * ensure that non-active buffers are fully flushed. {@code flushNotification} is completed
- * when this buffer is successfully flushed. When the buffer is promoted from inactive to active,
- * the deferred is replaced with a new one to indicate that the buffer is not yet flushed.
- *
- * Buffer is externally synchronized. When the active buffer, {@link #monitor}
- * synchronizes access to it.
- */
- private final class Buffer {
- private final List<BufferedOperation> operations = new ArrayList<>();
-
- private FlusherTask flusherTask = null;
-
- private Deferred<Void> flushNotification = Deferred.fromResult(null);
-
- public List<BufferedOperation> getOperations() {
- return operations;
- }
-
- @GuardedBy("monitor")
- public FlusherTask getFlusherTask() {
- if (flusherTask == null) {
- flusherTask = new FlusherTask();
- }
- return flusherTask;
- }
-
- /**
- * Returns a {@link Deferred} which will be completed when this buffer is flushed. If the buffer
- * is inactive (its flush is complete and it has been enqueued into {@link #inactiveBuffers}),
- * then the deferred will already be complete.
- */
- public Deferred<Void> getFlushNotification() {
- return flushNotification;
- }
-
- /**
- * Completes the buffer's flush notification. Should be called when the buffer has been
- * successfully flushed.
- */
- public void callbackFlushNotification() {
- LOG.trace("buffer flush notification fired: {}", this);
- flushNotification.callback(null);
- }
-
- /**
- * Resets the buffer's internal state. Should be called when the buffer is promoted from
- * inactive to active.
- */
- @GuardedBy("monitor")
- public void reset() {
- LOG.trace("buffer reset: {}", this);
- operations.clear();
- flushNotification = new Deferred<>();
- flusherTask = null;
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("operations", operations.size())
- .add("flusherTask", flusherTask)
- .add("flushNotification", flushNotification)
- .toString();
- }
- }
-
- /**
- * Container class holding all the state associated with a buffered operation.
- */
- private static final class BufferedOperation {
- /** Holds either a {@link LocatedTablet} or the failure exception if the lookup failed. */
- private Object tablet = null;
- private final Deferred<Void> tabletLookup;
- private final Operation operation;
-
- public BufferedOperation(Deferred<LocatedTablet> tablet,
- Operation operation) {
- tabletLookup = AsyncUtil.addBoth(tablet, new Callback<Void, Object>() {
- @Override
- public Void call(final Object tablet) {
- BufferedOperation.this.tablet = tablet;
- return null;
- }
- });
- this.operation = Preconditions.checkNotNull(operation);
- }
-
- /**
- * @return {@code true} if the tablet lookup failed.
- */
- public boolean tabletLookupFailed() {
- return !(tablet instanceof LocatedTablet);
- }
-
- /**
- * @return the located tablet
- * @throws ClassCastException if the tablet lookup failed,
- * check with {@link #tabletLookupFailed} before calling
- */
- public LocatedTablet getTablet() {
- return (LocatedTablet) tablet;
- }
-
- /**
- * @return the cause of the failed lookup
- * @throws ClassCastException if the tablet lookup succeeded,
- * check with {@link #tabletLookupFailed} before calling
- */
- public Exception getTabletLookupFailure() {
- return (Exception) tablet;
- }
-
- public Deferred<Void> getTabletLookup() {
- return tabletLookup;
- }
-
- public Operation getOperation() {
- return operation;
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("tablet", tablet)
- .add("operation", operation)
- .toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Batch.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Batch.java b/java/kudu-client/src/main/java/org/kududb/client/Batch.java
deleted file mode 100644
index ed1d870..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Batch.java
+++ /dev/null
@@ -1,199 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
-import com.google.protobuf.Message;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.client.Statistics.Statistic;
-import org.kududb.client.Statistics.TabletStatistics;
-import org.kududb.tserver.Tserver;
-import org.kududb.tserver.Tserver.TabletServerErrorPB;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
-
-/**
- * Used internally to group Operations for a single tablet together before sending to the tablet
- * server.
- */
-@InterfaceAudience.Private
-class Batch extends KuduRpc<BatchResponse> {
-
- /** Holds batched operations. */
- final List<Operation> operations = new ArrayList<>();
-
- /** The tablet this batch will be routed to. */
- private final LocatedTablet tablet;
-
- /**
- * This size will be set when serialize is called. It stands for the size of rows in all
- * operations in this batch.
- */
- private long rowOperationsSizeBytes = 0;
-
- /** See {@link SessionConfiguration#setIgnoreAllDuplicateRows(boolean)} */
- private final boolean ignoreAllDuplicateRows;
-
-
- Batch(KuduTable table, LocatedTablet tablet, boolean ignoreAllDuplicateRows) {
- super(table);
- this.ignoreAllDuplicateRows = ignoreAllDuplicateRows;
- this.tablet = tablet;
- }
-
- /**
- * Returns the bytes size of this batch's row operations after serialization.
- * @return size in bytes
- * @throws IllegalStateException thrown if this RPC hasn't been serialized eg sent to a TS
- */
- long getRowOperationsSizeBytes() {
- if (this.rowOperationsSizeBytes == 0) {
- throw new IllegalStateException("This row hasn't been serialized yet");
- }
- return this.rowOperationsSizeBytes;
- }
-
- public void add(Operation operation) {
- assert Bytes.memcmp(operation.partitionKey(),
- tablet.getPartition().getPartitionKeyStart()) >= 0 &&
- (tablet.getPartition().getPartitionKeyEnd().length == 0 ||
- Bytes.memcmp(operation.partitionKey(),
- tablet.getPartition().getPartitionKeyEnd()) < 0);
-
- operations.add(operation);
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- final Tserver.WriteRequestPB.Builder builder = Operation.createAndFillWriteRequestPB(operations);
- rowOperationsSizeBytes = builder.getRowOperations().getRows().size() +
- builder.getRowOperations().getIndirectData().size();
- builder.setTabletId(ZeroCopyLiteralByteString.wrap(getTablet().getTabletIdAsBytes()));
- builder.setExternalConsistencyMode(externalConsistencyMode.pbVersion());
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- String serviceName() {
- return TABLET_SERVER_SERVICE_NAME;
- }
-
- @Override
- String method() {
- return Operation.METHOD;
- }
-
- @Override
- Pair<BatchResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- Tserver.WriteResponsePB.Builder builder = Tserver.WriteResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), builder);
-
- List<Tserver.WriteResponsePB.PerRowErrorPB> errorsPB = builder.getPerRowErrorsList();
- if (ignoreAllDuplicateRows) {
- boolean allAlreadyPresent = true;
- for (Tserver.WriteResponsePB.PerRowErrorPB errorPB : errorsPB) {
- if (errorPB.getError().getCode() != WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT) {
- allAlreadyPresent = false;
- break;
- }
- }
- if (allAlreadyPresent) {
- errorsPB = Collections.emptyList();
- }
- }
-
- BatchResponse response = new BatchResponse(deadlineTracker.getElapsedMillis(), tsUUID,
- builder.getTimestamp(), errorsPB, operations);
-
- if (injectedError != null) {
- if (injectedlatencyMs > 0) {
- try {
- Thread.sleep(injectedlatencyMs);
- } catch (InterruptedException e) {
- }
- }
- return new Pair<BatchResponse, Object>(response, injectedError);
- }
-
- return new Pair<BatchResponse, Object>(response, builder.hasError() ? builder.getError() : null);
- }
-
- @Override
- public byte[] partitionKey() {
- return tablet.getPartition().getPartitionKeyStart();
- }
-
- @Override
- boolean isRequestTracked() {
- return true;
- }
-
- @Override
- void updateStatistics(Statistics statistics, BatchResponse response) {
- Slice tabletId = this.getTablet().getTabletId();
- String tableName = this.getTable().getName();
- TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId);
- if (response == null) {
- tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, operations.size());
- tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1);
- return;
- }
- tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1);
- for (OperationResponse opResponse : response.getIndividualResponses()) {
- if (opResponse.hasRowError()) {
- tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1);
- } else {
- tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1);
- }
- }
- tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationsSizeBytes());
- }
-
- @Override
- public String toString() {
- return MoreObjects.toStringHelper(this)
- .add("operations", operations.size())
- .add("tablet", tablet)
- .add("ignoreAllDuplicateRows", ignoreAllDuplicateRows)
- .toString();
- }
-
- private static TabletServerErrorPB injectedError;
- private static int injectedlatencyMs;
-
- /**
- * Inject tablet server side error for Batch rpc related tests.
- * @param error error response from tablet server
- * @param latencyMs blocks response handling thread for some time to simulate
- * write latency
- */
- @VisibleForTesting
- static void injectTabletServerErrorAndLatency(TabletServerErrorPB error, int latencyMs) {
- injectedError = error;
- injectedlatencyMs = latencyMs;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/BatchResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/BatchResponse.java b/java/kudu-client/src/main/java/org/kududb/client/BatchResponse.java
deleted file mode 100644
index f67153b..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/BatchResponse.java
+++ /dev/null
@@ -1,105 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.tserver.Tserver;
-
-/**
- * Response type for Batch (which is used internally by AsyncKuduSession).
- * Provides the Hybrid Time write timestamp returned by the Tablet Server.
- */
-@InterfaceAudience.Private
-public class BatchResponse extends KuduRpcResponse {
-
- private final long writeTimestamp;
- private final List<RowError> rowErrors;
- private final List<OperationResponse> individualResponses;
-
- /**
- * Package-private constructor to be used by the RPCs.
- * @param elapsedMillis time in milliseconds since RPC creation to now
- * @param writeTimestamp HT's write timestamp
- * @param errorsPB a list of row errors, can be empty
- * @param operations the list of operations which created this response
- */
- BatchResponse(long elapsedMillis, String tsUUID, long writeTimestamp,
- List<Tserver.WriteResponsePB.PerRowErrorPB> errorsPB,
- List<Operation> operations) {
- super(elapsedMillis, tsUUID);
- this.writeTimestamp = writeTimestamp;
- individualResponses = new ArrayList<>(operations.size());
- if (errorsPB.isEmpty()) {
- rowErrors = Collections.emptyList();
- } else {
- rowErrors = new ArrayList<>(errorsPB.size());
- }
-
- // Populate the list of individual row responses and the list of row errors. Not all the rows
- // maybe have errors, but 'errorsPB' contains them in the same order as the operations that
- // were sent.
- int currentErrorIndex = 0;
- Operation currentOperation;
- for (int i = 0; i < operations.size(); i++) {
- RowError rowError = null;
- currentOperation = operations.get(i);
- if (currentErrorIndex < errorsPB.size() &&
- errorsPB.get(currentErrorIndex).getRowIndex() == i) {
- rowError = RowError.fromRowErrorPb(errorsPB.get(currentErrorIndex),
- currentOperation, tsUUID);
- rowErrors.add(rowError);
- currentErrorIndex++;
- }
- individualResponses.add(
- new OperationResponse(currentOperation.deadlineTracker.getElapsedMillis(), tsUUID,
- writeTimestamp, currentOperation, rowError));
- }
- assert (rowErrors.size() == errorsPB.size());
- assert (individualResponses.size() == operations.size());
- }
-
- BatchResponse(List<OperationResponse> individualResponses) {
- super(0, null);
- writeTimestamp = 0;
- rowErrors = ImmutableList.of();
- this.individualResponses = individualResponses;
- }
-
- /**
- * Gives the write timestamp that was returned by the Tablet Server.
- * @return a timestamp in milliseconds, 0 if the external consistency mode set in AsyncKuduSession
- * wasn't CLIENT_PROPAGATED
- */
- public long getWriteTimestamp() {
- return writeTimestamp;
- }
-
- /**
- * Package-private method to get the individual responses.
- * @return a list of OperationResponses
- */
- List<OperationResponse> getIndividualResponses() {
- return individualResponses;
- }
-
-}
[36/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 2)
Posted by jd...@apache.org.
[java-client] repackage to org.apache.kudu (Part 2)
Find/replace org.kududb with org.apache.kudu.
Change-Id: If806b89fb35b56ecce3fc1034c5304f39e2c0867
Reviewed-on: http://gerrit.cloudera.org:8080/3737
Tested-by: Kudu Jenkins
Reviewed-by: Jean-Daniel Cryans <jd...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/81b2d9a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/81b2d9a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/81b2d9a7
Branch: refs/heads/master
Commit: 81b2d9a7c925174c525a85c04a6ffd5a6d8537f6
Parents: 5c30568
Author: Dan Burkert <da...@cloudera.com>
Authored: Sat Jul 23 11:30:21 2016 -0700
Committer: Jean-Daniel Cryans <jd...@apache.org>
Committed: Mon Jul 25 17:14:23 2016 +0000
----------------------------------------------------------------------
java/interface-annotations/pom.xml | 2 +-
.../kudu/annotations/InterfaceAudience.java | 2 +-
.../kudu/annotations/InterfaceStability.java | 8 +++---
.../ExcludePrivateAnnotationsJDiffDoclet.java | 4 +--
...ExcludePrivateAnnotationsStandardDoclet.java | 4 +--
.../IncludePublicAnnotationsStandardDoclet.java | 4 +--
.../annotations/tools/RootDocProcessor.java | 6 ++--
.../annotations/tools/StabilityOptions.java | 2 +-
.../kudu/annotations/tools/package-info.java | 4 +--
java/kudu-client-tools/pom.xml | 8 +++---
.../apache/kudu/mapreduce/tools/CsvParser.java | 8 +++---
.../apache/kudu/mapreduce/tools/ImportCsv.java | 10 +++----
.../kudu/mapreduce/tools/ImportCsvMapper.java | 16 +++++------
.../tools/IntegrationTestBigLinkedList.java | 20 ++++++-------
.../apache/kudu/mapreduce/tools/RowCounter.java | 12 ++++----
.../kudu/mapreduce/tools/ITImportCsv.java | 18 ++++++------
.../tools/ITIntegrationTestBigLinkedList.java | 8 +++---
.../kudu/mapreduce/tools/ITRowCounter.java | 8 +++---
java/kudu-client/pom.xml | 12 ++++----
.../main/java/org/apache/kudu/ColumnSchema.java | 10 +++----
.../src/main/java/org/apache/kudu/Schema.java | 10 +++----
.../src/main/java/org/apache/kudu/Type.java | 8 +++---
.../kudu/client/AbstractKuduScannerBuilder.java | 12 ++++----
.../apache/kudu/client/AlterTableOptions.java | 12 ++++----
.../apache/kudu/client/AlterTableRequest.java | 8 +++---
.../apache/kudu/client/AlterTableResponse.java | 6 ++--
.../org/apache/kudu/client/AsyncKuduClient.java | 26 ++++++++---------
.../apache/kudu/client/AsyncKuduScanner.java | 24 ++++++++--------
.../apache/kudu/client/AsyncKuduSession.java | 12 ++++----
.../main/java/org/apache/kudu/client/Batch.java | 18 ++++++------
.../org/apache/kudu/client/BatchResponse.java | 6 ++--
.../main/java/org/apache/kudu/client/Bytes.java | 6 ++--
.../org/apache/kudu/client/CallResponse.java | 8 +++---
.../kudu/client/ColumnRangePredicate.java | 10 +++----
.../apache/kudu/client/CreateTableOptions.java | 12 ++++----
.../apache/kudu/client/CreateTableRequest.java | 10 +++----
.../apache/kudu/client/CreateTableResponse.java | 6 ++--
.../org/apache/kudu/client/DeadlineTracker.java | 2 +-
.../java/org/apache/kudu/client/Delete.java | 10 +++----
.../apache/kudu/client/DeleteTableRequest.java | 8 +++---
.../apache/kudu/client/DeleteTableResponse.java | 6 ++--
.../org/apache/kudu/client/ErrorCollector.java | 6 ++--
.../kudu/client/ExternalConsistencyMode.java | 8 +++---
.../client/GetMasterRegistrationReceived.java | 12 ++++----
.../client/GetMasterRegistrationRequest.java | 10 +++----
.../client/GetMasterRegistrationResponse.java | 10 +++----
.../kudu/client/GetTableLocationsRequest.java | 8 +++---
.../kudu/client/GetTableSchemaRequest.java | 10 +++----
.../kudu/client/GetTableSchemaResponse.java | 6 ++--
.../kudu/client/HasFailedRpcException.java | 6 ++--
.../java/org/apache/kudu/client/IPCUtil.java | 4 +--
.../java/org/apache/kudu/client/Insert.java | 6 ++--
.../kudu/client/IsAlterTableDoneRequest.java | 8 +++---
.../kudu/client/IsAlterTableDoneResponse.java | 6 ++--
.../kudu/client/IsCreateTableDoneRequest.java | 8 +++---
.../java/org/apache/kudu/client/KeyEncoder.java | 12 ++++----
.../java/org/apache/kudu/client/KuduClient.java | 10 +++----
.../org/apache/kudu/client/KuduException.java | 6 ++--
.../org/apache/kudu/client/KuduPredicate.java | 14 ++++-----
.../java/org/apache/kudu/client/KuduRpc.java | 10 +++----
.../org/apache/kudu/client/KuduRpcResponse.java | 4 +--
.../org/apache/kudu/client/KuduScanToken.java | 12 ++++----
.../org/apache/kudu/client/KuduScanner.java | 10 +++----
.../org/apache/kudu/client/KuduSession.java | 4 +--
.../java/org/apache/kudu/client/KuduTable.java | 8 +++---
.../apache/kudu/client/ListTablesRequest.java | 8 +++---
.../apache/kudu/client/ListTablesResponse.java | 6 ++--
.../kudu/client/ListTabletServersRequest.java | 8 +++---
.../kudu/client/ListTabletServersResponse.java | 6 ++--
.../apache/kudu/client/ListTabletsRequest.java | 12 ++++----
.../apache/kudu/client/ListTabletsResponse.java | 4 +--
.../org/apache/kudu/client/LocatedTablet.java | 10 +++----
.../client/NoLeaderMasterFoundException.java | 6 ++--
.../kudu/client/NonCoveredRangeCache.java | 4 +--
.../kudu/client/NonCoveredRangeException.java | 6 ++--
.../kudu/client/NonRecoverableException.java | 6 ++--
.../java/org/apache/kudu/client/Operation.java | 26 ++++++++---------
.../apache/kudu/client/OperationResponse.java | 8 +++---
.../java/org/apache/kudu/client/PartialRow.java | 12 ++++----
.../java/org/apache/kudu/client/Partition.java | 6 ++--
.../org/apache/kudu/client/PartitionSchema.java | 8 +++---
.../kudu/client/PleaseThrottleException.java | 6 ++--
.../org/apache/kudu/client/ProtobufHelper.java | 16 +++++------
.../kudu/client/RecoverableException.java | 6 ++--
.../org/apache/kudu/client/RequestTracker.java | 4 +--
.../java/org/apache/kudu/client/RowError.java | 10 +++----
.../kudu/client/RowErrorsAndOverflowStatus.java | 6 ++--
.../java/org/apache/kudu/client/RowResult.java | 16 +++++------
.../apache/kudu/client/RowResultIterator.java | 12 ++++----
.../org/apache/kudu/client/SecureRpcHelper.java | 6 ++--
.../kudu/client/SessionConfiguration.java | 6 ++--
.../java/org/apache/kudu/client/Statistics.java | 10 +++----
.../java/org/apache/kudu/client/Status.java | 12 ++++----
.../org/apache/kudu/client/TabletClient.java | 14 ++++-----
.../java/org/apache/kudu/client/Update.java | 6 ++--
.../java/org/apache/kudu/client/Upsert.java | 6 ++--
.../java/org/apache/kudu/util/AsyncUtil.java | 4 +--
.../org/apache/kudu/util/HybridTimeUtil.java | 4 +--
.../main/java/org/apache/kudu/util/NetUtil.java | 4 +--
.../main/java/org/apache/kudu/util/Pair.java | 4 +--
.../main/java/org/apache/kudu/util/Slice.java | 4 +--
.../main/java/org/apache/kudu/util/Slices.java | 4 +--
.../org/apache/kudu/client/BaseKuduTest.java | 10 +++----
.../java/org/apache/kudu/client/ITClient.java | 2 +-
.../kudu/client/ITScannerMultiTablet.java | 4 +--
.../org/apache/kudu/client/MiniKuduCluster.java | 4 +--
.../apache/kudu/client/TestAsyncKuduClient.java | 8 +++---
.../kudu/client/TestAsyncKuduSession.java | 10 +++----
.../java/org/apache/kudu/client/TestBitSet.java | 2 +-
.../java/org/apache/kudu/client/TestBytes.java | 2 +-
.../kudu/client/TestColumnRangePredicate.java | 8 +++---
.../apache/kudu/client/TestDeadlineTracker.java | 2 +-
.../apache/kudu/client/TestErrorCollector.java | 2 +-
.../kudu/client/TestFlexiblePartitioning.java | 8 +++---
.../org/apache/kudu/client/TestHybridTime.java | 12 ++++----
.../org/apache/kudu/client/TestKeyEncoding.java | 16 +++++------
.../org/apache/kudu/client/TestKuduClient.java | 18 ++++++------
.../apache/kudu/client/TestKuduPredicate.java | 20 ++++++-------
.../org/apache/kudu/client/TestKuduSession.java | 2 +-
.../org/apache/kudu/client/TestKuduTable.java | 8 +++---
.../apache/kudu/client/TestLeaderFailover.java | 2 +-
.../apache/kudu/client/TestMasterFailover.java | 2 +-
.../apache/kudu/client/TestMiniKuduCluster.java | 2 +-
.../org/apache/kudu/client/TestOperation.java | 14 ++++-----
.../apache/kudu/client/TestRequestTracker.java | 2 +-
.../org/apache/kudu/client/TestRowErrors.java | 2 +-
.../org/apache/kudu/client/TestRowResult.java | 4 +--
.../apache/kudu/client/TestScanPredicate.java | 10 +++----
.../kudu/client/TestScannerMultiTablet.java | 8 +++---
.../org/apache/kudu/client/TestStatistics.java | 4 +--
.../java/org/apache/kudu/client/TestStatus.java | 4 +--
.../org/apache/kudu/client/TestTestUtils.java | 2 +-
.../org/apache/kudu/client/TestTimeouts.java | 2 +-
.../java/org/apache/kudu/client/TestUtils.java | 2 +-
.../org/apache/kudu/util/TestAsyncUtil.java | 2 +-
.../org/apache/kudu/util/TestMurmurHash.java | 2 +-
.../java/org/apache/kudu/util/TestNetUtil.java | 2 +-
java/kudu-csd/pom.xml | 2 +-
java/kudu-flume-sink/pom.xml | 6 ++--
.../kudu/flume/sink/KuduEventProducer.java | 12 ++++----
.../org/apache/kudu/flume/sink/KuduSink.java | 26 ++++++++---------
.../sink/KuduSinkConfigurationConstants.java | 6 ++--
.../flume/sink/SimpleKuduEventProducer.java | 10 +++----
.../apache/kudu/flume/sink/KuduSinkTest.java | 14 ++++-----
java/kudu-mapreduce/pom.xml | 6 ++--
.../kudu/mapreduce/CommandLineParser.java | 10 +++----
.../org/apache/kudu/mapreduce/JarFinder.java | 2 +-
.../kudu/mapreduce/KuduTableInputFormat.java | 30 ++++++++++----------
.../kudu/mapreduce/KuduTableMapReduceUtil.java | 16 +++++------
.../mapreduce/KuduTableOutputCommitter.java | 6 ++--
.../kudu/mapreduce/KuduTableOutputFormat.java | 8 +++---
.../org/apache/kudu/mapreduce/TableReducer.java | 8 +++---
.../kudu/mapreduce/HadoopTestingUtility.java | 2 +-
.../apache/kudu/mapreduce/ITInputFormatJob.java | 8 +++---
.../kudu/mapreduce/ITKuduTableInputFormat.java | 6 ++--
.../kudu/mapreduce/ITKuduTableOutputFormat.java | 4 +--
.../kudu/mapreduce/ITOutputFormatJob.java | 4 +--
.../apache/kudu/mapreduce/TestJarFinder.java | 2 +-
java/kudu-spark/pom.xml | 8 +++---
.../apache/kudu/spark/kudu/DefaultSource.scala | 12 ++++----
.../apache/kudu/spark/kudu/KuduContext.scala | 10 +++----
.../org/apache/kudu/spark/kudu/KuduRDD.scala | 6 ++--
.../org/apache/kudu/spark/kudu/package.scala | 6 ++--
.../src/test/resources/log4j.properties | 2 +-
.../kudu/spark/kudu/DefaultSourceTest.scala | 4 +--
.../kudu/spark/kudu/KuduContextTest.scala | 2 +-
.../apache/kudu/spark/kudu/TestContext.scala | 12 ++++----
java/pom.xml | 10 +++----
src/kudu/cfile/cfile.proto | 2 +-
src/kudu/client/client.proto | 2 +-
src/kudu/common/common.proto | 2 +-
src/kudu/common/wire_protocol.proto | 4 +--
src/kudu/consensus/consensus.proto | 2 +-
src/kudu/consensus/log.proto | 2 +-
src/kudu/consensus/metadata.proto | 2 +-
src/kudu/consensus/opid.proto | 2 +-
src/kudu/fs/fs.proto | 2 +-
src/kudu/master/master.proto | 2 +-
src/kudu/rpc/rpc_header.proto | 2 +-
src/kudu/rpc/rpc_introspection.proto | 4 +--
src/kudu/server/server_base.proto | 2 +-
src/kudu/tablet/metadata.proto | 2 +-
src/kudu/tablet/tablet.proto | 2 +-
src/kudu/tserver/remote_bootstrap.proto | 2 +-
src/kudu/tserver/tserver.proto | 2 +-
src/kudu/tserver/tserver_admin.proto | 2 +-
src/kudu/tserver/tserver_service.proto | 2 +-
src/kudu/util/histogram.proto | 2 +-
src/kudu/util/maintenance_manager.proto | 4 +--
src/kudu/util/pb_util.proto | 2 +-
src/kudu/util/version_info.proto | 2 +-
191 files changed, 698 insertions(+), 698 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/java/interface-annotations/pom.xml b/java/interface-annotations/pom.xml
index 095a1aa..d91c55b 100644
--- a/java/interface-annotations/pom.xml
+++ b/java/interface-annotations/pom.xml
@@ -19,7 +19,7 @@
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java
index b834947..d254c56 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceAudience.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.annotations;
+package org.apache.kudu.annotations;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java
index 84950e6..f589ada 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/InterfaceStability.java
@@ -16,15 +16,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.annotations;
+package org.apache.kudu.annotations;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
-import org.kududb.annotations.InterfaceAudience.LimitedPrivate;
-import org.kududb.annotations.InterfaceAudience.Private;
-import org.kududb.annotations.InterfaceAudience.Public;
+import org.apache.kudu.annotations.InterfaceAudience.LimitedPrivate;
+import org.apache.kudu.annotations.InterfaceAudience.Private;
+import org.apache.kudu.annotations.InterfaceAudience.Public;
/**
* Annotation to inform users of how much to rely on a particular package,
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
index 5c0c7b8..dd04806 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsJDiffDoclet.java
@@ -16,14 +16,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.annotations.tools;
+package org.apache.kudu.annotations.tools;
import com.sun.javadoc.DocErrorReporter;
import com.sun.javadoc.LanguageVersion;
import com.sun.javadoc.RootDoc;
import jdiff.JDiff;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
/**
* A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
index af8b088..4b1165c 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/ExcludePrivateAnnotationsStandardDoclet.java
@@ -16,13 +16,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.annotations.tools;
+package org.apache.kudu.annotations.tools;
import com.sun.javadoc.DocErrorReporter;
import com.sun.javadoc.LanguageVersion;
import com.sun.javadoc.RootDoc;
import com.sun.tools.doclets.standard.Standard;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
/**
* A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
index b5a67b2..ea10e6a 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/IncludePublicAnnotationsStandardDoclet.java
@@ -16,13 +16,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.annotations.tools;
+package org.apache.kudu.annotations.tools;
import com.sun.javadoc.DocErrorReporter;
import com.sun.javadoc.LanguageVersion;
import com.sun.javadoc.RootDoc;
import com.sun.tools.doclets.standard.Standard;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
/**
* A <a href="http://java.sun.com/javase/6/docs/jdk/api/javadoc/doclet/">Doclet</a>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java
index c4f19fb..9fa5f4b 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/RootDocProcessor.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.annotations.tools;
+package org.apache.kudu.annotations.tools;
import com.sun.javadoc.AnnotationDesc;
import com.sun.javadoc.AnnotationTypeDoc;
@@ -39,8 +39,8 @@ import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Process the {@link RootDoc} by substituting with (nested) proxy objects that
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java
index d5cf5e1..c435bcf 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/StabilityOptions.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.annotations.tools;
+package org.apache.kudu.annotations.tools;
import com.sun.javadoc.DocErrorReporter;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java
----------------------------------------------------------------------
diff --git a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java
index ec0103b..8659a5a 100644
--- a/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java
+++ b/java/interface-annotations/src/main/java/org/apache/kudu/annotations/tools/package-info.java
@@ -17,6 +17,6 @@
* limitations under the License.
*/
@InterfaceAudience.Private
-package org.kududb.annotations.tools;
+package org.apache.kudu.annotations.tools;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/pom.xml b/java/kudu-client-tools/pom.xml
index 846f174..a63c172 100644
--- a/java/kudu-client-tools/pom.xml
+++ b/java/kudu-client-tools/pom.xml
@@ -21,7 +21,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
</parent>
@@ -31,19 +31,19 @@
<dependencies>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-mapreduce</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-mapreduce</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java
index 945b82c..7e42bd4 100644
--- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/CsvParser.java
@@ -12,14 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.mapreduce.tools;
+package org.apache.kudu.mapreduce.tools;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Bytes;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.Bytes;
import java.util.ArrayList;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java
index f407c5c..8c1c64c 100644
--- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsv.java
@@ -12,12 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.mapreduce.tools;
+package org.apache.kudu.mapreduce.tools;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.mapreduce.CommandLineParser;
+import org.apache.kudu.mapreduce.KuduTableMapReduceUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java
index 21f43b5..6d6e11b 100644
--- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/ImportCsvMapper.java
@@ -12,14 +12,14 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.mapreduce.tools;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.*;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
+package org.apache.kudu.mapreduce.tools;
+
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.*;
+import org.apache.kudu.mapreduce.KuduTableMapReduceUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
index 549e117..7a2d6ac 100644
--- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java
@@ -12,19 +12,19 @@
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
-package org.kududb.mapreduce.tools;
+package org.apache.kudu.mapreduce.tools;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.*;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
-import org.kududb.util.Pair;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.*;
+import org.apache.kudu.mapreduce.CommandLineParser;
+import org.apache.kudu.mapreduce.KuduTableMapReduceUtil;
+import org.apache.kudu.util.Pair;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java
index 45e7837..112a8cc 100644
--- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java
+++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/RowCounter.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce.tools;
+package org.apache.kudu.mapreduce.tools;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.KuduTableMapReduceUtil;
-import org.kududb.client.RowResult;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.mapreduce.CommandLineParser;
+import org.apache.kudu.mapreduce.KuduTableMapReduceUtil;
+import org.apache.kudu.client.RowResult;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.NullWritable;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
index d7b8352..3e76382 100644
--- a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
+++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
@@ -14,15 +14,15 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce.tools;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.HadoopTestingUtility;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.client.CreateTableOptions;
+package org.apache.kudu.mapreduce.tools;
+
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.mapreduce.CommandLineParser;
+import org.apache.kudu.mapreduce.HadoopTestingUtility;
+import org.apache.kudu.client.BaseKuduTest;
+import org.apache.kudu.client.CreateTableOptions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java
index 311c5ee..32e5d15 100644
--- a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java
+++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITIntegrationTestBigLinkedList.java
@@ -15,16 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce.tools;
+package org.apache.kudu.mapreduce.tools;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Test;
-import org.kududb.client.BaseKuduTest;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.HadoopTestingUtility;
+import org.apache.kudu.client.BaseKuduTest;
+import org.apache.kudu.mapreduce.CommandLineParser;
+import org.apache.kudu.mapreduce.HadoopTestingUtility;
public class ITIntegrationTestBigLinkedList extends BaseKuduTest {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java
----------------------------------------------------------------------
diff --git a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java
index 52984bb..6d590a6 100644
--- a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java
+++ b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITRowCounter.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.mapreduce.tools;
+package org.apache.kudu.mapreduce.tools;
-import org.kududb.mapreduce.CommandLineParser;
-import org.kududb.mapreduce.HadoopTestingUtility;
-import org.kududb.client.BaseKuduTest;
+import org.apache.kudu.mapreduce.CommandLineParser;
+import org.apache.kudu.mapreduce.HadoopTestingUtility;
+import org.apache.kudu.client.BaseKuduTest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-client/pom.xml b/java/kudu-client/pom.xml
index 64c6649..135c631 100644
--- a/java/kudu-client/pom.xml
+++ b/java/kudu-client/pom.xml
@@ -21,7 +21,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-parent</artifactId>
<version>0.10.0-SNAPSHOT</version>
</parent>
@@ -31,7 +31,7 @@
<dependencies>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>interface-annotations</artifactId>
<version>${project.version}</version>
</dependency>
@@ -183,19 +183,19 @@
<relocations>
<relocation>
<pattern>com.google.common</pattern>
- <shadedPattern>org.kududb.client.shaded.com.google.common</shadedPattern>
+ <shadedPattern>org.apache.kudu.client.shaded.com.google.common</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.protobuf</pattern>
- <shadedPattern>org.kududb.client.shaded.com.google.protobuf</shadedPattern>
+ <shadedPattern>org.apache.kudu.client.shaded.com.google.protobuf</shadedPattern>
</relocation>
<relocation>
<pattern>com.sangupta</pattern>
- <shadedPattern>org.kududb.client.shaded.com.sangupta</shadedPattern>
+ <shadedPattern>org.apache.kudu.client.shaded.com.sangupta</shadedPattern>
</relocation>
<relocation>
<pattern>org.jboss.netty</pattern>
- <shadedPattern>org.kududb.client.shaded.org.jboss.netty</shadedPattern>
+ <shadedPattern>org.apache.kudu.client.shaded.org.jboss.netty</shadedPattern>
</relocation>
</relocations>
<shadeTestJar>true</shadeTestJar>
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
index c3c617d..ac71ea2 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb;
+package org.apache.kudu;
-import org.kududb.Common.CompressionType;
-import org.kududb.Common.EncodingType;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.Common.CompressionType;
+import org.apache.kudu.Common.EncodingType;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Represents a Kudu Table column. Use {@link ColumnSchema.ColumnSchemaBuilder} in order to
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/Schema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/Schema.java b/java/kudu-client/src/main/java/org/apache/kudu/Schema.java
index 17e7798..74a0587 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/Schema.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/Schema.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb;
+package org.apache.kudu;
import com.google.common.collect.ImmutableList;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Bytes;
-import org.kududb.client.PartialRow;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.client.Bytes;
+import org.apache.kudu.client.PartialRow;
import java.util.ArrayList;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/Type.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/Type.java b/java/kudu-client/src/main/java/org/apache/kudu/Type.java
index fd23835..fed9dbb 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/Type.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/Type.java
@@ -14,15 +14,15 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb;
+package org.apache.kudu;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import com.google.common.primitives.Shorts;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
-import static org.kududb.Common.DataType;
+import static org.apache.kudu.Common.DataType;
/**
* Describes all the types available to build table schemas.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java
index b1b0712..47a391a 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AbstractKuduScannerBuilder.java
@@ -14,18 +14,18 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.google.common.collect.ImmutableList;
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.HybridTimeUtil;
+import org.apache.kudu.Common;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.tserver.Tserver;
+import org.apache.kudu.util.HybridTimeUtil;
/**
* Abstract class to extend in order to create builders for scanners.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
index ecffb5e..1d97ca3 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
-import static org.kududb.master.Master.AlterTableRequestPB;
+import static org.apache.kudu.master.Master.AlterTableRequestPB;
/**
* This builder must be used to alter a table. At least one change must be specified.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
index 751290c..bf3dce2 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableRequest.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
-import static org.kududb.master.Master.*;
+import static org.apache.kudu.master.Master.*;
/**
* RPC used to alter a table. When it returns it doesn't mean that the table is altered,
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java
index 7d1d581..116816c 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableResponse.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index c1ccb3f..fc29b71 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -24,7 +24,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
@@ -40,17 +40,17 @@ import com.stumbleupon.async.Deferred;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.socket.nio.NioWorkerPool;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
-import org.kududb.master.Master.GetTableLocationsResponsePB;
-import org.kududb.util.AsyncUtil;
-import org.kududb.util.NetUtil;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
+import org.apache.kudu.Common;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.consensus.Metadata;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.master.Master.GetTableLocationsResponsePB;
+import org.apache.kudu.util.AsyncUtil;
+import org.apache.kudu.util.NetUtil;
+import org.apache.kudu.util.Pair;
+import org.apache.kudu.util.Slice;
import org.jboss.netty.channel.ChannelEvent;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.DefaultChannelPipeline;
@@ -92,7 +92,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+import static org.apache.kudu.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
/**
* A fully asynchronous and thread-safe client for Kudu.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
index 7699536..b80e765 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
@@ -23,7 +23,7 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.Message;
@@ -31,13 +31,13 @@ import com.google.protobuf.ZeroCopyLiteralByteString;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
import org.jboss.netty.buffer.ChannelBuffer;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.Pair;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Common;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.tserver.Tserver;
+import org.apache.kudu.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,10 +48,10 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import static com.google.common.base.Preconditions.checkArgument;
-import static org.kududb.tserver.Tserver.NewScanRequestPB;
-import static org.kududb.tserver.Tserver.ScanRequestPB;
-import static org.kududb.tserver.Tserver.ScanResponsePB;
-import static org.kududb.tserver.Tserver.TabletServerErrorPB;
+import static org.apache.kudu.tserver.Tserver.NewScanRequestPB;
+import static org.apache.kudu.tserver.Tserver.ScanRequestPB;
+import static org.apache.kudu.tserver.Tserver.ScanResponsePB;
+import static org.apache.kudu.tserver.Tserver.TabletServerErrorPB;
/**
* Creates a scanner to read data from Kudu.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java
index 0290ee7..5b126f2 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduSession.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
@@ -23,10 +23,10 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Range;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.util.AsyncUtil;
-import org.kududb.util.Slice;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.util.AsyncUtil;
+import org.apache.kudu.util.Slice;
import org.jboss.netty.util.Timeout;
import org.jboss.netty.util.TimerTask;
import org.slf4j.Logger;
@@ -44,7 +44,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-import static org.kududb.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
+import static org.apache.kudu.client.ExternalConsistencyMode.CLIENT_PROPAGATED;
/**
* A AsyncKuduSession belongs to a specific AsyncKuduClient, and represents a context in
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java
index ed1d870..a0dacdd 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Batch.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
@@ -26,14 +26,14 @@ import java.util.Collections;
import java.util.List;
import org.jboss.netty.buffer.ChannelBuffer;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.client.Statistics.Statistic;
-import org.kududb.client.Statistics.TabletStatistics;
-import org.kududb.tserver.Tserver;
-import org.kududb.tserver.Tserver.TabletServerErrorPB;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
+import org.apache.kudu.WireProtocol;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.client.Statistics.Statistic;
+import org.apache.kudu.client.Statistics.TabletStatistics;
+import org.apache.kudu.tserver.Tserver;
+import org.apache.kudu.tserver.Tserver.TabletServerErrorPB;
+import org.apache.kudu.util.Pair;
+import org.apache.kudu.util.Slice;
/**
* Used internally to group Operations for a single tablet together before sending to the tablet
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java
index f67153b..8dd564a 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/BatchResponse.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
@@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.tserver.Tserver;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.tserver.Tserver;
/**
* Response type for Batch (which is used internally by AsyncKuduSession).
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java
index 0e68e93..9f2d021 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Bytes.java
@@ -23,13 +23,13 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.io.BaseEncoding;
import com.google.protobuf.ByteString;
import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Slice;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.util.Slice;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.util.CharsetUtil;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java
index 77cbd5e..631cc66 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CallResponse.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import java.util.List;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.rpc.RpcHeader;
-import org.kududb.util.Slice;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.rpc.RpcHeader;
+import org.apache.kudu.util.Slice;
import org.jboss.netty.buffer.ChannelBuffer;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java
index 0e24088..e643c81 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ColumnRangePredicate.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.tserver.Tserver;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.tserver.Tserver;
import java.util.ArrayList;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
index 20bc4c3..a352183 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
@@ -14,18 +14,18 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.List;
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
+import org.apache.kudu.Common;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.util.Pair;
/**
* This is a builder class for all the options that can be provided while creating a table.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java
index 31ed9a2..6dc00cb 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableRequest.java
@@ -14,17 +14,17 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
import java.util.Collection;
import java.util.List;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java
index 7906c5f..7fa75b2 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableResponse.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
@InterfaceAudience.Private
public class CreateTableResponse extends KuduRpcResponse {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java b/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
index fc30074..226952e 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/DeadlineTracker.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Stopwatch;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java
index 7a068bc..295d093 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Delete.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Class of Operation for whole row removals.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java
index 7f8fa51..8060d54 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableRequest.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java
index 51f3ba7..9a75769 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/DeleteTableResponse.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java
index db2952c..dda012b 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ErrorCollector.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Preconditions;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import java.util.ArrayDeque;
import java.util.Queue;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java b/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java
index adfb624..9f08a50 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ExternalConsistencyMode.java
@@ -14,11 +14,11 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.Common;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* The possible external consistency modes on which Kudu operates.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java
index f1a9075..ad4d0bb 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationReceived.java
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
@@ -23,11 +23,11 @@ import com.google.common.net.HostAndPort;
import com.google.protobuf.ByteString;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.Common;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
-import org.kududb.util.NetUtil;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.Common;
+import org.apache.kudu.consensus.Metadata;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.util.NetUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java
index bc3d81e..2e76e30 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationRequest.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import static org.kududb.consensus.Metadata.*;
-import static org.kududb.master.Master.*;
+import static org.apache.kudu.consensus.Metadata.*;
+import static org.apache.kudu.master.Master.*;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java
index 292710c..559861f 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetMasterRegistrationResponse.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.WireProtocol;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.consensus.Metadata;
-import org.kududb.master.Master;
+import org.apache.kudu.WireProtocol;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.consensus.Metadata;
+import org.apache.kudu.master.Master;
/**
* Response for {@link GetMasterRegistrationRequest}.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java
index 616b523..953525a 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableLocationsRequest.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java
index bb17816..d76b24a 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaRequest.java
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import static org.kududb.master.Master.*;
+import static org.apache.kudu.master.Master.*;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java
index 72ac68e..7421970 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/GetTableSchemaResponse.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
@InterfaceAudience.Private
public class GetTableSchemaResponse extends KuduRpcResponse {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java b/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java
index 08dda52..fa78d0a 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/HasFailedRpcException.java
@@ -23,10 +23,10 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Interface implemented by {@link KuduException}s that can tell you which
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java
index 45240dd..9da17ed 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IPCUtil.java
@@ -16,11 +16,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceAudience;
import java.io.IOException;
import java.io.OutputStream;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java
index 67b389f..36490ea 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Insert.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Represents a single row insert. Instances of this class should not be reused.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java
index ca161f5..d923984 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneRequest.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.Message;
-import static org.kududb.master.Master.*;
+import static org.apache.kudu.master.Master.*;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java
index 356c085..4dcc424 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IsAlterTableDoneResponse.java
@@ -14,10 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
/**
* Response to a isAlterTableDone command to use to know if an alter table is currently running on
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java b/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java
index 8e4679c..61b9ef6 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/IsCreateTableDoneRequest.java
@@ -14,13 +14,13 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.master.Master;
-import org.kududb.util.Pair;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.master.Master;
+import org.apache.kudu.util.Pair;
import org.jboss.netty.buffer.ChannelBuffer;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
index 2fbde58..49a0a72 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KeyEncoder.java
@@ -14,15 +14,15 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.google.common.primitives.UnsignedLongs;
import com.sangupta.murmur.Murmur2;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.client.PartitionSchema.HashBucketSchema;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.client.PartitionSchema.HashBucketSchema;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/81b2d9a7/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
index cfd4662..edb6857 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
@@ -14,12 +14,12 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-package org.kududb.client;
+package org.apache.kudu.client;
import com.stumbleupon.async.Deferred;
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
+import org.apache.kudu.Schema;
+import org.apache.kudu.annotations.InterfaceAudience;
+import org.apache.kudu.annotations.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,7 +33,7 @@ import java.util.concurrent.Executor;
* <p>
* This class acts as a wrapper around {@link AsyncKuduClient}. The {@link Deferred} objects are
* joined against using the default admin operation timeout
- * (see {@link org.kududb.client.KuduClient.KuduClientBuilder#defaultAdminOperationTimeoutMs(long)} (long)}).
+ * (see {@link org.apache.kudu.client.KuduClient.KuduClientBuilder#defaultAdminOperationTimeoutMs(long)} (long)}).
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
[15/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ListTabletsResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ListTabletsResponse.java b/java/kudu-client/src/main/java/org/kududb/client/ListTabletsResponse.java
deleted file mode 100644
index be2ed65..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ListTabletsResponse.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-
-import java.util.List;
-
-@InterfaceAudience.Private
-public class ListTabletsResponse extends KuduRpcResponse {
-
- private final List<String> tabletsList;
-
- ListTabletsResponse(long ellapsedMillis, String tsUUID, List<String> tabletsList) {
- super(ellapsedMillis, tsUUID);
- this.tabletsList = tabletsList;
- }
-
- /**
- * Get the list of tablets as specified in the request.
- * @return a list of tablet uuids
- */
- public List<String> getTabletsList() {
- return tabletsList;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/LocatedTablet.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/LocatedTablet.java b/java/kudu-client/src/main/java/org/kududb/client/LocatedTablet.java
deleted file mode 100644
index 67934db..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/LocatedTablet.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.kududb.client;
-
-import java.util.List;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.consensus.Metadata.RaftPeerPB.Role;
-import org.kududb.master.Master.TabletLocationsPB.ReplicaPB;
-
-/**
- * Information about the locations of tablets in a Kudu table.
- * This should be treated as immutable data (it does not reflect
- * any updates the client may have heard since being constructed).
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class LocatedTablet {
- private final Partition partition;
- private final byte[] tabletId;
-
- private final List<Replica> replicas;
-
- LocatedTablet(AsyncKuduClient.RemoteTablet tablet) {
- partition = tablet.getPartition();
- tabletId = tablet.getTabletIdAsBytes();
- replicas = tablet.getReplicas();
- }
-
- public List<Replica> getReplicas() {
- return replicas;
- }
-
- public Partition getPartition() {
- return partition;
- }
-
- /**
- * DEPRECATED: use {@link #getPartition()}
- */
- @Deprecated
- public byte[] getStartKey() {
- return getPartition().getPartitionKeyStart();
- }
-
- /**
- * DEPRECATED: use {@link #getPartition()}
- */
- @Deprecated()
- public byte[] getEndKey() {
- return getPartition().getPartitionKeyEnd();
- }
-
- public byte[] getTabletId() {
- return tabletId;
- }
-
- /**
- * Return the current leader, or null if there is none.
- */
- public Replica getLeaderReplica() {
- return getOneOfRoleOrNull(Role.LEADER);
- }
-
- /**
- * Return the first occurrence for the given role, or null if there is none.
- */
- private Replica getOneOfRoleOrNull(Role role) {
- for (Replica r : replicas) {
- if (r.getRole() == role.toString()) return r;
- }
- return null;
- }
-
- @Override
- public String toString() {
- return Bytes.pretty(tabletId) + " " + partition.toString();
- }
-
- /**
- * One of the replicas of the tablet.
- */
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public static class Replica {
- private final ReplicaPB pb;
-
- Replica(ReplicaPB pb) {
- this.pb = pb;
- }
-
- public String getRpcHost() {
- if (pb.getTsInfo().getRpcAddressesList().isEmpty()) {
- return null;
- }
- return pb.getTsInfo().getRpcAddressesList().get(0).getHost();
- }
-
- public Integer getRpcPort() {
- if (pb.getTsInfo().getRpcAddressesList().isEmpty()) {
- return null;
- }
- return pb.getTsInfo().getRpcAddressesList().get(0).getPort();
- }
-
- public String getRole() {
- return pb.getRole().toString();
- }
-
- public String toString() {
- return pb.toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/NoLeaderMasterFoundException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/NoLeaderMasterFoundException.java b/java/kudu-client/src/main/java/org/kududb/client/NoLeaderMasterFoundException.java
deleted file mode 100644
index 1cde694..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/NoLeaderMasterFoundException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.util.List;
-
-/**
- * Indicates that the request failed because we couldn't find a leader master server.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-final class NoLeaderMasterFoundException extends RecoverableException {
-
- NoLeaderMasterFoundException(Status status) {
- super(status);
- }
- NoLeaderMasterFoundException(Status status, Exception cause) {
- super(status, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeCache.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeCache.java b/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeCache.java
deleted file mode 100644
index 1c3b024..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeCache.java
+++ /dev/null
@@ -1,104 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Joiner;
-import com.google.common.primitives.UnsignedBytes;
-
-import java.util.Comparator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import javax.annotation.concurrent.ThreadSafe;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A cache of the non-covered range partitions in a Kudu table.
- *
- * Currently entries are never invalidated from the cache.
- */
-@ThreadSafe
-@InterfaceAudience.Private
-class NonCoveredRangeCache {
- private static final Logger LOG = LoggerFactory.getLogger(NonCoveredRangeCache.class);
- private static final Comparator<byte[]> COMPARATOR = UnsignedBytes.lexicographicalComparator();
-
- private final ConcurrentNavigableMap<byte[], byte[]> nonCoveredRanges =
- new ConcurrentSkipListMap<>(COMPARATOR);
-
- /**
- * Retrieves a non-covered range from the cache.
- *
- * The pair contains the inclusive start partition key and the exclusive end
- * partition key containing the provided partition key. If there is no such
- * cached range, null is returned.
- *
- * @param partitionKey the partition key to lookup in the cache
- * @return the non covered range, or null
- */
- public Map.Entry<byte[], byte[]> getNonCoveredRange(byte[] partitionKey) {
- Map.Entry<byte[], byte[]> range = nonCoveredRanges.floorEntry(partitionKey);
- if (range == null ||
- (range.getValue().length != 0 && COMPARATOR.compare(partitionKey, range.getValue()) >= 0)) {
- return null;
- } else {
- return range;
- }
- }
-
- /**
- * Adds a non-covered range to the cache.
- *
- * @param startPartitionKey the inclusive start partition key of the non-covered range
- * @param endPartitionKey the exclusive end partition key of the non-covered range
- */
- public void addNonCoveredRange(byte[] startPartitionKey, byte[] endPartitionKey) {
- if (startPartitionKey == null || endPartitionKey == null) {
- throw new IllegalArgumentException("Non-covered partition range keys may not be null");
- }
- // Concurrent additions of the same non-covered range key are handled by
- // serializing puts through the concurrent map.
- if (nonCoveredRanges.put(startPartitionKey, endPartitionKey) == null) {
- LOG.info("Discovered non-covered partition range [{}, {})",
- Bytes.hex(startPartitionKey), Bytes.hex(endPartitionKey));
- }
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append('[');
- boolean isFirst = true;
- for (Map.Entry<byte[], byte[]> range : nonCoveredRanges.entrySet()) {
- if (isFirst) {
- isFirst = false;
- } else {
- sb.append(", ");
- }
- sb.append('[');
- sb.append(range.getKey().length == 0 ? "<start>" : Bytes.hex(range.getKey()));
- sb.append(", ");
- sb.append(range.getValue().length == 0 ? "<end>" : Bytes.hex(range.getValue()));
- sb.append(')');
- }
- sb.append(']');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeException.java b/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeException.java
deleted file mode 100644
index b704441..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/NonCoveredRangeException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Exception indicating that an operation attempted to access a non-covered range partition.
- */
-@InterfaceAudience.Private
-class NonCoveredRangeException extends NonRecoverableException {
- private final byte[] nonCoveredRangeStart;
- private final byte[] nonCoveredRangeEnd;
-
- public NonCoveredRangeException(byte[] nonCoveredRangeStart, byte[] nonCoveredRangeEnd) {
- super(Status.NotFound("non-covered range"));
- this.nonCoveredRangeStart = nonCoveredRangeStart;
- this.nonCoveredRangeEnd = nonCoveredRangeEnd;
- }
-
- byte[] getNonCoveredRangeStart() {
- return nonCoveredRangeStart;
- }
-
- byte[] getNonCoveredRangeEnd() {
- return nonCoveredRangeEnd;
- }
-
- @Override
- public String toString() {
- return String.format(
- "NonCoveredRangeException([%s, %s))",
- nonCoveredRangeStart.length == 0 ? "<start>" : Bytes.hex(nonCoveredRangeStart),
- nonCoveredRangeEnd.length == 0 ? "<end>" : Bytes.hex(nonCoveredRangeEnd));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/NonRecoverableException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/NonRecoverableException.java b/java/kudu-client/src/main/java/org/kududb/client/NonRecoverableException.java
deleted file mode 100644
index 7bcb81d..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/NonRecoverableException.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-@SuppressWarnings("serial")
-class NonRecoverableException extends KuduException {
-
- /**
- * Constructor.
- * @param status status object containing the reason for the exception
- * trace.
- */
- NonRecoverableException(Status status) {
- super(status);
- }
-
- /**
- * Constructor.
- * @param status status object containing the reason for the exception
- * @param cause The exception that caused this one to be thrown.
- */
- NonRecoverableException(Status status, Throwable cause) {
- super(status, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Operation.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Operation.java b/java/kudu-client/src/main/java/org/kududb/client/Operation.java
deleted file mode 100644
index e27c222..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Operation.java
+++ /dev/null
@@ -1,345 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.collect.ImmutableList;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.Message;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.WireProtocol;
-import org.kududb.WireProtocol.RowOperationsPB;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Statistics.Statistic;
-import org.kududb.client.Statistics.TabletStatistics;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.Pair;
-import org.kududb.util.Slice;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Base class for the RPCs that related to WriteRequestPB. It contains almost all the logic
- * and knows how to serialize its child classes.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public abstract class Operation extends KuduRpc<OperationResponse> {
- /**
- * This size will be set when serialize is called. It stands for the size of the row in this
- * operation.
- */
- private long rowOperationSizeBytes = 0;
-
- enum ChangeType {
- INSERT((byte)RowOperationsPB.Type.INSERT.getNumber()),
- UPDATE((byte)RowOperationsPB.Type.UPDATE.getNumber()),
- DELETE((byte)RowOperationsPB.Type.DELETE.getNumber()),
- SPLIT_ROWS((byte)RowOperationsPB.Type.SPLIT_ROW.getNumber()),
- UPSERT((byte)RowOperationsPB.Type.UPSERT.getNumber()),
- RANGE_LOWER_BOUND((byte) RowOperationsPB.Type.RANGE_LOWER_BOUND.getNumber()),
- RANGE_UPPER_BOUND((byte) RowOperationsPB.Type.RANGE_UPPER_BOUND.getNumber());
-
- ChangeType(byte encodedByte) {
- this.encodedByte = encodedByte;
- }
-
- byte toEncodedByte() {
- return encodedByte;
- }
-
- /** The byte used to encode this in a RowOperationsPB */
- private byte encodedByte;
- }
-
- static final String METHOD = "Write";
-
- private final PartialRow row;
-
- /** See {@link SessionConfiguration#setIgnoreAllDuplicateRows(boolean)} */
- boolean ignoreAllDuplicateRows = false;
-
- /**
- * Package-private constructor. Subclasses need to be instantiated via AsyncKuduSession
- * @param table table with the schema to use for this operation
- */
- Operation(KuduTable table) {
- super(table);
- this.row = table.getSchema().newPartialRow();
- }
-
- /** See {@link SessionConfiguration#setIgnoreAllDuplicateRows(boolean)} */
- void setIgnoreAllDuplicateRows(boolean ignoreAllDuplicateRows) {
- this.ignoreAllDuplicateRows = ignoreAllDuplicateRows;
- }
-
- /**
- * Classes extending Operation need to have a specific ChangeType
- * @return Operation's ChangeType
- */
- abstract ChangeType getChangeType();
-
- /**
- * Returns the size in bytes of this operation's row after serialization.
- * @return size in bytes
- * @throws IllegalStateException thrown if this RPC hasn't been serialized eg sent to a TS
- */
- long getRowOperationSizeBytes() {
- if (this.rowOperationSizeBytes == 0) {
- throw new IllegalStateException("This row hasn't been serialized yet");
- }
- return this.rowOperationSizeBytes;
- }
-
- @Override
- String serviceName() { return TABLET_SERVER_SERVICE_NAME; }
-
- @Override
- String method() {
- return METHOD;
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- final Tserver.WriteRequestPB.Builder builder =
- createAndFillWriteRequestPB(ImmutableList.of(this));
- this.rowOperationSizeBytes = builder.getRowOperations().getRows().size()
- + builder.getRowOperations().getIndirectData().size();
- builder.setTabletId(ZeroCopyLiteralByteString.wrap(getTablet().getTabletIdAsBytes()));
- builder.setExternalConsistencyMode(this.externalConsistencyMode.pbVersion());
- if (this.propagatedTimestamp != AsyncKuduClient.NO_TIMESTAMP) {
- builder.setPropagatedTimestamp(this.propagatedTimestamp);
- }
- return toChannelBuffer(header, builder.build());
- }
-
- @Override
- Pair<OperationResponse, Object> deserialize(CallResponse callResponse,
- String tsUUID) throws Exception {
- Tserver.WriteResponsePB.Builder builder = Tserver.WriteResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), builder);
- Tserver.WriteResponsePB.PerRowErrorPB error = null;
- if (builder.getPerRowErrorsCount() != 0) {
- error = builder.getPerRowErrors(0);
- if (ignoreAllDuplicateRows &&
- error.getError().getCode() == WireProtocol.AppStatusPB.ErrorCode.ALREADY_PRESENT) {
- error = null;
- }
- }
- OperationResponse response = new OperationResponse(deadlineTracker.getElapsedMillis(), tsUUID,
- builder.getTimestamp(), this, error);
- return new Pair<OperationResponse, Object>(
- response, builder.hasError() ? builder.getError() : null);
- }
-
- @Override
- public byte[] partitionKey() {
- return this.getTable().getPartitionSchema().encodePartitionKey(row);
- }
-
- @Override
- boolean isRequestTracked() {
- return true;
- }
-
- /**
- * Get the underlying row to modify.
- * @return a partial row that will be sent with this Operation
- */
- public PartialRow getRow() {
- return this.row;
- }
-
- @Override
- void updateStatistics(Statistics statistics, OperationResponse response) {
- Slice tabletId = this.getTablet().getTabletId();
- String tableName = this.getTable().getName();
- TabletStatistics tabletStatistics = statistics.getTabletStatistics(tableName, tabletId);
- if (response == null) {
- tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1);
- tabletStatistics.incrementStatistic(Statistic.RPC_ERRORS, 1);
- return;
- }
- tabletStatistics.incrementStatistic(Statistic.WRITE_RPCS, 1);
- if (response.hasRowError()) {
- // If ignoreAllDuplicateRows is set, the already_present exception will be
- // discarded and wont't be recorded here
- tabletStatistics.incrementStatistic(Statistic.OPS_ERRORS, 1);
- } else {
- tabletStatistics.incrementStatistic(Statistic.WRITE_OPS, 1);
- }
- tabletStatistics.incrementStatistic(Statistic.BYTES_WRITTEN, getRowOperationSizeBytes());
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder(super.toString());
- sb.append(" row_key=");
- sb.append(row.stringifyRowKey());
- return sb.toString();
- }
-
- /**
- * Helper method that puts a list of Operations together into a WriteRequestPB.
- * @param operations The list of ops to put together in a WriteRequestPB
- * @return A fully constructed WriteRequestPB containing the passed rows, or
- * null if no rows were passed.
- */
- static Tserver.WriteRequestPB.Builder createAndFillWriteRequestPB(List<Operation> operations) {
- if (operations == null || operations.isEmpty()) return null;
- Schema schema = operations.get(0).table.getSchema();
- RowOperationsPB rowOps = new OperationsEncoder().encodeOperations(operations);
- if (rowOps == null) return null;
-
- Tserver.WriteRequestPB.Builder requestBuilder = Tserver.WriteRequestPB.newBuilder();
- requestBuilder.setSchema(ProtobufHelper.schemaToPb(schema));
- requestBuilder.setRowOperations(rowOps);
- return requestBuilder;
- }
-
- static class OperationsEncoder {
- private Schema schema;
- private ByteBuffer rows;
- // We're filling this list as we go through the operations in encodeRow() and at the same time
- // compute the total size, which will be used to right-size the array in toPB().
- private List<ByteBuffer> indirect;
- private long indirectWrittenBytes;
-
- /**
- * Initializes the state of the encoder based on the schema and number of operations to encode.
- *
- * @param schema the schema of the table which the operations belong to.
- * @param numOperations the number of operations.
- */
- private void init(Schema schema, int numOperations) {
- this.schema = schema;
-
- // Set up the encoded data.
- // Estimate a maximum size for the data. This is conservative, but avoids
- // having to loop through all the operations twice.
- final int columnBitSetSize = Bytes.getBitSetSize(schema.getColumnCount());
- int sizePerRow = 1 /* for the op type */ + schema.getRowSize() + columnBitSetSize;
- if (schema.hasNullableColumns()) {
- // nullsBitSet is the same size as the columnBitSet
- sizePerRow += columnBitSetSize;
- }
-
- // TODO: would be more efficient to use a buffer which "chains" smaller allocations
- // instead of a doubling buffer like BAOS.
- this.rows = ByteBuffer.allocate(sizePerRow * numOperations)
- .order(ByteOrder.LITTLE_ENDIAN);
- this.indirect = new ArrayList<>(schema.getVarLengthColumnCount() * numOperations);
- }
-
- /**
- * Builds the row operations protobuf message with encoded operations.
- * @return the row operations protobuf message.
- */
- private RowOperationsPB toPB() {
- RowOperationsPB.Builder rowOpsBuilder = RowOperationsPB.newBuilder();
-
- // TODO: we could implement a ZeroCopy approach here by subclassing LiteralByteString.
- // We have ZeroCopyLiteralByteString, but that only supports an entire array. Here
- // we've only partially filled in rows.array(), so we have to make the extra copy.
- rows.limit(rows.position());
- rows.flip();
- rowOpsBuilder.setRows(ByteString.copyFrom(rows));
- if (indirect.size() > 0) {
- // TODO: same as above, we could avoid a copy here by using an implementation that allows
- // zero-copy on a slice of an array.
- byte[] indirectData = new byte[(int)indirectWrittenBytes];
- int offset = 0;
- for (ByteBuffer bb : indirect) {
- int bbSize = bb.remaining();
- bb.get(indirectData, offset, bbSize);
- offset += bbSize;
- }
- rowOpsBuilder.setIndirectData(ZeroCopyLiteralByteString.wrap(indirectData));
- }
- return rowOpsBuilder.build();
- }
-
- private void encodeRow(PartialRow row, ChangeType type) {
- rows.put(type.toEncodedByte());
- rows.put(Bytes.fromBitSet(row.getColumnsBitSet(), schema.getColumnCount()));
- if (schema.hasNullableColumns()) {
- rows.put(Bytes.fromBitSet(row.getNullsBitSet(), schema.getColumnCount()));
- }
- int colIdx = 0;
- byte[] rowData = row.getRowAlloc();
- int currentRowOffset = 0;
- for (ColumnSchema col : row.getSchema().getColumns()) {
- // Keys should always be specified, maybe check?
- if (row.isSet(colIdx) && !row.isSetToNull(colIdx)) {
- if (col.getType() == Type.STRING || col.getType() == Type.BINARY) {
- ByteBuffer varLengthData = row.getVarLengthData().get(colIdx);
- varLengthData.reset();
- rows.putLong(indirectWrittenBytes);
- int bbSize = varLengthData.remaining();
- rows.putLong(bbSize);
- indirect.add(varLengthData);
- indirectWrittenBytes += bbSize;
- } else {
- // This is for cols other than strings
- rows.put(rowData, currentRowOffset, col.getType().getSize());
- }
- }
- currentRowOffset += col.getType().getSize();
- colIdx++;
- }
- }
-
- public RowOperationsPB encodeOperations(List<Operation> operations) {
- if (operations == null || operations.isEmpty()) return null;
- init(operations.get(0).table.getSchema(), operations.size());
- for (Operation operation : operations) {
- encodeRow(operation.row, operation.getChangeType());
- }
- return toPB();
- }
-
- public RowOperationsPB encodeSplitRowsRangeBounds(List<PartialRow> splitRows,
- List<Pair<PartialRow, PartialRow>> rangeBounds) {
- if (splitRows.isEmpty() && rangeBounds.isEmpty()) {
- return null;
- }
-
- Schema schema = splitRows.isEmpty() ? rangeBounds.get(0).getFirst().getSchema()
- : splitRows.get(0).getSchema();
- init(schema, splitRows.size() + 2 * rangeBounds.size());
-
- for (PartialRow row : splitRows) {
- encodeRow(row, ChangeType.SPLIT_ROWS);
- }
-
- for (Pair<PartialRow, PartialRow> bound : rangeBounds) {
- encodeRow(bound.getFirst(), ChangeType.RANGE_LOWER_BOUND);
- encodeRow(bound.getSecond(), ChangeType.RANGE_UPPER_BOUND);
- }
-
- return toPB();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/OperationResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/OperationResponse.java b/java/kudu-client/src/main/java/org/kududb/client/OperationResponse.java
deleted file mode 100644
index bf707ce..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/OperationResponse.java
+++ /dev/null
@@ -1,111 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class OperationResponse extends KuduRpcResponse {
-
- private final long writeTimestamp;
- private final RowError rowError;
- private final Operation operation;
-
- /**
- * Package-private constructor to build an OperationResponse with a row error in the pb format.
- * @param elapsedMillis time in milliseconds since RPC creation to now
- * @param writeTimestamp HT's write timestamp
- * @param operation the operation that created this response
- * @param errorPB a row error in pb format, can be null
- */
- OperationResponse(long elapsedMillis, String tsUUID, long writeTimestamp,
- Operation operation, Tserver.WriteResponsePB.PerRowErrorPB errorPB) {
- super(elapsedMillis, tsUUID);
- this.writeTimestamp = writeTimestamp;
- this.rowError = errorPB == null ? null : RowError.fromRowErrorPb(errorPB, operation, tsUUID);
- this.operation = operation;
- }
-
- /**
- * Package-private constructor to build an OperationResponse with a row error.
- * @param elapsedMillis time in milliseconds since RPC creation to now
- * @param writeTimestamp HT's write timestamp
- * @param operation the operation that created this response
- * @param rowError a parsed row error, can be null
- */
- OperationResponse(long elapsedMillis, String tsUUID, long writeTimestamp,
- Operation operation, RowError rowError) {
- super(elapsedMillis, tsUUID);
- this.writeTimestamp = writeTimestamp;
- this.rowError = rowError;
- this.operation = operation;
- }
-
- /**
- * Utility method that collects all the row errors from the given list of responses.
- * @param responses a list of operation responses to collect the row errors from
- * @return a combined list of row errors
- */
- public static List<RowError> collectErrors(List<OperationResponse> responses) {
- List<RowError> errors = new ArrayList<>(responses.size());
- for (OperationResponse resp : responses) {
- if (resp.hasRowError()) {
- errors.add(resp.getRowError());
- }
- }
- return errors;
- }
-
- /**
- * Gives the write timestamp that was returned by the Tablet Server.
- * @return a timestamp in milliseconds, 0 if the external consistency mode set
- * in AsyncKuduSession wasn't CLIENT_PROPAGATED, or if the operation failed.
- */
- public long getWriteTimestamp() {
- return writeTimestamp;
- }
-
- /**
- * Returns a row error. If {@link #hasRowError()} returns false, then this method returns null.
- * @return a row error, or null if the operation was successful
- */
- public RowError getRowError() {
- return rowError;
- }
-
- /**
- * Tells if this operation response contains a row error.
- * @return true if this operation response has errors, else false
- */
- public boolean hasRowError() {
- return rowError != null;
- }
-
- /**
- * Returns the operation associated with this response.
- * @return an operation, cannot be null
- */
- Operation getOperation() {
- return operation;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/PartialRow.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/PartialRow.java b/java/kudu-client/src/main/java/org/kududb/client/PartialRow.java
deleted file mode 100644
index b5f3069..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/PartialRow.java
+++ /dev/null
@@ -1,626 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.List;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Class used to represent parts of a row along with its schema.<p>
- *
- * Values can be replaced as often as needed, but once the enclosing {@link Operation} is applied
- * then they cannot be changed again. This means that a PartialRow cannot be reused.<p>
- *
- * Each PartialRow is backed by an byte array where all the cells (except strings and binary data)
- * are written. The others are kept in a List.<p>
- *
- * This class isn't thread-safe.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class PartialRow {
-
- private final Schema schema;
-
- // Variable length data. If string, will be UTF-8 encoded. Elements of this list _must_ have a
- // mark that we can reset() to. Readers of these fields (encoders, etc) must call reset() before
- // attempting to read these values.
- private final List<ByteBuffer> varLengthData;
- private final byte[] rowAlloc;
-
- private final BitSet columnsBitSet;
- private final BitSet nullsBitSet;
-
- private boolean frozen = false;
-
- /**
- * This is not a stable API, prefer using {@link Schema#newPartialRow()}
- * to create a new partial row.
- * @param schema the schema to use for this row
- */
- public PartialRow(Schema schema) {
- this.schema = schema;
- this.columnsBitSet = new BitSet(this.schema.getColumnCount());
- this.nullsBitSet = schema.hasNullableColumns() ?
- new BitSet(this.schema.getColumnCount()) : null;
- this.rowAlloc = new byte[schema.getRowSize()];
- // Pre-fill the array with nulls. We'll only replace cells that have varlen values.
- this.varLengthData = Arrays.asList(new ByteBuffer[this.schema.getColumnCount()]);
- }
-
- /**
- * Creates a new partial row by deep-copying the data-fields of the provided partial row.
- * @param row the partial row to copy
- */
- PartialRow(PartialRow row) {
- this.schema = row.schema;
-
- this.varLengthData = Lists.newArrayListWithCapacity(row.varLengthData.size());
- for (ByteBuffer data: row.varLengthData) {
- if (data == null) {
- this.varLengthData.add(null);
- } else {
- data.reset();
- // Deep copy the ByteBuffer.
- ByteBuffer clone = ByteBuffer.allocate(data.remaining());
- clone.put(data);
- clone.flip();
-
- clone.mark(); // We always expect a mark.
- this.varLengthData.add(clone);
- }
- }
-
- this.rowAlloc = row.rowAlloc.clone();
- this.columnsBitSet = (BitSet) row.columnsBitSet.clone();
- this.nullsBitSet = row.nullsBitSet == null ? null : (BitSet) row.nullsBitSet.clone();
- }
-
- /**
- * Add a boolean for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addBoolean(int columnIndex, boolean val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.BOOL);
- rowAlloc[getPositionInRowAllocAndSetBitSet(columnIndex)] = (byte) (val ? 1 : 0);
- }
-
- /**
- * Add a boolean for the specified column.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addBoolean(String columnName, boolean val) {
- addBoolean(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add a byte for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addByte(int columnIndex, byte val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.INT8);
- rowAlloc[getPositionInRowAllocAndSetBitSet(columnIndex)] = val;
- }
-
- /**
- * Add a byte for the specified column.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addByte(String columnName, byte val) {
- addByte(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add a short for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addShort(int columnIndex, short val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.INT16);
- Bytes.setShort(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
- }
-
- /**
- * Add a short for the specified column.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addShort(String columnName, short val) {
- addShort(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add an int for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addInt(int columnIndex, int val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.INT32);
- Bytes.setInt(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
- }
-
- /**
- * Add an int for the specified column.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addInt(String columnName, int val) {
- addInt(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add an long for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addLong(int columnIndex, long val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.INT64, Type.TIMESTAMP);
- Bytes.setLong(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
- }
-
- /**
- * Add an long for the specified column.
- *
- * If this is a TIMESTAMP column, the long value provided should be the number of microseconds
- * between a given time and January 1, 1970 UTC.
- * For example, to encode the current time, use setLong(System.currentTimeMillis() * 1000);
- *
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addLong(String columnName, long val) {
- addLong(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add an float for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addFloat(int columnIndex, float val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.FLOAT);
- Bytes.setFloat(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
- }
-
- /**
- * Add an float for the specified column.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addFloat(String columnName, float val) {
- addFloat(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add an double for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addDouble(int columnIndex, double val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.DOUBLE);
- Bytes.setDouble(rowAlloc, val, getPositionInRowAllocAndSetBitSet(columnIndex));
- }
-
- /**
- * Add an double for the specified column.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addDouble(String columnName, double val) {
- addDouble(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add a String for the specified column.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addString(int columnIndex, String val) {
- addStringUtf8(columnIndex, Bytes.fromString(val));
- }
-
- /**
- * Add a String for the specified column.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addString(String columnName, String val) {
- addStringUtf8(columnName, Bytes.fromString(val));
- }
-
- /**
- * Add a String for the specified value, encoded as UTF8.
- * Note that the provided value must not be mutated after this.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addStringUtf8(int columnIndex, byte[] val) {
- // TODO: use Utf8.isWellFormed from Guava 16 to verify that
- // the user isn't putting in any garbage data.
- checkColumn(schema.getColumnByIndex(columnIndex), Type.STRING);
- addVarLengthData(columnIndex, val);
- }
-
- /**
- * Add a String for the specified value, encoded as UTF8.
- * Note that the provided value must not be mutated after this.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- *
- */
- public void addStringUtf8(String columnName, byte[] val) {
- addStringUtf8(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add binary data with the specified value.
- * Note that the provided value must not be mutated after this.
- * @param columnIndex the column's index in the schema
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addBinary(int columnIndex, byte[] val) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.BINARY);
- addVarLengthData(columnIndex, val);
- }
-
- /**
- * Add binary data with the specified value, from the current ByteBuffer's position to its limit.
- * This method duplicates the ByteBuffer but doesn't copy the data. This means that the wrapped
- * data must not be mutated after this.
- * @param columnIndex the column's index in the schema
- * @param value byte buffer to get the value from
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addBinary(int columnIndex, ByteBuffer value) {
- checkColumn(schema.getColumnByIndex(columnIndex), Type.BINARY);
- addVarLengthData(columnIndex, value);
- }
-
- /**
- * Add binary data with the specified value.
- * Note that the provided value must not be mutated after this.
- * @param columnName Name of the column
- * @param val value to add
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addBinary(String columnName, byte[] val) {
- addBinary(schema.getColumnIndex(columnName), val);
- }
-
- /**
- * Add binary data with the specified value, from the current ByteBuffer's position to its limit.
- * This method duplicates the ByteBuffer but doesn't copy the data. This means that the wrapped
- * data must not be mutated after this.
- * @param columnName Name of the column
- * @param value byte buffer to get the value from
- * @throws IllegalArgumentException if the column doesn't exist or if the value doesn't match
- * the column's type
- * @throws IllegalStateException if the row was already applied
- */
- public void addBinary(String columnName, ByteBuffer value) {
- addBinary(schema.getColumnIndex(columnName), value);
- }
-
- private void addVarLengthData(int columnIndex, byte[] val) {
- addVarLengthData(columnIndex, ByteBuffer.wrap(val));
- }
-
- private void addVarLengthData(int columnIndex, ByteBuffer val) {
- // A duplicate will copy all the original's metadata but still point to the same content.
- ByteBuffer duplicate = val.duplicate();
- // Mark the current position so we can reset to it.
- duplicate.mark();
-
- varLengthData.set(columnIndex, duplicate);
- // Set the usage bit but we don't care where it is.
- getPositionInRowAllocAndSetBitSet(columnIndex);
- // We don't set anything in row alloc, it will be managed at encoding time.
- }
-
- /**
- * Set the specified column to null
- * @param columnIndex the column's index in the schema
- * @throws IllegalArgumentException if the column doesn't exist or cannot be set to null
- * @throws IllegalStateException if the row was already applied
- */
- public void setNull(int columnIndex) {
- setNull(this.schema.getColumnByIndex(columnIndex));
- }
-
- /**
- * Set the specified column to null
- * @param columnName Name of the column
- * @throws IllegalArgumentException if the column doesn't exist or cannot be set to null
- * @throws IllegalStateException if the row was already applied
- */
- public void setNull(String columnName) {
- setNull(this.schema.getColumn(columnName));
- }
-
- private void setNull(ColumnSchema column) {
- assert nullsBitSet != null;
- checkNotFrozen();
- checkColumnExists(column);
- if (!column.isNullable()) {
- throw new IllegalArgumentException(column.getName() + " cannot be set to null");
- }
- int idx = schema.getColumns().indexOf(column);
- columnsBitSet.set(idx);
- nullsBitSet.set(idx);
- }
-
- /**
- * Verifies if the column exists and belongs to one of the specified types
- * It also does some internal accounting
- * @param column column the user wants to set
- * @param types types we expect
- * @throws IllegalArgumentException if the column or type was invalid
- * @throws IllegalStateException if the row was already applied
- */
- private void checkColumn(ColumnSchema column, Type... types) {
- checkNotFrozen();
- checkColumnExists(column);
- for(Type type : types) {
- if (column.getType().equals(type)) return;
- }
- throw new IllegalArgumentException(String.format("%s isn't %s, it's %s", column.getName(),
- Arrays.toString(types), column.getType().getName()));
- }
-
- /**
- * @param column column the user wants to set
- * @throws IllegalArgumentException if the column doesn't exist
- */
- private void checkColumnExists(ColumnSchema column) {
- if (column == null)
- throw new IllegalArgumentException("Column name isn't present in the table's schema");
- }
-
- /**
- * @throws IllegalStateException if the row was already applied
- */
- private void checkNotFrozen() {
- if (frozen) {
- throw new IllegalStateException("This row was already applied and cannot be modified.");
- }
- }
-
- /**
- * Sets the column bit set for the column index, and returns the column's offset.
- * @param columnIndex the index of the column to get the position for and mark as set
- * @return the offset in rowAlloc for the column
- */
- private int getPositionInRowAllocAndSetBitSet(int columnIndex) {
- columnsBitSet.set(columnIndex);
- return schema.getColumnOffset(columnIndex);
- }
-
- /**
- * Tells if the specified column was set by the user
- * @param column column's index in the schema
- * @return true if it was set, else false
- */
- boolean isSet(int column) {
- return this.columnsBitSet.get(column);
- }
-
- /**
- * Tells if the specified column was set to null by the user
- * @param column column's index in the schema
- * @return true if it was set, else false
- */
- boolean isSetToNull(int column) {
- if (this.nullsBitSet == null) {
- return false;
- }
- return this.nullsBitSet.get(column);
- }
-
- /**
- * Returns the encoded primary key of the row.
- * @return a byte array containing an encoded primary key
- */
- public byte[] encodePrimaryKey() {
- return new KeyEncoder().encodePrimaryKey(this);
- }
-
- /**
- * Transforms the row key into a string representation where each column is in the format:
- * "type col_name=value".
- * @return a string representation of the operation's row key
- */
- public String stringifyRowKey() {
- int numRowKeys = schema.getPrimaryKeyColumnCount();
- StringBuilder sb = new StringBuilder();
- sb.append("(");
- for (int i = 0; i < numRowKeys; i++) {
- if (i > 0) {
- sb.append(", ");
- }
-
- ColumnSchema col = schema.getColumnByIndex(i);
- assert !col.isNullable();
- Preconditions.checkState(columnsBitSet.get(i),
- "Full row key not specified, missing at least col: " + col.getName());
- Type type = col.getType();
- sb.append(type.getName());
- sb.append(" ");
- sb.append(col.getName());
- sb.append("=");
-
- if (type == Type.STRING || type == Type.BINARY) {
- ByteBuffer value = getVarLengthData().get(i).duplicate();
- value.reset(); // Make sure we start at the beginning.
- byte[] data = new byte[value.limit()];
- value.get(data);
- if (type == Type.STRING) {
- sb.append(Bytes.getString(data));
- } else {
- sb.append(Bytes.pretty(data));
- }
- } else {
- switch (type) {
- case INT8:
- sb.append(Bytes.getByte(rowAlloc, schema.getColumnOffset(i)));
- break;
- case INT16:
- sb.append(Bytes.getShort(rowAlloc, schema.getColumnOffset(i)));
- break;
- case INT32:
- sb.append(Bytes.getInt(rowAlloc, schema.getColumnOffset(i)));
- break;
- case INT64:
- sb.append(Bytes.getLong(rowAlloc, schema.getColumnOffset(i)));
- break;
- case TIMESTAMP:
- sb.append(Bytes.getLong(rowAlloc, schema.getColumnOffset(i)));
- break;
- default:
- throw new IllegalArgumentException(String.format(
- "The column type %s is not a valid key component type", type));
- }
- }
- }
- sb.append(")");
-
- return sb.toString();
- }
-
- /**
- * Get the schema used for this row.
- * @return a schema that came from KuduTable
- */
- Schema getSchema() {
- return schema;
- }
-
- /**
- * Get the list variable length data cells that were added to this row.
- * @return a list of binary data, may be empty
- */
- List<ByteBuffer> getVarLengthData() {
- return varLengthData;
- }
-
- /**
- * Get the byte array that contains all the data added to this partial row. Variable length data
- * is contained separately, see {@link #getVarLengthData()}. In their place you'll find their
- * index in that list and their size.
- * @return a byte array containing the data for this row, except strings
- */
- byte[] getRowAlloc() {
- return rowAlloc;
- }
-
- /**
- * Get the bit set that indicates which columns were set.
- * @return a bit set for columns with data
- */
- BitSet getColumnsBitSet() {
- return columnsBitSet;
- }
-
- /**
- * Get the bit set for the columns that were specifically set to null
- * @return a bit set for null columns
- */
- BitSet getNullsBitSet() {
- return nullsBitSet;
- }
-
- /**
- * Prevents this PartialRow from being modified again. Can be called multiple times.
- */
- void freeze() {
- this.frozen = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/Partition.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/Partition.java b/java/kudu-client/src/main/java/org/kududb/client/Partition.java
deleted file mode 100644
index bdc089b..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/Partition.java
+++ /dev/null
@@ -1,182 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Objects;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A Partition describes the set of rows that a Tablet is responsible for
- * serving. Each tablet is assigned a single Partition.<p>
- *
- * Partitions consist primarily of a start and end partition key. Every row with
- * a partition key that falls in a Tablet's Partition will be served by that
- * tablet.<p>
- *
- * In addition to the start and end partition keys, a Partition holds metadata
- * to determine if a scan can prune, or skip, a partition based on the scan's
- * start and end primary keys, and predicates.
- *
- * This class is new, and not considered stable or suitable for public use.
- */
-@InterfaceAudience.LimitedPrivate("Impala")
-@InterfaceStability.Unstable
-public class Partition implements Comparable<Partition> {
- final byte[] partitionKeyStart;
- final byte[] partitionKeyEnd;
-
- final byte[] rangeKeyStart;
- final byte[] rangeKeyEnd;
-
- final List<Integer> hashBuckets;
-
- /**
- * Size of an encoded hash bucket component in a partition key.
- */
- private static final int ENCODED_BUCKET_SIZE = 4;
-
- /**
- * Creates a new partition with the provided start and end keys, and hash buckets.
- * @param partitionKeyStart the start partition key
- * @param partitionKeyEnd the end partition key
- * @param hashBuckets the partition hash buckets
- */
- Partition(byte[] partitionKeyStart,
- byte[] partitionKeyEnd,
- List<Integer> hashBuckets) {
- this.partitionKeyStart = partitionKeyStart;
- this.partitionKeyEnd = partitionKeyEnd;
- this.hashBuckets = hashBuckets;
- this.rangeKeyStart = rangeKey(partitionKeyStart, hashBuckets.size());
- this.rangeKeyEnd = rangeKey(partitionKeyEnd, hashBuckets.size());
- }
-
- /**
- * Gets the start partition key.
- * @return the start partition key
- */
- public byte[] getPartitionKeyStart() {
- return partitionKeyStart;
- }
-
- /**
- * Gets the end partition key.
- * @return the end partition key
- */
- public byte[] getPartitionKeyEnd() {
- return partitionKeyEnd;
- }
-
- /**
- * Gets the start range key.
- * @return the start range key
- */
- public byte[] getRangeKeyStart() {
- return rangeKeyStart;
- }
-
- /**
- * Gets the end range key.
- * @return the end range key
- */
- public byte[] getRangeKeyEnd() {
- return rangeKeyEnd;
- }
-
- /**
- * Gets the partition hash buckets.
- * @return the partition hash buckets
- */
- public List<Integer> getHashBuckets() {
- return hashBuckets;
- }
-
- /**
- * @return true if the partition is the absolute end partition
- */
- public boolean isEndPartition() {
- return partitionKeyEnd.length == 0;
- }
-
- /**
- * Equality only holds for partitions from the same table. Partition equality only takes into
- * account the partition keys, since there is a 1 to 1 correspondence between partition keys and
- * the hash buckets and range keys.
- *
- * @return the hash code
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- Partition partition = (Partition) o;
- return Arrays.equals(partitionKeyStart, partition.partitionKeyStart)
- && Arrays.equals(partitionKeyEnd, partition.partitionKeyEnd);
- }
-
- /**
- * The hash code only takes into account the partition keys, since there is a 1 to 1
- * correspondence between partition keys and the hash buckets and range keys.
- *
- * @return the hash code
- */
- @Override
- public int hashCode() {
- return Objects.hashCode(Arrays.hashCode(partitionKeyStart), Arrays.hashCode(partitionKeyEnd));
- }
-
- /**
- * Partition comparison is only reasonable when comparing partitions from the same table, and
- * since Kudu does not yet allow partition splitting, no two distinct partitions can have the
- * same start partition key. Accordingly, partitions are compared strictly by the start partition
- * key.
- *
- * @param other the other partition of the same table
- * @return the comparison of the partitions
- */
- @Override
- public int compareTo(Partition other) {
- return Bytes.memcmp(this.partitionKeyStart, other.partitionKeyStart);
- }
-
- /**
- * Returns the range key portion of a partition key given the number of buckets in the partition
- * schema.
- * @param partitionKey the partition key containing the range key
- * @param numHashBuckets the number of hash bucket components of the table
- * @return the range key
- */
- private static byte[] rangeKey(byte[] partitionKey, int numHashBuckets) {
- int bucketsLen = numHashBuckets * ENCODED_BUCKET_SIZE;
- if (partitionKey.length > bucketsLen) {
- return Arrays.copyOfRange(partitionKey, bucketsLen, partitionKey.length);
- } else {
- return AsyncKuduClient.EMPTY_ARRAY;
- }
- }
-
- @Override
- public String toString() {
- return String.format("[%s, %s)",
- partitionKeyStart.length == 0 ? "<start>" : Bytes.hex(partitionKeyStart),
- partitionKeyEnd.length == 0 ? "<end>" : Bytes.hex(partitionKeyEnd));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/PartitionSchema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/PartitionSchema.java b/java/kudu-client/src/main/java/org/kududb/client/PartitionSchema.java
deleted file mode 100644
index fdee32e..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/PartitionSchema.java
+++ /dev/null
@@ -1,142 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.Schema;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import java.util.List;
-
-/**
- * A partition schema describes how the rows of a table are distributed among
- * tablets.
- *
- * Primarily, a table's partition schema is responsible for translating the
- * primary key column values of a row into a partition key that can be used to
- * find the tablet containing the key.
- *
- * The partition schema is made up of zero or more hash bucket components,
- * followed by a single range component.
- *
- * Each hash bucket component includes one or more columns from the primary key
- * column set, with the restriction that an individual primary key column may
- * only be included in a single hash component.
- *
- * This class is new, and not considered stable or suitable for public use.
- */
-@InterfaceAudience.LimitedPrivate("Impala")
-@InterfaceStability.Unstable
-public class PartitionSchema {
-
- private final RangeSchema rangeSchema;
- private final List<HashBucketSchema> hashBucketSchemas;
- private final boolean isSimple;
-
- /**
- * Creates a new partition schema from the range and hash bucket schemas.
- *
- * @param rangeSchema the range schema
- * @param hashBucketSchemas the hash bucket schemas
- * @param schema the table schema
- */
- PartitionSchema(RangeSchema rangeSchema,
- List<HashBucketSchema> hashBucketSchemas,
- Schema schema) {
- this.rangeSchema = rangeSchema;
- this.hashBucketSchemas = hashBucketSchemas;
-
- boolean isSimple = hashBucketSchemas.isEmpty()
- && rangeSchema.columns.size() == schema.getPrimaryKeyColumnCount();
- if (isSimple) {
- int i = 0;
- for (Integer id : rangeSchema.columns) {
- if (schema.getColumnIndex(id) != i++) {
- isSimple = false;
- break;
- }
- }
- }
- this.isSimple = isSimple;
- }
-
- /**
- * Returns the encoded partition key of the row.
- * @return a byte array containing the encoded partition key of the row
- */
- public byte[] encodePartitionKey(PartialRow row) {
- return new KeyEncoder().encodePartitionKey(row, this);
- }
-
- public RangeSchema getRangeSchema() {
- return rangeSchema;
- }
-
- public List<HashBucketSchema> getHashBucketSchemas() {
- return hashBucketSchemas;
- }
-
- /**
- * Returns true if the partition schema if the partition schema does not include any hash
- * components, and the range columns match the table's primary key columns.
- *
- * @return whether the partition schema is the default simple range partitioning.
- */
- boolean isSimpleRangePartitioning() {
- return isSimple;
- }
-
- public static class RangeSchema {
- private final List<Integer> columns;
-
- RangeSchema(List<Integer> columns) {
- this.columns = columns;
- }
-
- public List<Integer> getColumns() {
- return columns;
- }
- }
-
- public static class HashBucketSchema {
- private final List<Integer> columnIds;
- private int numBuckets;
- private int seed;
-
- HashBucketSchema(List<Integer> columnIds, int numBuckets, int seed) {
- this.columnIds = columnIds;
- this.numBuckets = numBuckets;
- this.seed = seed;
- }
-
- /**
- * Gets the column IDs of the columns in the hash partition.
- * @return the column IDs of the columns in the has partition
- */
- public List<Integer> getColumnIds() {
- return columnIds;
- }
-
- public int getNumBuckets() {
- return numBuckets;
- }
-
- public int getSeed() {
- return seed;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/PleaseThrottleException.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/PleaseThrottleException.java b/java/kudu-client/src/main/java/org/kududb/client/PleaseThrottleException.java
deleted file mode 100644
index 3ca98e2..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/PleaseThrottleException.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright (C) 2010-2012 The Async HBase Authors. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * - Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * - Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * - Neither the name of the StumbleUpon nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-package org.kududb.client;
-
-import com.stumbleupon.async.Deferred;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * This exception notifies the application to throttle its use of Kudu.
- * <p>
- * Since all APIs of {@link AsyncKuduSession} are asynchronous and non-blocking,
- * it's possible that the application would produce RPCs at a rate higher
- * than Kudu is able to handle. When this happens, {@link AsyncKuduSession}
- * will typically do some buffering up to a certain point beyond which RPCs
- * will fail-fast with this exception, to prevent the application from
- * running itself out of memory.
- * <p>
- * This exception is expected to be handled by having the application
- * throttle or pause itself for a short period of time before retrying the
- * RPC that failed with this exception as well as before sending other RPCs.
- * The reason this exception inherits from {@link NonRecoverableException}
- * instead of {@link RecoverableException} is that the usual course of action
- * when handling a {@link RecoverableException} is to retry right away, which
- * would defeat the whole purpose of this exception. Here, we want the
- * application to <b>retry after a reasonable delay</b> as well as <b>throttle
- * the pace of creation of new RPCs</b>. What constitutes a "reasonable
- * delay" depends on the nature of RPCs and rate at which they're produced.
- * <p>
- * One effective strategy to handle this exception is to set a flag to true
- * when this exception is first emitted that causes the application to pause
- * or throttle its use of Kudu. Then you can retry the RPC that failed
- * (which is accessible through {@link #getFailedRpc}) and add a callback to
- * it in order to unset the flag once the RPC completes successfully.
- * Note that low-throughput applications will typically rarely (if ever)
- * hit this exception, so they don't need complex throttling logic.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-@SuppressWarnings("serial")
-public final class PleaseThrottleException extends RecoverableException
- implements HasFailedRpcException {
-
- /** The RPC that was failed with this exception. */
- private final Operation rpc;
-
- /** A deferred one can wait on before retrying the failed RPC. */
- private final Deferred deferred;
-
- /**
- * Constructor.
- * @param status status object containing the reason for the exception
- * @param cause The exception that requires the application to throttle
- * itself (can be {@code null})
- * @param rpc The RPC that was made to fail with this exception
- * @param deferred A deferred one can wait on before retrying the failed RPC
- */
- PleaseThrottleException(Status status,
- KuduException cause,
- Operation rpc,
- Deferred deferred) {
- super(status, cause);
- this.rpc = rpc;
- this.deferred = deferred;
- }
-
- /**
- * The RPC that was made to fail with this exception.
- */
- public Operation getFailedRpc() {
- return rpc;
- }
-
- /**
- * Returns a deferred one can wait on before retrying the failed RPC.
- * @since 1.3
- */
- public Deferred getDeferred() {
- return deferred;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/ProtobufHelper.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/ProtobufHelper.java b/java/kudu-client/src/main/java/org/kududb/client/ProtobufHelper.java
deleted file mode 100644
index 2b2cf64..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/ProtobufHelper.java
+++ /dev/null
@@ -1,253 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.ImmutableList;
-import com.google.common.net.HostAndPort;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ZeroCopyLiteralByteString;
-import org.kududb.ColumnSchema;
-import org.kududb.Common;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-
-@InterfaceAudience.Private
-public class ProtobufHelper {
-
- /**
- * Utility method to convert a Schema to its wire format.
- * @param schema Schema to convert
- * @return a list of ColumnSchemaPB
- */
- public static List<Common.ColumnSchemaPB> schemaToListPb(Schema schema) {
- ArrayList<Common.ColumnSchemaPB> columns =
- new ArrayList<Common.ColumnSchemaPB>(schema.getColumnCount());
- Common.ColumnSchemaPB.Builder schemaBuilder = Common.ColumnSchemaPB.newBuilder();
- for (ColumnSchema col : schema.getColumns()) {
- columns.add(columnToPb(schemaBuilder, col));
- schemaBuilder.clear();
- }
- return columns;
- }
-
- public static Common.SchemaPB schemaToPb(Schema schema) {
- Common.SchemaPB.Builder builder = Common.SchemaPB.newBuilder();
- builder.addAllColumns(schemaToListPb(schema));
- return builder.build();
- }
-
- public static Common.ColumnSchemaPB columnToPb(ColumnSchema column) {
- return columnToPb(Common.ColumnSchemaPB.newBuilder(), column);
- }
-
- public static Common.ColumnSchemaPB
- columnToPb(Common.ColumnSchemaPB.Builder schemaBuilder, ColumnSchema column) {
- schemaBuilder
- .setName(column.getName())
- .setType(column.getType().getDataType())
- .setIsKey(column.isKey())
- .setIsNullable(column.isNullable())
- .setCfileBlockSize(column.getDesiredBlockSize());
- if (column.getEncoding() != null) {
- schemaBuilder.setEncoding(column.getEncoding().getInternalPbType());
- }
- if (column.getCompressionAlgorithm() != null) {
- schemaBuilder.setCompression(column.getCompressionAlgorithm().getInternalPbType());
- }
- if (column.getDefaultValue() != null) schemaBuilder.setReadDefaultValue
- (ZeroCopyLiteralByteString.wrap(objectToWireFormat(column, column.getDefaultValue())));
- return schemaBuilder.build();
- }
-
- public static ColumnSchema pbToColumnSchema(Common.ColumnSchemaPB pb) {
- Type type = Type.getTypeForDataType(pb.getType());
- Object defaultValue = pb.hasReadDefaultValue() ?
- byteStringToObject(type, pb.getReadDefaultValue()) : null;
- ColumnSchema.Encoding encoding = ColumnSchema.Encoding.valueOf(pb.getEncoding().name());
- ColumnSchema.CompressionAlgorithm compressionAlgorithm =
- ColumnSchema.CompressionAlgorithm.valueOf(pb.getCompression().name());
- return new ColumnSchema.ColumnSchemaBuilder(pb.getName(), type)
- .key(pb.getIsKey())
- .nullable(pb.getIsNullable())
- .defaultValue(defaultValue)
- .encoding(encoding)
- .compressionAlgorithm(compressionAlgorithm)
- .build();
- }
-
- public static Schema pbToSchema(Common.SchemaPB schema) {
- List<ColumnSchema> columns = new ArrayList<>(schema.getColumnsCount());
- List<Integer> columnIds = new ArrayList<>(schema.getColumnsCount());
- for (Common.ColumnSchemaPB columnPb : schema.getColumnsList()) {
- columns.add(pbToColumnSchema(columnPb));
- int id = columnPb.getId();
- if (id < 0) {
- throw new IllegalArgumentException("Illegal column ID: " + id);
- }
- columnIds.add(id);
- }
- return new Schema(columns, columnIds);
- }
-
- /**
- * Factory method for creating a {@code PartitionSchema} from a protobuf message.
- *
- * @param pb the partition schema protobuf message
- * @return a partition instance
- */
- static PartitionSchema pbToPartitionSchema(Common.PartitionSchemaPB pb, Schema schema) {
- List<Integer> rangeColumns = pbToIds(pb.getRangeSchema().getColumnsList());
- PartitionSchema.RangeSchema rangeSchema = new PartitionSchema.RangeSchema(rangeColumns);
-
- ImmutableList.Builder<PartitionSchema.HashBucketSchema> hashSchemas = ImmutableList.builder();
-
- for (Common.PartitionSchemaPB.HashBucketSchemaPB hashBucketSchemaPB
- : pb.getHashBucketSchemasList()) {
- List<Integer> hashColumnIds = pbToIds(hashBucketSchemaPB.getColumnsList());
-
- PartitionSchema.HashBucketSchema hashSchema =
- new PartitionSchema.HashBucketSchema(hashColumnIds,
- hashBucketSchemaPB.getNumBuckets(),
- hashBucketSchemaPB.getSeed());
-
- hashSchemas.add(hashSchema);
- }
-
- return new PartitionSchema(rangeSchema, hashSchemas.build(), schema);
- }
-
- /**
- * Constructs a new {@code Partition} instance from the a protobuf message.
- * @param pb the protobuf message
- * @return the {@code Partition} corresponding to the message
- */
- static Partition pbToPartition(Common.PartitionPB pb) {
- return new Partition(pb.getPartitionKeyStart().toByteArray(),
- pb.getPartitionKeyEnd().toByteArray(),
- pb.getHashBucketsList());
- }
-
- /**
- * Deserializes a list of column identifier protobufs into a list of column IDs. This method
- * relies on the fact that the master will aways send a partition schema with column IDs, and not
- * column names (column names are only used when the client is sending the partition schema to
- * the master as part of the create table process).
- *
- * @param columnIdentifiers the column identifiers
- * @return the column IDs
- */
- private static List<Integer> pbToIds(
- List<Common.PartitionSchemaPB.ColumnIdentifierPB> columnIdentifiers) {
- ImmutableList.Builder<Integer> columnIds = ImmutableList.builder();
- for (Common.PartitionSchemaPB.ColumnIdentifierPB column : columnIdentifiers) {
- switch (column.getIdentifierCase()) {
- case ID:
- columnIds.add(column.getId());
- break;
- case NAME:
- throw new IllegalArgumentException(
- String.format("Expected column ID from master: %s", column));
- case IDENTIFIER_NOT_SET:
- throw new IllegalArgumentException("Unknown column: " + column);
- }
- }
- return columnIds.build();
- }
-
- private static byte[] objectToWireFormat(ColumnSchema col, Object value) {
- switch (col.getType()) {
- case BOOL:
- return Bytes.fromBoolean((Boolean) value);
- case INT8:
- return new byte[] {(Byte) value};
- case INT16:
- return Bytes.fromShort((Short) value);
- case INT32:
- return Bytes.fromInt((Integer) value);
- case INT64:
- case TIMESTAMP:
- return Bytes.fromLong((Long) value);
- case STRING:
- return ((String) value).getBytes(Charsets.UTF_8);
- case BINARY:
- return (byte[]) value;
- case FLOAT:
- return Bytes.fromFloat((Float) value);
- case DOUBLE:
- return Bytes.fromDouble((Double) value);
- default:
- throw new IllegalArgumentException("The column " + col.getName() + " is of type " + col
- .getType() + " which is unknown");
- }
- }
-
- private static Object byteStringToObject(Type type, ByteString value) {
- byte[] buf = ZeroCopyLiteralByteString.zeroCopyGetBytes(value);
- switch (type) {
- case BOOL:
- return Bytes.getBoolean(buf);
- case INT8:
- return Bytes.getByte(buf);
- case INT16:
- return Bytes.getShort(buf);
- case INT32:
- return Bytes.getInt(buf);
- case INT64:
- case TIMESTAMP:
- return Bytes.getLong(buf);
- case FLOAT:
- return Bytes.getFloat(buf);
- case DOUBLE:
- return Bytes.getDouble(buf);
- case STRING:
- return new String(buf, Charsets.UTF_8);
- case BINARY:
- return buf;
- default:
- throw new IllegalArgumentException("This type is unknown: " + type);
- }
- }
-
- /**
- * Convert a {@link com.google.common.net.HostAndPort} to {@link org.kududb.Common.HostPortPB}
- * protobuf message for serialization.
- * @param hostAndPort The host and port object. Both host and port must be specified.
- * @return An initialized HostPortPB object.
- */
- public static Common.HostPortPB hostAndPortToPB(HostAndPort hostAndPort) {
- return Common.HostPortPB.newBuilder()
- .setHost(hostAndPort.getHostText())
- .setPort(hostAndPort.getPort())
- .build();
- }
-
- /**
- * Convert a {@link org.kududb.Common.HostPortPB} to {@link com.google.common.net.HostAndPort}.
- * @param hostPortPB The fully initialized HostPortPB object. Must have both host and port
- * specified.
- * @return An initialized initialized HostAndPort object.
- */
- public static HostAndPort hostAndPortFromPB(Common.HostPortPB hostPortPB) {
- return HostAndPort.fromParts(hostPortPB.getHost(), hostPortPB.getPort());
- }
-}
[21/36] incubator-kudu git commit: [java-client] repackage to
org.apache.kudu (Part 1)
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java b/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java
new file mode 100644
index 0000000..c9d2719
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/Slice.java
@@ -0,0 +1,702 @@
+/*
+ * Copyright 2009 Red Hat, Inc.
+ *
+ * Red Hat licenses this file to you under the Apache License, version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * Copyright 2011 Dain Sundstrom <da...@iq80.com>
+ * Copyright 2011 FuseSource Corp. http://fusesource.com
+ */
+package org.kududb.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import com.google.common.primitives.Shorts;
+import org.kududb.annotations.InterfaceAudience;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.ScatteringByteChannel;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
+/**
+ * Little Endian slice of a byte array.
+ */
+@InterfaceAudience.Private
+public final class Slice implements Comparable<Slice>
+{
+ private final byte[] data;
+ private final int offset;
+ private final int length;
+
+ private int hash;
+
+ public Slice(int length)
+ {
+ data = new byte[length];
+ this.offset = 0;
+ this.length = length;
+ }
+
+ public Slice(byte[] data)
+ {
+ Preconditions.checkNotNull(data, "array is null");
+ this.data = data;
+ this.offset = 0;
+ this.length = data.length;
+ }
+
+ public Slice(byte[] data, int offset, int length)
+ {
+ Preconditions.checkNotNull(data, "array is null");
+ this.data = data;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ /**
+ * Length of this slice.
+ */
+ public int length()
+ {
+ return length;
+ }
+
+ /**
+ * Gets the array underlying this slice.
+ */
+ public byte[] getRawArray()
+ {
+ return data;
+ }
+
+ /**
+ * Gets the offset of this slice in the underlying array.
+ */
+ public int getRawOffset()
+ {
+ return offset;
+ }
+
+ /**
+ * Gets a byte at the specified absolute {@code index} in this buffer.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 1} is greater than {@code this.capacity}
+ */
+ public byte getByte(int index)
+ {
+ Preconditions.checkPositionIndexes(index, index + 1, this.length);
+ index += offset;
+ return data[index];
+ }
+
+ /**
+ * Gets an unsigned byte at the specified absolute {@code index} in this
+ * buffer.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 1} is greater than {@code this.capacity}
+ */
+ public short getUnsignedByte(int index)
+ {
+ return (short) (getByte(index) & 0xFF);
+ }
+
+ /**
+ * Gets a 16-bit short integer at the specified absolute {@code index} in
+ * this slice.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 2} is greater than {@code this.capacity}
+ */
+ public short getShort(int index)
+ {
+ Preconditions.checkPositionIndexes(index, index + Shorts.BYTES, this.length);
+ index += offset;
+ return (short) (data[index] & 0xFF | data[index + 1] << 8);
+ }
+
+ /**
+ * Gets a 32-bit integer at the specified absolute {@code index} in
+ * this buffer.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 4} is greater than {@code this.capacity}
+ */
+ public int getInt(int index)
+ {
+ Preconditions.checkPositionIndexes(index, index + Ints.BYTES, this.length);
+ index += offset;
+ return (data[index] & 0xff) |
+ (data[index + 1] & 0xff) << 8 |
+ (data[index + 2] & 0xff) << 16 |
+ (data[index + 3] & 0xff) << 24;
+ }
+
+ /**
+ * Gets a 64-bit long integer at the specified absolute {@code index} in
+ * this buffer.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 8} is greater than {@code this.capacity}
+ */
+ public long getLong(int index)
+ {
+ Preconditions.checkPositionIndexes(index, index + Longs.BYTES, this.length);
+ index += offset;
+ return ((long) data[index] & 0xff) |
+ ((long) data[index + 1] & 0xff) << 8 |
+ ((long) data[index + 2] & 0xff) << 16 |
+ ((long) data[index + 3] & 0xff) << 24 |
+ ((long) data[index + 4] & 0xff) << 32 |
+ ((long) data[index + 5] & 0xff) << 40 |
+ ((long) data[index + 6] & 0xff) << 48 |
+ ((long) data[index + 7] & 0xff) << 56;
+ }
+
+ /**
+ * Transfers this buffer's data to the specified destination starting at
+ * the specified absolute {@code index}.
+ *
+ * @param dstIndex the first index of the destination
+ * @param length the number of bytes to transfer
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
+ * if the specified {@code dstIndex} is less than {@code 0},
+ * if {@code index + length} is greater than
+ * {@code this.capacity}, or
+ * if {@code dstIndex + length} is greater than
+ * {@code dst.capacity}
+ */
+ public void getBytes(int index, Slice dst, int dstIndex, int length)
+ {
+ getBytes(index, dst.data, dstIndex, length);
+ }
+
+ /**
+ * Transfers this buffer's data to the specified destination starting at
+ * the specified absolute {@code index}.
+ *
+ * @param destinationIndex the first index of the destination
+ * @param length the number of bytes to transfer
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
+ * if the specified {@code dstIndex} is less than {@code 0},
+ * if {@code index + length} is greater than
+ * {@code this.capacity}, or
+ * if {@code dstIndex + length} is greater than
+ * {@code dst.length}
+ */
+ public void getBytes(int index, byte[] destination, int destinationIndex, int length)
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ Preconditions.checkPositionIndexes(destinationIndex, destinationIndex + length, destination.length);
+ index += offset;
+ System.arraycopy(data, index, destination, destinationIndex, length);
+ }
+
+ public byte[] getBytes()
+ {
+ return getBytes(0, length);
+ }
+
+ public byte[] getBytes(int index, int length)
+ {
+ index += offset;
+ if (index == 0) {
+ return Arrays.copyOf(data, length);
+ } else {
+ byte[] value = new byte[length];
+ System.arraycopy(data, index, value, 0, length);
+ return value;
+ }
+ }
+
+ /**
+ * Transfers this buffer's data to the specified destination starting at
+ * the specified absolute {@code index} until the destination's position
+ * reaches its limit.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * if {@code index + dst.remaining()} is greater than
+ * {@code this.capacity}
+ */
+ public void getBytes(int index, ByteBuffer destination)
+ {
+ Preconditions.checkPositionIndex(index, this.length);
+ index += offset;
+ destination.put(data, index, Math.min(length, destination.remaining()));
+ }
+
+ /**
+ * Transfers this buffer's data to the specified stream starting at the
+ * specified absolute {@code index}.
+ *
+ * @param length the number of bytes to transfer
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * if {@code index + length} is greater than
+ * {@code this.capacity}
+ * @throws java.io.IOException if the specified stream threw an exception during I/O
+ */
+ public void getBytes(int index, OutputStream out, int length)
+ throws IOException
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ index += offset;
+ out.write(data, index, length);
+ }
+
+ /**
+ * Transfers this buffer's data to the specified channel starting at the
+ * specified absolute {@code index}.
+ *
+ * @param length the maximum number of bytes to transfer
+ * @return the actual number of bytes written out to the specified channel
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * if {@code index + length} is greater than
+ * {@code this.capacity}
+ * @throws java.io.IOException if the specified channel threw an exception during I/O
+ */
+ public int getBytes(int index, GatheringByteChannel out, int length)
+ throws IOException
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ index += offset;
+ return out.write(ByteBuffer.wrap(data, index, length));
+ }
+
+ /**
+ * Sets the specified 16-bit short integer at the specified absolute
+ * {@code index} in this buffer. The 16 high-order bits of the specified
+ * value are ignored.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 2} is greater than {@code this.capacity}
+ */
+ public void setShort(int index, int value)
+ {
+ Preconditions.checkPositionIndexes(index, index + Shorts.BYTES, this.length);
+ index += offset;
+ data[index] = (byte) (value);
+ data[index + 1] = (byte) (value >>> 8);
+ }
+
+ /**
+ * Sets the specified 32-bit integer at the specified absolute
+ * {@code index} in this buffer.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 4} is greater than {@code this.capacity}
+ */
+ public void setInt(int index, int value)
+ {
+ Preconditions.checkPositionIndexes(index, index + Ints.BYTES, this.length);
+ index += offset;
+ data[index] = (byte) (value);
+ data[index + 1] = (byte) (value >>> 8);
+ data[index + 2] = (byte) (value >>> 16);
+ data[index + 3] = (byte) (value >>> 24);
+ }
+
+ /**
+ * Sets the specified 64-bit long integer at the specified absolute
+ * {@code index} in this buffer.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 8} is greater than {@code this.capacity}
+ */
+ public void setLong(int index, long value)
+ {
+ Preconditions.checkPositionIndexes(index, index + Longs.BYTES, this.length);
+ index += offset;
+ data[index] = (byte) (value);
+ data[index + 1] = (byte) (value >>> 8);
+ data[index + 2] = (byte) (value >>> 16);
+ data[index + 3] = (byte) (value >>> 24);
+ data[index + 4] = (byte) (value >>> 32);
+ data[index + 5] = (byte) (value >>> 40);
+ data[index + 6] = (byte) (value >>> 48);
+ data[index + 7] = (byte) (value >>> 56);
+ }
+
+ /**
+ * Sets the specified byte at the specified absolute {@code index} in this
+ * buffer. The 24 high-order bits of the specified value are ignored.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * {@code index + 1} is greater than {@code this.capacity}
+ */
+ public void setByte(int index, int value)
+ {
+ Preconditions.checkPositionIndexes(index, index + 1, this.length);
+ index += offset;
+ data[index] = (byte) value;
+ }
+
+ /**
+ * Transfers the specified source buffer's data to this buffer starting at
+ * the specified absolute {@code index}.
+ *
+ * @param srcIndex the first index of the source
+ * @param length the number of bytes to transfer
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
+ * if the specified {@code srcIndex} is less than {@code 0},
+ * if {@code index + length} is greater than
+ * {@code this.capacity}, or
+ * if {@code srcIndex + length} is greater than
+ * {@code src.capacity}
+ */
+ public void setBytes(int index, Slice src, int srcIndex, int length)
+ {
+ setBytes(index, src.data, src.offset + srcIndex, length);
+ }
+
+ /**
+ * Transfers the specified source array's data to this buffer starting at
+ * the specified absolute {@code index}.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0},
+ * if the specified {@code srcIndex} is less than {@code 0},
+ * if {@code index + length} is greater than
+ * {@code this.capacity}, or
+ * if {@code srcIndex + length} is greater than {@code src.length}
+ */
+ public void setBytes(int index, byte[] source, int sourceIndex, int length)
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ Preconditions.checkPositionIndexes(sourceIndex, sourceIndex + length, source.length);
+ index += offset;
+ System.arraycopy(source, sourceIndex, data, index, length);
+ }
+
+ /**
+ * Transfers the specified source buffer's data to this buffer starting at
+ * the specified absolute {@code index} until the source buffer's position
+ * reaches its limit.
+ *
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * if {@code index + src.remaining()} is greater than
+ * {@code this.capacity}
+ */
+ public void setBytes(int index, ByteBuffer source)
+ {
+ Preconditions.checkPositionIndexes(index, index + source.remaining(), this.length);
+ index += offset;
+ source.get(data, index, source.remaining());
+ }
+
+ /**
+ * Transfers the content of the specified source stream to this buffer
+ * starting at the specified absolute {@code index}.
+ *
+ * @param length the number of bytes to transfer
+ * @return the actual number of bytes read in from the specified channel.
+ * {@code -1} if the specified channel is closed.
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * if {@code index + length} is greater than {@code this.capacity}
+ * @throws java.io.IOException if the specified stream threw an exception during I/O
+ */
+ public int setBytes(int index, InputStream in, int length)
+ throws IOException
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ index += offset;
+ int readBytes = 0;
+ do {
+ int localReadBytes = in.read(data, index, length);
+ if (localReadBytes < 0) {
+ if (readBytes == 0) {
+ return -1;
+ }
+ else {
+ break;
+ }
+ }
+ readBytes += localReadBytes;
+ index += localReadBytes;
+ length -= localReadBytes;
+ } while (length > 0);
+
+ return readBytes;
+ }
+
+ /**
+ * Transfers the content of the specified source channel to this buffer
+ * starting at the specified absolute {@code index}.
+ *
+ * @param length the maximum number of bytes to transfer
+ * @return the actual number of bytes read in from the specified channel.
+ * {@code -1} if the specified channel is closed.
+ * @throws IndexOutOfBoundsException if the specified {@code index} is less than {@code 0} or
+ * if {@code index + length} is greater than {@code this.capacity}
+ * @throws java.io.IOException if the specified channel threw an exception during I/O
+ */
+ public int setBytes(int index, ScatteringByteChannel in, int length)
+ throws IOException
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ index += offset;
+ ByteBuffer buf = ByteBuffer.wrap(data, index, length);
+ int readBytes = 0;
+
+ do {
+ int localReadBytes;
+ try {
+ localReadBytes = in.read(buf);
+ }
+ catch (ClosedChannelException e) {
+ localReadBytes = -1;
+ }
+ if (localReadBytes < 0) {
+ if (readBytes == 0) {
+ return -1;
+ }
+ else {
+ break;
+ }
+ }
+ else if (localReadBytes == 0) {
+ break;
+ }
+ readBytes += localReadBytes;
+ } while (readBytes < length);
+
+ return readBytes;
+ }
+
+ public int setBytes(int index, FileChannel in, int position, int length)
+ throws IOException
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ index += offset;
+ ByteBuffer buf = ByteBuffer.wrap(data, index, length);
+ int readBytes = 0;
+
+ do {
+ int localReadBytes;
+ try {
+ localReadBytes = in.read(buf, position + readBytes);
+ }
+ catch (ClosedChannelException e) {
+ localReadBytes = -1;
+ }
+ if (localReadBytes < 0) {
+ if (readBytes == 0) {
+ return -1;
+ }
+ else {
+ break;
+ }
+ }
+ else if (localReadBytes == 0) {
+ break;
+ }
+ readBytes += localReadBytes;
+ } while (readBytes < length);
+
+ return readBytes;
+ }
+
+ public Slice copySlice()
+ {
+ return copySlice(0, length);
+ }
+
+ /**
+ * Returns a copy of this buffer's sub-region. Modifying the content of
+ * the returned buffer or this buffer does not affect each other at all.
+ */
+ public Slice copySlice(int index, int length)
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+
+ index += offset;
+ byte[] copiedArray = new byte[length];
+ System.arraycopy(data, index, copiedArray, 0, length);
+ return new Slice(copiedArray);
+ }
+
+ public byte[] copyBytes()
+ {
+ return copyBytes(0, length);
+ }
+
+ public byte[] copyBytes(int index, int length)
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ index += offset;
+ if (index == 0) {
+ return Arrays.copyOf(data, length);
+ } else {
+ byte[] value = new byte[length];
+ System.arraycopy(data, index, value, 0, length);
+ return value;
+ }
+ }
+
+ /**
+ * Returns a slice of this buffer's readable bytes. Modifying the content
+ * of the returned buffer or this buffer affects each other's content
+ * while they maintain separate indexes and marks.
+ */
+ public Slice slice()
+ {
+ return slice(0, length);
+ }
+
+ /**
+ * Returns a slice of this buffer's sub-region. Modifying the content of
+ * the returned buffer or this buffer affects each other's content while
+ * they maintain separate indexes and marks.
+ */
+ public Slice slice(int index, int length)
+ {
+ if (index == 0 && length == this.length) {
+ return this;
+ }
+
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ if (index >= 0 && length == 0) {
+ return Slices.EMPTY_SLICE;
+ }
+ return new Slice(data, offset + index, length);
+ }
+
+ /**
+ * Converts this buffer's readable bytes into a NIO buffer. The returned
+ * buffer shares the content with this buffer.
+ */
+ public ByteBuffer toByteBuffer()
+ {
+ return toByteBuffer(0, length);
+ }
+
+ /**
+ * Converts this buffer's sub-region into a NIO buffer. The returned
+ * buffer shares the content with this buffer.
+ */
+ public ByteBuffer toByteBuffer(int index, int length)
+ {
+ Preconditions.checkPositionIndexes(index, index + length, this.length);
+ index += offset;
+ return ByteBuffer.wrap(data, index, length).order(LITTLE_ENDIAN);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ Slice slice = (Slice) o;
+
+ // do lengths match
+ if (length != slice.length) {
+ return false;
+ }
+
+ // if arrays have same base offset, some optimizations can be taken...
+ if (offset == slice.offset && data == slice.data) {
+ return true;
+ }
+ for (int i = 0; i < length; i++) {
+ if (data[offset + i] != slice.data[slice.offset + i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ if (hash != 0) {
+ return hash;
+ }
+
+ int result = length;
+ for (int i = offset; i < offset + length; i++) {
+ result = 31 * result + data[i];
+ }
+ if (result == 0) {
+ result = 1;
+ }
+ hash = result;
+ return hash;
+ }
+
+ /**
+ * Compares the content of the specified buffer to the content of this
+ * buffer. This comparison is performed byte by byte using an unsigned
+ * comparison.
+ */
+ public int compareTo(Slice that)
+ {
+ if (this == that) {
+ return 0;
+ }
+ if (this.data == that.data && length == that.length && offset == that.offset) {
+ return 0;
+ }
+
+ int minLength = Math.min(this.length, that.length);
+ for (int i = 0; i < minLength; i++) {
+ int thisByte = 0xFF & this.data[this.offset + i];
+ int thatByte = 0xFF & that.data[that.offset + i];
+ if (thisByte != thatByte) {
+ return (thisByte) - (thatByte);
+ }
+ }
+ return this.length - that.length;
+ }
+
+ /**
+ * Decodes this buffer's readable bytes into a string with the specified
+ * character set name.
+ */
+ public String toString(Charset charset)
+ {
+ return toString(0, length, charset);
+ }
+
+ /**
+ * Decodes this buffer's sub-region into a string with the specified
+ * character set.
+ */
+ public String toString(int index, int length, Charset charset)
+ {
+ if (length == 0) {
+ return "";
+ }
+
+ return Slices.decodeString(toByteBuffer(index, length), charset);
+ }
+
+ public String toString()
+ {
+ return getClass().getSimpleName() + '(' +
+ "length=" + length() +
+ ')';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java b/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java
new file mode 100644
index 0000000..c2cdbde
--- /dev/null
+++ b/java/kudu-client/src/main/java/org/apache/kudu/util/Slices.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * Copyright 2011 Dain Sundstrom <da...@iq80.com>
+ * Copyright 2011 FuseSource Corp. http://fusesource.com
+ */
+package org.kududb.util;
+
+import com.google.common.base.Preconditions;
+import org.kududb.annotations.InterfaceAudience;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+@InterfaceAudience.Private
+public final class Slices
+{
+ /**
+ * A buffer whose capacity is {@code 0}.
+ */
+ public static final Slice EMPTY_SLICE = new Slice(0);
+
+ private Slices()
+ {
+ }
+
+ public static Slice ensureSize(Slice existingSlice, int minWritableBytes)
+ {
+ if (existingSlice == null) {
+ existingSlice = EMPTY_SLICE;
+ }
+
+ if (minWritableBytes <= existingSlice.length()) {
+ return existingSlice;
+ }
+
+ int newCapacity;
+ if (existingSlice.length() == 0) {
+ newCapacity = 1;
+ }
+ else {
+ newCapacity = existingSlice.length();
+ }
+ int minNewCapacity = existingSlice.length() + minWritableBytes;
+ while (newCapacity < minNewCapacity) {
+ newCapacity <<= 1;
+ }
+
+ Slice newSlice = Slices.allocate(newCapacity);
+ newSlice.setBytes(0, existingSlice, 0, existingSlice.length());
+ return newSlice;
+ }
+
+ public static Slice allocate(int capacity)
+ {
+ if (capacity == 0) {
+ return EMPTY_SLICE;
+ }
+ return new Slice(capacity);
+ }
+
+ public static Slice wrappedBuffer(byte[] array)
+ {
+ if (array.length == 0) {
+ return EMPTY_SLICE;
+ }
+ return new Slice(array);
+ }
+
+ public static Slice copiedBuffer(ByteBuffer source, int sourceOffset, int length)
+ {
+ Preconditions.checkNotNull(source, "source is null");
+ int newPosition = source.position() + sourceOffset;
+ return copiedBuffer((ByteBuffer) source.duplicate().order(ByteOrder.LITTLE_ENDIAN).clear().limit(newPosition + length).position(newPosition));
+ }
+
+ public static Slice copiedBuffer(ByteBuffer source)
+ {
+ Preconditions.checkNotNull(source, "source is null");
+ Slice copy = allocate(source.limit() - source.position());
+ copy.setBytes(0, source.duplicate().order(ByteOrder.LITTLE_ENDIAN));
+ return copy;
+ }
+
+ public static Slice copiedBuffer(String string, Charset charset)
+ {
+ Preconditions.checkNotNull(string, "string is null");
+ Preconditions.checkNotNull(charset, "charset is null");
+
+ return wrappedBuffer(string.getBytes(charset));
+ }
+
+ public static ByteBuffer encodeString(CharBuffer src, Charset charset)
+ {
+ final CharsetEncoder encoder = getEncoder(charset);
+ final ByteBuffer dst = ByteBuffer.allocate(
+ (int) ((double) src.remaining() * encoder.maxBytesPerChar()));
+ try {
+ CoderResult cr = encoder.encode(src, dst, true);
+ if (!cr.isUnderflow()) {
+ cr.throwException();
+ }
+ cr = encoder.flush(dst);
+ if (!cr.isUnderflow()) {
+ cr.throwException();
+ }
+ }
+ catch (CharacterCodingException x) {
+ throw new IllegalStateException(x);
+ }
+ dst.flip();
+ return dst;
+ }
+
+ public static String decodeString(ByteBuffer src, Charset charset)
+ {
+ final CharsetDecoder decoder = getDecoder(charset);
+ final CharBuffer dst = CharBuffer.allocate(
+ (int) ((double) src.remaining() * decoder.maxCharsPerByte()));
+ try {
+ CoderResult cr = decoder.decode(src, dst, true);
+ if (!cr.isUnderflow()) {
+ cr.throwException();
+ }
+ cr = decoder.flush(dst);
+ if (!cr.isUnderflow()) {
+ cr.throwException();
+ }
+ }
+ catch (CharacterCodingException x) {
+ throw new IllegalStateException(x);
+ }
+ return dst.flip().toString();
+ }
+
+ /**
+ * Toggles the endianness of the specified 16-bit short integer.
+ */
+ public static short swapShort(short value)
+ {
+ return (short) (value << 8 | value >>> 8 & 0xff);
+ }
+
+ /**
+ * Toggles the endianness of the specified 32-bit integer.
+ */
+ public static int swapInt(int value)
+ {
+ return swapShort((short) value) << 16 |
+ swapShort((short) (value >>> 16)) & 0xffff;
+ }
+
+ /**
+ * Toggles the endianness of the specified 64-bit long integer.
+ */
+ public static long swapLong(long value)
+ {
+ return (long) swapInt((int) value) << 32 |
+ swapInt((int) (value >>> 32)) & 0xffffffffL;
+ }
+
+ private static final ThreadLocal<Map<Charset, CharsetEncoder>> encoders =
+ new ThreadLocal<Map<Charset, CharsetEncoder>>()
+ {
+ @Override
+ protected Map<Charset, CharsetEncoder> initialValue()
+ {
+ return new IdentityHashMap<Charset, CharsetEncoder>();
+ }
+ };
+
+ private static final ThreadLocal<Map<Charset, CharsetDecoder>> decoders =
+ new ThreadLocal<Map<Charset, CharsetDecoder>>()
+ {
+ @Override
+ protected Map<Charset, CharsetDecoder> initialValue()
+ {
+ return new IdentityHashMap<Charset, CharsetDecoder>();
+ }
+ };
+
+ /**
+ * Returns a cached thread-local {@link CharsetEncoder} for the specified
+ * <tt>charset</tt>.
+ */
+ private static CharsetEncoder getEncoder(Charset charset)
+ {
+ if (charset == null) {
+ throw new NullPointerException("charset");
+ }
+
+ Map<Charset, CharsetEncoder> map = encoders.get();
+ CharsetEncoder e = map.get(charset);
+ if (e != null) {
+ e.reset();
+ e.onMalformedInput(CodingErrorAction.REPLACE);
+ e.onUnmappableCharacter(CodingErrorAction.REPLACE);
+ return e;
+ }
+
+ e = charset.newEncoder();
+ e.onMalformedInput(CodingErrorAction.REPLACE);
+ e.onUnmappableCharacter(CodingErrorAction.REPLACE);
+ map.put(charset, e);
+ return e;
+ }
+
+
+ /**
+ * Returns a cached thread-local {@link CharsetDecoder} for the specified
+ * <tt>charset</tt>.
+ */
+ private static CharsetDecoder getDecoder(Charset charset)
+ {
+ if (charset == null) {
+ throw new NullPointerException("charset");
+ }
+
+ Map<Charset, CharsetDecoder> map = decoders.get();
+ CharsetDecoder d = map.get(charset);
+ if (d != null) {
+ d.reset();
+ d.onMalformedInput(CodingErrorAction.REPLACE);
+ d.onUnmappableCharacter(CodingErrorAction.REPLACE);
+ return d;
+ }
+
+ d = charset.newDecoder();
+ d.onMalformedInput(CodingErrorAction.REPLACE);
+ d.onUnmappableCharacter(CodingErrorAction.REPLACE);
+ map.put(charset, d);
+ return d;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/ColumnSchema.java b/java/kudu-client/src/main/java/org/kududb/ColumnSchema.java
deleted file mode 100644
index c3c617d..0000000
--- a/java/kudu-client/src/main/java/org/kududb/ColumnSchema.java
+++ /dev/null
@@ -1,301 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb;
-
-import org.kududb.Common.CompressionType;
-import org.kududb.Common.EncodingType;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-/**
- * Represents a Kudu Table column. Use {@link ColumnSchema.ColumnSchemaBuilder} in order to
- * create columns.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class ColumnSchema {
-
- private final String name;
- private final Type type;
- private final boolean key;
- private final boolean nullable;
- private final Object defaultValue;
- private final int desiredBlockSize;
- private final Encoding encoding;
- private final CompressionAlgorithm compressionAlgorithm;
-
- /**
- * Specifies the encoding of data for a column on disk.
- * Not all encodings are available for all data types.
- * Refer to the Kudu documentation for more information on each encoding.
- */
- public enum Encoding {
- UNKNOWN(EncodingType.UNKNOWN_ENCODING),
- AUTO_ENCODING(EncodingType.AUTO_ENCODING),
- PLAIN_ENCODING(EncodingType.PLAIN_ENCODING),
- PREFIX_ENCODING(EncodingType.PREFIX_ENCODING),
- GROUP_VARINT(EncodingType.GROUP_VARINT),
- RLE(EncodingType.RLE),
- DICT_ENCODING(EncodingType.DICT_ENCODING),
- BIT_SHUFFLE(EncodingType.BIT_SHUFFLE);
-
- final EncodingType internalPbType;
-
- Encoding(EncodingType internalPbType) {
- this.internalPbType = internalPbType;
- }
-
- @InterfaceAudience.Private
- public EncodingType getInternalPbType() {
- return internalPbType;
- }
- };
-
- /**
- * Specifies the compression algorithm of data for a column on disk.
- */
- public enum CompressionAlgorithm {
- UNKNOWN(CompressionType.UNKNOWN_COMPRESSION),
- DEFAULT_COMPRESSION(CompressionType.DEFAULT_COMPRESSION),
- NO_COMPRESSION(CompressionType.NO_COMPRESSION),
- SNAPPY(CompressionType.SNAPPY),
- LZ4(CompressionType.LZ4),
- ZLIB(CompressionType.ZLIB);
-
- final CompressionType internalPbType;
-
- CompressionAlgorithm(CompressionType internalPbType) {
- this.internalPbType = internalPbType;
- }
-
- @InterfaceAudience.Private
- public CompressionType getInternalPbType() {
- return internalPbType;
- }
- };
-
- private ColumnSchema(String name, Type type, boolean key, boolean nullable,
- Object defaultValue, int desiredBlockSize, Encoding encoding,
- CompressionAlgorithm compressionAlgorithm) {
- this.name = name;
- this.type = type;
- this.key = key;
- this.nullable = nullable;
- this.defaultValue = defaultValue;
- this.desiredBlockSize = desiredBlockSize;
- this.encoding = encoding;
- this.compressionAlgorithm = compressionAlgorithm;
- }
-
- /**
- * Get the column's Type
- * @return the type
- */
- public Type getType() {
- return type;
- }
-
- /**
- * Get the column's name
- * @return A string representation of the name
- */
- public String getName() {
- return name;
- }
-
- /**
- * Answers if the column part of the key
- * @return true if the column is part of the key, else false
- */
- public boolean isKey() {
- return key;
- }
-
- /**
- * Answers if the column can be set to null
- * @return true if it can be set to null, else false
- */
- public boolean isNullable() {
- return nullable;
- }
-
- /**
- * The Java object representation of the default value that's read
- * @return the default read value
- */
- public Object getDefaultValue() {
- return defaultValue;
- }
-
- /**
- * Gets the desired block size for this column.
- * If no block size has been explicitly specified for this column,
- * returns 0 to indicate that the server-side default will be used.
- *
- * @return the block size, in bytes, or 0 if none has been configured.
- */
- public int getDesiredBlockSize() {
- return desiredBlockSize;
- }
-
- /**
- * Return the encoding of this column, or null if it is not known.
- */
- public Encoding getEncoding() {
- return encoding;
- }
-
- /**
- * Return the compression algorithm of this column, or null if it is not known.
- */
- public CompressionAlgorithm getCompressionAlgorithm() {
- return compressionAlgorithm;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- ColumnSchema that = (ColumnSchema) o;
-
- if (key != that.key) return false;
- if (!name.equals(that.name)) return false;
- if (!type.equals(that.type)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = name.hashCode();
- result = 31 * result + type.hashCode();
- result = 31 * result + (key ? 1 : 0);
- return result;
- }
-
- @Override
- public String toString() {
- return "Column name: " + name + ", type: " + type.getName();
- }
-
- /**
- * Builder for ColumnSchema.
- */
- public static class ColumnSchemaBuilder {
- private final String name;
- private final Type type;
- private boolean key = false;
- private boolean nullable = false;
- private Object defaultValue = null;
- private int blockSize = 0;
- private Encoding encoding = null;
- private CompressionAlgorithm compressionAlgorithm = null;
-
- /**
- * Constructor for the required parameters.
- * @param name column's name
- * @param type column's type
- */
- public ColumnSchemaBuilder(String name, Type type) {
- this.name = name;
- this.type = type;
- }
-
- /**
- * Sets if the column is part of the row key. False by default.
- * @param key a boolean that indicates if the column is part of the key
- * @return this instance
- */
- public ColumnSchemaBuilder key(boolean key) {
- this.key = key;
- return this;
- }
-
- /**
- * Marks the column as allowing null values. False by default.
- * @param nullable a boolean that indicates if the column allows null values
- * @return this instance
- */
- public ColumnSchemaBuilder nullable(boolean nullable) {
- this.nullable = nullable;
- return this;
- }
-
- /**
- * Sets the default value that will be read from the column. Null by default.
- * @param defaultValue a Java object representation of the default value that's read
- * @return this instance
- */
- public ColumnSchemaBuilder defaultValue(Object defaultValue) {
- this.defaultValue = defaultValue;
- return this;
- }
-
- /**
- * Set the desired block size for this column.
- *
- * This is the number of bytes of user data packed per block on disk, and
- * represents the unit of IO when reading this column. Larger values
- * may improve scan performance, particularly on spinning media. Smaller
- * values may improve random access performance, particularly for workloads
- * that have high cache hit rates or operate on fast storage such as SSD.
- *
- * Note that the block size specified here corresponds to uncompressed data.
- * The actual size of the unit read from disk may be smaller if
- * compression is enabled.
- *
- * It's recommended that this not be set any lower than 4096 (4KB) or higher
- * than 1048576 (1MB).
- * @param blockSize the desired block size, in bytes
- * @return this instance
- * <!-- TODO(KUDU-1107): move the above info to docs -->
- */
- public ColumnSchemaBuilder desiredBlockSize(int blockSize) {
- this.blockSize = blockSize;
- return this;
- }
-
- /**
- * Set the block encoding for this column. See the documentation for the list
- * of valid options.
- */
- public ColumnSchemaBuilder encoding(Encoding encoding) {
- this.encoding = encoding;
- return this;
- }
-
- /**
- * Set the compression algorithm for this column. See the documentation for the list
- * of valid options.
- */
- public ColumnSchemaBuilder compressionAlgorithm(CompressionAlgorithm compressionAlgorithm) {
- this.compressionAlgorithm = compressionAlgorithm;
- return this;
- }
-
- /**
- * Builds a {@link ColumnSchema} using the passed parameters.
- * @return a new {@link ColumnSchema}
- */
- public ColumnSchema build() {
- return new ColumnSchema(name, type,
- key, nullable, defaultValue,
- blockSize, encoding, compressionAlgorithm);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/Schema.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/Schema.java b/java/kudu-client/src/main/java/org/kududb/Schema.java
deleted file mode 100644
index 17e7798..0000000
--- a/java/kudu-client/src/main/java/org/kududb/Schema.java
+++ /dev/null
@@ -1,291 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb;
-
-import com.google.common.collect.ImmutableList;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.client.Bytes;
-import org.kududb.client.PartialRow;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Represents table's schema which is essentially a list of columns.
- * This class offers a few utility methods for querying it.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Schema {
-
- /**
- * Mapping of column index to column.
- */
- private final List<ColumnSchema> columnsByIndex;
-
- /**
- * The primary key columns.
- */
- private final List<ColumnSchema> primaryKeyColumns = new ArrayList<>();
-
- /**
- * Mapping of column name to index.
- */
- private final Map<String, Integer> columnsByName;
-
- /**
- * Mapping of column ID to index, or null if the schema does not have assigned column IDs.
- */
- private final Map<Integer, Integer> columnsById;
-
- /**
- * Mapping of column index to backing byte array offset.
- */
- private final int[] columnOffsets;
-
- private final int varLengthColumnCount;
- private final int rowSize;
- private final boolean hasNullableColumns;
-
- /**
- * Constructs a schema using the specified columns and does some internal accounting
- * @param columns the columns in index order
- * @throws IllegalArgumentException If the key columns aren't specified first
- *
- * See {@code ColumnPBsToSchema()} in {@code src/kudu/common/wire_protocol.cc}
- */
- public Schema(List<ColumnSchema> columns) {
- this(columns, null);
- }
-
- /**
- * Constructs a schema using the specified columns and IDs.
- *
- * This is not a stable API, prefer using {@link Schema#Schema(List)} to create a new schema.
- *
- * @param columns the columns in index order
- * @param columnIds the column ids of the provided columns, or null
- * @throws IllegalArgumentException If the primary key columns aren't specified first
- * @throws IllegalArgumentException If the column ids length does not match the columns length
- *
- * See {@code ColumnPBsToSchema()} in {@code src/kudu/common/wire_protocol.cc}
- */
- public Schema(List<ColumnSchema> columns, List<Integer> columnIds) {
- boolean hasColumnIds = columnIds != null;
- if (hasColumnIds && columns.size() != columnIds.size()) {
- throw new IllegalArgumentException(
- "Schema must be constructed with all column IDs, or none.");
- }
-
- this.columnsByIndex = ImmutableList.copyOf(columns);
- int varLenCnt = 0;
- this.columnOffsets = new int[columns.size()];
- this.columnsByName = new HashMap<>(columns.size());
- this.columnsById = hasColumnIds ? new HashMap<Integer, Integer>(columnIds.size()) : null;
- int offset = 0;
- boolean hasNulls = false;
- // pre-compute a few counts and offsets
- for (int index = 0; index < columns.size(); index++) {
- final ColumnSchema column = columns.get(index);
- if (column.isKey()) {
- primaryKeyColumns.add(column);
- }
-
- hasNulls |= column.isNullable();
- columnOffsets[index] = offset;
- offset += column.getType().getSize();
- if (this.columnsByName.put(column.getName(), index) != null) {
- throw new IllegalArgumentException(
- String.format("Column names must be unique: %s", columns));
- }
- if (column.getType() == Type.STRING || column.getType() == Type.BINARY) {
- varLenCnt++;
- }
-
- if (hasColumnIds) {
- if (this.columnsById.put(columnIds.get(index), index) != null) {
- throw new IllegalArgumentException(
- String.format("Column IDs must be unique: %s", columnIds));
- }
- }
- }
-
- this.hasNullableColumns = hasNulls;
- this.varLengthColumnCount = varLenCnt;
- this.rowSize = getRowSize(this.columnsByIndex);
- }
-
- /**
- * Get the list of columns used to create this schema
- * @return list of columns
- */
- public List<ColumnSchema> getColumns() {
- return this.columnsByIndex;
- }
-
- /**
- * Get the count of columns with variable length (BINARY/STRING) in
- * this schema.
- * @return strings count
- */
- public int getVarLengthColumnCount() {
- return this.varLengthColumnCount;
- }
-
- /**
- * Get the size a row built using this schema would be
- * @return size in bytes
- */
- public int getRowSize() {
- return this.rowSize;
- }
-
- /**
- * Get the index at which this column can be found in the backing byte array
- * @param idx column's index
- * @return column's offset
- */
- public int getColumnOffset(int idx) {
- return this.columnOffsets[idx];
- }
-
- /**
- * Get the index for the provided column name.
- * @param columnName column to search for
- * @return an index in the schema
- */
- public int getColumnIndex(String columnName) {
- Integer index = this.columnsByName.get(columnName);
- if (index == null) {
- throw new IllegalArgumentException(
- String.format("Unknown column: %s", columnName));
- }
- return index;
- }
-
- /**
- * Get the column at the specified index in the original list
- * @param idx column's index
- * @return the column
- */
- public ColumnSchema getColumnByIndex(int idx) {
- return this.columnsByIndex.get(idx);
- }
-
- /**
- * Get the column index of the column with the provided ID.
- * This method is not part of the stable API.
- * @param columnId the column id of the column
- * @return the column index of the column.
- */
- public int getColumnIndex(int columnId) {
- if (!hasColumnIds()) throw new IllegalStateException("Schema does not have Column IDs");
- Integer index = this.columnsById.get(columnId);
- if (index == null) throw new IllegalArgumentException(
- String.format("Unknown column id: %s", columnId));
- return index;
- }
-
- /**
- * Get the column associated with the specified name
- * @param columnName column's name
- * @return the column
- */
- public ColumnSchema getColumn(String columnName) {
- return columnsByIndex.get(getColumnIndex(columnName));
- }
-
- /**
- * Get the count of columns in this schema
- * @return count of columns
- */
- public int getColumnCount() {
- return this.columnsByIndex.size();
- }
-
- /**
- * Get the count of columns that are part of the primary key.
- * @return count of primary key columns.
- */
- public int getPrimaryKeyColumnCount() {
- return this.primaryKeyColumns.size();
- }
-
- /**
- * Get the primary key columns.
- * @return the primary key columns.
- */
- public List<ColumnSchema> getPrimaryKeyColumns() {
- return primaryKeyColumns;
- }
-
- /**
- * Get a schema that only contains the columns which are part of the key
- * @return new schema with only the keys
- */
- public Schema getRowKeyProjection() {
- return new Schema(primaryKeyColumns);
- }
-
- /**
- * Tells if there's at least one nullable column
- * @return true if at least one column is nullable, else false.
- */
- public boolean hasNullableColumns() {
- return this.hasNullableColumns;
- }
-
- /**
- * Tells whether this schema includes IDs for columns. A schema created by a client as part of
- * table creation will not include IDs, but schemas for open tables will include IDs.
- * This method is not part of the stable API.
- *
- * @return whether this schema includes column IDs.
- */
- public boolean hasColumnIds() {
- return columnsById != null;
- }
-
- /**
- * Gives the size in bytes for a single row given the specified schema
- * @param columns the row's columns
- * @return row size in bytes
- */
- private static int getRowSize(List<ColumnSchema> columns) {
- int totalSize = 0;
- boolean hasNullables = false;
- for (ColumnSchema column : columns) {
- totalSize += column.getType().getSize();
- hasNullables |= column.isNullable();
- }
- if (hasNullables) {
- totalSize += Bytes.getBitSetSize(columns.size());
- }
- return totalSize;
- }
-
- /**
- * Creates a new partial row for the schema.
- * @return a new partial row
- */
- public PartialRow newPartialRow() {
- return new PartialRow(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/Type.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/Type.java b/java/kudu-client/src/main/java/org/kududb/Type.java
deleted file mode 100644
index fd23835..0000000
--- a/java/kudu-client/src/main/java/org/kududb/Type.java
+++ /dev/null
@@ -1,136 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import com.google.common.primitives.Shorts;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import static org.kududb.Common.DataType;
-
-/**
- * Describes all the types available to build table schemas.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public enum Type {
-
- INT8 (DataType.INT8, "int8"),
- INT16 (DataType.INT16, "int16"),
- INT32 (DataType.INT32, "int32"),
- INT64 (DataType.INT64, "int64"),
- BINARY (DataType.BINARY, "binary"),
- STRING (DataType.STRING, "string"),
- BOOL (DataType.BOOL, "bool"),
- FLOAT (DataType.FLOAT, "float"),
- DOUBLE (DataType.DOUBLE, "double"),
- TIMESTAMP (DataType.TIMESTAMP, "timestamp");
-
-
- private final DataType dataType;
- private final String name;
- private final int size;
-
- /**
- * Private constructor used to pre-create the types
- * @param dataType DataType from the common's pb
- * @param name string representation of the type
- */
- private Type(DataType dataType, String name) {
- this.dataType = dataType;
- this.name = name;
- this.size = getTypeSize(this.dataType);
- }
-
- /**
- * Get the data type from the common's pb
- * @return A DataType
- */
- public DataType getDataType() {
- return this.dataType;
- }
-
- /**
- * Get the string representation of this type
- * @return The type's name
- */
- public String getName() {
- return this.name;
- }
-
- /**
- * The size of this type on the wire
- * @return A size
- */
- public int getSize() {
- return this.size;
- }
-
- @Override
- public String toString() {
- return "Type: " + this.name + ", size: " + this.size;
- }
-
- /**
- * Gives the size in bytes for a given DataType, as per the pb specification
- * @param type pb type
- * @return size in bytes
- */
- static int getTypeSize(DataType type) {
- switch (type) {
- case STRING:
- case BINARY: return 8 + 8; // offset then string length
- case BOOL:
- case INT8: return 1;
- case INT16: return Shorts.BYTES;
- case INT32:
- case FLOAT: return Ints.BYTES;
- case INT64:
- case DOUBLE:
- case TIMESTAMP: return Longs.BYTES;
- default: throw new IllegalArgumentException("The provided data type doesn't map" +
- " to know any known one.");
- }
- }
-
- /**
- * Convert the pb DataType to a Type
- * @param type DataType to convert
- * @return a matching Type
- */
- public static Type getTypeForDataType(DataType type) {
- switch (type) {
- case STRING: return STRING;
- case BINARY: return BINARY;
- case BOOL: return BOOL;
- case INT8: return INT8;
- case INT16: return INT16;
- case INT32: return INT32;
- case INT64: return INT64;
- case TIMESTAMP: return TIMESTAMP;
- case FLOAT: return FLOAT;
- case DOUBLE: return DOUBLE;
- default:
- throw new IllegalArgumentException("The provided data type doesn't map" +
- " to know any known one: " + type.getDescriptorForType().getFullName());
-
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/AbstractKuduScannerBuilder.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AbstractKuduScannerBuilder.java b/java/kudu-client/src/main/java/org/kududb/client/AbstractKuduScannerBuilder.java
deleted file mode 100644
index b1b0712..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/AbstractKuduScannerBuilder.java
+++ /dev/null
@@ -1,338 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.collect.ImmutableList;
-import org.kududb.Common;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-import org.kududb.tserver.Tserver;
-import org.kududb.util.HybridTimeUtil;
-
-/**
- * Abstract class to extend in order to create builders for scanners.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public abstract class AbstractKuduScannerBuilder
- <S extends AbstractKuduScannerBuilder<? super S, T>, T> {
- final AsyncKuduClient client;
- final KuduTable table;
-
- /** Map of column name to predicate */
- final Map<String, KuduPredicate> predicates = new HashMap<>();
-
- AsyncKuduScanner.ReadMode readMode = AsyncKuduScanner.ReadMode.READ_LATEST;
- Common.OrderMode orderMode = Common.OrderMode.UNORDERED;
- int batchSizeBytes = 1024*1024;
- long limit = Long.MAX_VALUE;
- boolean prefetching = false;
- boolean cacheBlocks = true;
- long htTimestamp = AsyncKuduClient.NO_TIMESTAMP;
- byte[] lowerBoundPrimaryKey = AsyncKuduClient.EMPTY_ARRAY;
- byte[] upperBoundPrimaryKey = AsyncKuduClient.EMPTY_ARRAY;
- byte[] lowerBoundPartitionKey = AsyncKuduClient.EMPTY_ARRAY;
- byte[] upperBoundPartitionKey = AsyncKuduClient.EMPTY_ARRAY;
- List<String> projectedColumnNames = null;
- List<Integer> projectedColumnIndexes = null;
- long scanRequestTimeout;
-
- AbstractKuduScannerBuilder(AsyncKuduClient client, KuduTable table) {
- this.client = client;
- this.table = table;
- this.scanRequestTimeout = client.getDefaultOperationTimeoutMs();
- }
-
- /**
- * Sets the read mode, the default is to read the latest values.
- * @param readMode a read mode for the scanner
- * @return this instance
- */
- public S readMode(AsyncKuduScanner.ReadMode readMode) {
- this.readMode = readMode;
- return (S) this;
- }
-
- /**
- * Return scan results in primary key sorted order.
- *
- * If the table is hash partitioned, the scan must have an equality predicate
- * on all hashed columns.
- *
- * Package private until proper hash partitioning equality predicate checks
- * are in place.
- *
- * Disabled by default.
- * @return this instance
- */
- @InterfaceAudience.Private
- @InterfaceStability.Unstable
- S sortResultsByPrimaryKey() {
- orderMode = Common.OrderMode.ORDERED;
- readMode = AsyncKuduScanner.ReadMode.READ_AT_SNAPSHOT;
- return (S) this;
- }
-
- /**
- * Adds a predicate for a column.
- * @param predicate predicate for a column to add
- * @return this instance
- * @deprecated use {@link #addPredicate(KuduPredicate)}
- */
- @Deprecated
- public S addColumnRangePredicate(ColumnRangePredicate predicate) {
- return addPredicate(predicate.toKuduPredicate());
- }
-
- /**
- * Adds a list of predicates in their raw format,
- * as given by {@link ColumnRangePredicate#toByteArray(List)}.
- * @param predicateBytes predicates to add
- * @return this instance
- * @throws IllegalArgumentException thrown when the passed bytes aren't valid
- * @deprecated use {@link #addPredicate}
- */
- @Deprecated
- public S addColumnRangePredicatesRaw(byte[] predicateBytes) {
- for (Tserver.ColumnRangePredicatePB pb : ColumnRangePredicate.fromByteArray(predicateBytes)) {
- addPredicate(ColumnRangePredicate.fromPb(pb).toKuduPredicate());
- }
- return (S) this;
- }
-
- /**
- * Adds a predicate to the scan.
- * @param predicate predicate to add
- * @return this instance
- */
- public S addPredicate(KuduPredicate predicate) {
- String columnName = predicate.getColumn().getName();
- KuduPredicate existing = predicates.get(columnName);
- if (existing == null) {
- predicates.put(columnName, predicate);
- } else {
- predicates.put(columnName, existing.merge(predicate));
- }
- return (S) this;
- }
-
- /**
- * Set which columns will be read by the Scanner.
- * Calling this method after {@link #setProjectedColumnIndexes(List)} will reset the projected
- * columns to those specified in {@code columnNames}.
- * @param columnNames the names of columns to read, or 'null' to read all columns
- * (the default)
- */
- public S setProjectedColumnNames(List<String> columnNames) {
- projectedColumnIndexes = null;
- if (columnNames != null) {
- projectedColumnNames = ImmutableList.copyOf(columnNames);
- } else {
- projectedColumnNames = null;
- }
- return (S) this;
- }
-
- /**
- * Set which columns will be read by the Scanner.
- * Calling this method after {@link #setProjectedColumnNames(List)} will reset the projected
- * columns to those specified in {@code columnIndexes}.
- * @param columnIndexes the indexes of columns to read, or 'null' to read all columns
- * (the default)
- */
- public S setProjectedColumnIndexes(List<Integer> columnIndexes) {
- projectedColumnNames = null;
- if (columnIndexes != null) {
- projectedColumnIndexes = ImmutableList.copyOf(columnIndexes);
- } else {
- projectedColumnIndexes = null;
- }
- return (S) this;
- }
-
- /**
- * Sets the maximum number of bytes returned by the scanner, on each batch. The default is 1MB.
- * <p>
- * Kudu may actually return more than this many bytes because it will not
- * truncate a rowResult in the middle.
- * @param batchSizeBytes a strictly positive number of bytes
- * @return this instance
- */
- public S batchSizeBytes(int batchSizeBytes) {
- this.batchSizeBytes = batchSizeBytes;
- return (S) this;
- }
-
- /**
- * Sets a limit on the number of rows that will be returned by the scanner. There's no limit
- * by default.
- * @param limit a positive long
- * @return this instance
- */
- public S limit(long limit) {
- this.limit = limit;
- return (S) this;
- }
-
- /**
- * Enables prefetching of rows for the scanner, i.e. whether to send a request for more data
- * to the server immediately after we receive a response (instead of waiting for the user
- * to call {@code nextRows()}). Disabled by default.
- * NOTE: This is risky until KUDU-1260 is resolved.
- * @param prefetching a boolean that indicates if the scanner should prefetch rows
- * @return this instance
- */
- public S prefetching(boolean prefetching) {
- this.prefetching = prefetching;
- return (S) this;
- }
-
- /**
- * Sets the block caching policy for the scanner. If true, scanned data blocks will be cached
- * in memory and made available for future scans. Enabled by default.
- * @param cacheBlocks a boolean that indicates if data blocks should be cached or not
- * @return this instance
- */
- public S cacheBlocks(boolean cacheBlocks) {
- this.cacheBlocks = cacheBlocks;
- return (S) this;
- }
-
- /**
- * Sets a previously encoded HT timestamp as a snapshot timestamp, for tests. None is used by
- * default.
- * Requires that the ReadMode is READ_AT_SNAPSHOT.
- * @param htTimestamp a long representing a HybridClock-encoded timestamp
- * @return this instance
- * @throws IllegalArgumentException on build(), if the timestamp is less than 0 or if the
- * read mode was not set to READ_AT_SNAPSHOT
- */
- @InterfaceAudience.Private
- public S snapshotTimestampRaw(long htTimestamp) {
- this.htTimestamp = htTimestamp;
- return (S) this;
- }
-
- /**
- * Sets the timestamp the scan must be executed at, in microseconds since the Unix epoch. None is
- * used by default.
- * Requires that the ReadMode is READ_AT_SNAPSHOT.
- * @param timestamp a long representing an instant in microseconds since the unix epoch.
- * @return this instance
- * @throws IllegalArgumentException on build(), if the timestamp is less than 0 or if the
- * read mode was not set to READ_AT_SNAPSHOT
- */
- public S snapshotTimestampMicros(long timestamp) {
- this.htTimestamp = HybridTimeUtil.physicalAndLogicalToHTTimestamp(timestamp, 0);
- return (S) this;
- }
-
- /**
- * Sets how long each scan request to a server can last.
- * Defaults to {@link KuduClient#getDefaultOperationTimeoutMs()}.
- * @param scanRequestTimeout a long representing time in milliseconds
- * @return this instance
- */
- public S scanRequestTimeout(long scanRequestTimeout) {
- this.scanRequestTimeout = scanRequestTimeout;
- return (S) this;
- }
-
- /**
- * Add a lower bound (inclusive) primary key for the scan.
- * If any bound is already added, this bound is intersected with that one.
- * @param partialRow a partial row with specified key columns
- * @return this instance
- */
- public S lowerBound(PartialRow partialRow) {
- return lowerBoundRaw(partialRow.encodePrimaryKey());
- }
-
- /**
- * Like lowerBoundPrimaryKey() but the encoded primary key is an opaque byte array obtained elsewhere.
- * @param startPrimaryKey bytes containing an encoded start key
- * @return this instance
- * @deprecated use {@link #lowerBound(PartialRow)}
- */
- @Deprecated
- public S lowerBoundRaw(byte[] startPrimaryKey) {
- if (lowerBoundPrimaryKey == AsyncKuduClient.EMPTY_ARRAY ||
- Bytes.memcmp(startPrimaryKey, lowerBoundPrimaryKey) > 0) {
- this.lowerBoundPrimaryKey = startPrimaryKey;
- }
- return (S) this;
- }
-
- /**
- * Add an upper bound (exclusive) primary key for the scan.
- * If any bound is already added, this bound is intersected with that one.
- * @param partialRow a partial row with specified key columns
- * @return this instance
- */
- public S exclusiveUpperBound(PartialRow partialRow) {
- return exclusiveUpperBoundRaw(partialRow.encodePrimaryKey());
- }
-
- /**
- * Like exclusiveUpperBound() but the encoded primary key is an opaque byte array obtained elsewhere.
- * @param endPrimaryKey bytes containing an encoded end key
- * @return this instance
- * @deprecated use {@link #exclusiveUpperBound(PartialRow)}
- */
- @Deprecated
- public S exclusiveUpperBoundRaw(byte[] endPrimaryKey) {
- if (upperBoundPrimaryKey == AsyncKuduClient.EMPTY_ARRAY ||
- Bytes.memcmp(endPrimaryKey, upperBoundPrimaryKey) < 0) {
- this.upperBoundPrimaryKey = endPrimaryKey;
- }
- return (S) this;
- }
-
- /**
- * Set an encoded (inclusive) start partition key for the scan.
- *
- * @param partitionKey the encoded partition key
- * @return this instance
- */
- @InterfaceAudience.LimitedPrivate("Impala")
- public S lowerBoundPartitionKeyRaw(byte[] partitionKey) {
- if (Bytes.memcmp(partitionKey, lowerBoundPartitionKey) > 0) {
- this.lowerBoundPartitionKey = partitionKey;
- }
- return (S) this;
- }
-
- /**
- * Set an encoded (exclusive) end partition key for the scan.
- *
- * @param partitionKey the encoded partition key
- * @return this instance
- */
- @InterfaceAudience.LimitedPrivate("Impala")
- public S exclusiveUpperBoundPartitionKeyRaw(byte[] partitionKey) {
- if (upperBoundPartitionKey.length == 0 || Bytes.memcmp(partitionKey, upperBoundPartitionKey) < 0) {
- this.upperBoundPartitionKey = partitionKey;
- }
- return (S) this;
- }
-
- public abstract T build();
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/AlterTableOptions.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AlterTableOptions.java b/java/kudu-client/src/main/java/org/kududb/client/AlterTableOptions.java
deleted file mode 100644
index ecffb5e..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/AlterTableOptions.java
+++ /dev/null
@@ -1,107 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.ColumnSchema;
-import org.kududb.Type;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-import static org.kududb.master.Master.AlterTableRequestPB;
-
-/**
- * This builder must be used to alter a table. At least one change must be specified.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class AlterTableOptions {
-
- AlterTableRequestPB.Builder pb = AlterTableRequestPB.newBuilder();
-
- /**
- * Change a table's name.
- * @param newName new table's name, must be used to check progress
- * @return this instance
- */
- public AlterTableOptions renameTable(String newName) {
- pb.setNewTableName(newName);
- return this;
- }
-
- /**
- * Add a new column that's not nullable.
- * @param name name of the new column
- * @param type type of the new column
- * @param defaultVal default value used for the currently existing rows
- * @return this instance
- */
- public AlterTableOptions addColumn(String name, Type type, Object defaultVal) {
- if (defaultVal == null) {
- throw new IllegalArgumentException("A new column must have a default value, " +
- "use addNullableColumn() to add a NULLABLE column");
- }
- AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
- step.setType(AlterTableRequestPB.StepType.ADD_COLUMN);
- step.setAddColumn(AlterTableRequestPB.AddColumn.newBuilder().setSchema(ProtobufHelper
- .columnToPb(new ColumnSchema.ColumnSchemaBuilder(name, type)
- .defaultValue(defaultVal)
- .build())));
- return this;
- }
-
- /**
- * Add a new column that's nullable, thus has no default value.
- * @param name name of the new column
- * @param type type of the new column
- * @return this instance
- */
- public AlterTableOptions addNullableColumn(String name, Type type) {
- AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
- step.setType(AlterTableRequestPB.StepType.ADD_COLUMN);
- step.setAddColumn(AlterTableRequestPB.AddColumn.newBuilder().setSchema(ProtobufHelper
- .columnToPb(new ColumnSchema.ColumnSchemaBuilder(name, type)
- .nullable(true)
- .build())));
- return this;
- }
-
- /**
- * Drop a column.
- * @param name name of the column
- * @return this instance
- */
- public AlterTableOptions dropColumn(String name) {
- AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
- step.setType(AlterTableRequestPB.StepType.DROP_COLUMN);
- step.setDropColumn(AlterTableRequestPB.DropColumn.newBuilder().setName(name));
- return this;
- }
-
- /**
- * Change the name of a column.
- * @param oldName old column's name, must exist
- * @param newName new name to use
- * @return this instance
- */
- public AlterTableOptions renameColumn(String oldName, String newName) {
- AlterTableRequestPB.Step.Builder step = pb.addAlterSchemaStepsBuilder();
- step.setType(AlterTableRequestPB.StepType.RENAME_COLUMN);
- step.setRenameColumn(AlterTableRequestPB.RenameColumn.newBuilder().setOldName(oldName)
- .setNewName(newName));
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/AlterTableRequest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AlterTableRequest.java b/java/kudu-client/src/main/java/org/kududb/client/AlterTableRequest.java
deleted file mode 100644
index 751290c..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/AlterTableRequest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import com.google.protobuf.Message;
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.util.Pair;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-import static org.kududb.master.Master.*;
-
-/**
- * RPC used to alter a table. When it returns it doesn't mean that the table is altered,
- * a success just means that the master accepted it.
- */
-@InterfaceAudience.Private
-class AlterTableRequest extends KuduRpc<AlterTableResponse> {
-
- static final String ALTER_TABLE = "AlterTable";
- private final String name;
- private final AlterTableRequestPB.Builder builder;
-
- AlterTableRequest(KuduTable masterTable, String name, AlterTableOptions ato) {
- super(masterTable);
- this.name = name;
- this.builder = ato.pb;
- }
-
- @Override
- ChannelBuffer serialize(Message header) {
- assert header.isInitialized();
- TableIdentifierPB tableID =
- TableIdentifierPB.newBuilder().setTableName(name).build();
- this.builder.setTable(tableID);
- return toChannelBuffer(header, this.builder.build());
- }
-
- @Override
- String serviceName() { return MASTER_SERVICE_NAME; }
-
- @Override
- String method() {
- return ALTER_TABLE;
- }
-
- @Override
- Pair<AlterTableResponse, Object> deserialize(final CallResponse callResponse,
- String tsUUID) throws Exception {
- final AlterTableResponsePB.Builder respBuilder = AlterTableResponsePB.newBuilder();
- readProtobuf(callResponse.getPBMessage(), respBuilder);
- AlterTableResponse response = new AlterTableResponse(deadlineTracker.getElapsedMillis(),
- tsUUID);
- return new Pair<AlterTableResponse, Object>(
- response, respBuilder.hasError() ? respBuilder.getError() : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/5c305689/java/kudu-client/src/main/java/org/kududb/client/AlterTableResponse.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/AlterTableResponse.java b/java/kudu-client/src/main/java/org/kududb/client/AlterTableResponse.java
deleted file mode 100644
index 7d1d581..0000000
--- a/java/kudu-client/src/main/java/org/kududb/client/AlterTableResponse.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.kududb.client;
-
-import org.kududb.annotations.InterfaceAudience;
-import org.kududb.annotations.InterfaceStability;
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class AlterTableResponse extends KuduRpcResponse {
-
- /**
- * @param ellapsedMillis Time in milliseconds since RPC creation to now.
- */
- AlterTableResponse(long ellapsedMillis, String tsUUID) {
- super(ellapsedMillis, tsUUID);
- }
-}
\ No newline at end of file