You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/07/12 19:20:45 UTC
[1/2] kudu git commit: [Java] Fix the use of resource files in tests
Repository: kudu
Updated Branches:
refs/heads/master 978cf3912 -> f62bbdd64
[Java] Fix the use of resource files in tests
This is preliminary work needed to run the java tests
on dist-test.
Our tests currently reference files/resources that are
not available in the output test artifacts. This means
our tests work when they are run from source in the
expected location but fail when they are run from the
compiled artifacts.
This fixes those failing tests by discovering the correct
location via Class.getResource and ensuring all needed
files are packaged in the resources directory.
Change-Id: Ia3a90bac67652b58f2eef802c509983c68d8a299
Reviewed-on: http://gerrit.cloudera.org:8080/10924
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Tested-by: Grant Henke <gr...@apache.org>
Reviewed-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fff3cf09
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fff3cf09
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fff3cf09
Branch: refs/heads/master
Commit: fff3cf0923782cab8a43e081caeccc88a53f11f9
Parents: 978cf39
Author: Grant Henke <gr...@apache.org>
Authored: Wed Jul 11 14:17:04 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Thu Jul 12 17:14:53 2018 +0000
----------------------------------------------------------------------
java/kudu-flume-sink/build.gradle | 6 ++++++
java/kudu-flume-sink/pom.xml | 3 +++
.../avro/testAvroKuduOperationsProducer.avsc | 14 -------------
.../sink/AvroKuduOperationsProducerTest.java | 22 ++++++++++----------
.../testAvroKuduOperationsProducer.avsc | 14 +++++++++++++
.../spark/tools/TestImportExportFiles.scala | 10 +++++----
6 files changed, 40 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/build.gradle
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/build.gradle b/java/kudu-flume-sink/build.gradle
index 5767103..e09f0ae 100644
--- a/java/kudu-flume-sink/build.gradle
+++ b/java/kudu-flume-sink/build.gradle
@@ -33,4 +33,10 @@ dependencies {
testCompile project(path: ":kudu-client", configuration: "shadowTest")
testCompile libs.junit
+}
+
+// Configure the Avro plugin to compile the schemas in the
+// resource directory so the schemas are available in tests.
+generateTestAvroJava {
+ source("src/test/resources")
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/pom.xml b/java/kudu-flume-sink/pom.xml
index aa8a859..7b664fb 100644
--- a/java/kudu-flume-sink/pom.xml
+++ b/java/kudu-flume-sink/pom.xml
@@ -126,6 +126,9 @@
<goal>schema</goal>
</goals>
<configuration>
+ <!-- Configure the Avro plugin to compile the schemas in the
+ resource directory so the schemas are available in tests. -->
+ <testSourceDirectory>${basedir}/src/test/resources</testSourceDirectory>
<enableDecimalLogicalType>true</enableDecimalLogicalType>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc b/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
deleted file mode 100644
index 6bcf6d2..0000000
--- a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
+++ /dev/null
@@ -1,14 +0,0 @@
-{"namespace": "org.apache.kudu.flume.sink",
- "type": "record",
- "name": "AvroKuduOperationsProducerTestRecord",
- "fields": [
- {"name": "key", "type": "int"},
- {"name": "longField", "type": "long"},
- {"name": "doubleField", "type": "double"},
- {"name": "nullableField", "type": ["string", "null"]},
- {"name": "stringField", "type": "string"},
- {"name": "decimalField", "type": {
- "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 1}
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
index 8e3c8dc..42c94cd 100644
--- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
+++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
@@ -33,8 +33,9 @@ import static org.junit.Assert.assertEquals;
import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.IOException;
import java.math.BigDecimal;
+import java.net.URL;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -51,15 +52,12 @@ import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
-import org.apache.flume.FlumeException;
import org.apache.flume.Sink;
import org.apache.flume.Transaction;
import org.apache.flume.channel.MemoryChannel;
import org.apache.flume.conf.Configurables;
import org.apache.flume.event.EventBuilder;
import org.apache.kudu.util.DecimalUtil;
-import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.kudu.ColumnSchema;
@@ -70,13 +68,17 @@ import org.apache.kudu.client.CreateTableOptions;
import org.apache.kudu.client.KuduTable;
public class AvroKuduOperationsProducerTest extends BaseKuduTest {
- private static final String schemaPath = "src/test/avro/testAvroKuduOperationsProducer.avsc";
+ private static String schemaUriString;
private static String schemaLiteral;
static {
try {
- schemaLiteral = Files.toString(new File(schemaPath), UTF_8);
- } catch (IOException e) {
+ String schemaPath = "/testAvroKuduOperationsProducer.avsc";
+ URL schemaUrl = AvroKuduOperationsProducerTest.class.getResource(schemaPath);
+ File schemaFile = Paths.get(schemaUrl.toURI()).toFile();
+ schemaUriString = schemaFile.getAbsoluteFile().toURI().toString();
+ schemaLiteral = Files.asCharSource(schemaFile, UTF_8).read();
+ } catch (Exception e) {
throw new RuntimeException(e);
}
}
@@ -115,9 +117,8 @@ public class AvroKuduOperationsProducerTest extends BaseKuduTest {
KuduTable table = createNewTable(
String.format("test%sevents%s", eventCount, schemaLocation));
String tableName = table.getName();
- String schemaURI = new File(schemaPath).getAbsoluteFile().toURI().toString();
Context ctx = schemaLocation != SchemaLocation.GLOBAL ? new Context()
- : new Context(ImmutableMap.of(PRODUCER_PREFIX + SCHEMA_PROP, schemaURI));
+ : new Context(ImmutableMap.of(PRODUCER_PREFIX + SCHEMA_PROP, schemaUriString));
KuduSink sink = createSink(tableName, ctx);
Channel channel = new MemoryChannel();
@@ -190,8 +191,7 @@ public class AvroKuduOperationsProducerTest extends BaseKuduTest {
encoder.flush();
Event e = EventBuilder.withBody(out.toByteArray());
if (schemaLocation == SchemaLocation.URL) {
- String schemaURI = new File(schemaPath).getAbsoluteFile().toURI().toString();
- e.setHeaders(ImmutableMap.of(SCHEMA_URL_HEADER, schemaURI));
+ e.setHeaders(ImmutableMap.of(SCHEMA_URL_HEADER, schemaUriString));
} else if (schemaLocation == SchemaLocation.LITERAL) {
e.setHeaders(ImmutableMap.of(SCHEMA_LITERAL_HEADER, schemaLiteral));
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc b/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc
new file mode 100644
index 0000000..6bcf6d2
--- /dev/null
+++ b/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc
@@ -0,0 +1,14 @@
+{"namespace": "org.apache.kudu.flume.sink",
+ "type": "record",
+ "name": "AvroKuduOperationsProducerTestRecord",
+ "fields": [
+ {"name": "key", "type": "int"},
+ {"name": "longField", "type": "long"},
+ {"name": "doubleField", "type": "double"},
+ {"name": "nullableField", "type": ["string", "null"]},
+ {"name": "stringField", "type": "string"},
+ {"name": "decimalField", "type": {
+ "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 1}
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
index e470ddc..ceca8d1 100644
--- a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala
@@ -17,7 +17,7 @@
package org.apache.kudu.spark.tools
-import java.io.File
+import java.nio.file.Paths
import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
import org.apache.kudu.{Schema, Type}
@@ -35,7 +35,7 @@ import scala.collection.JavaConverters._
class TestImportExportFiles extends FunSuite with TestContext with Matchers {
private val TABLE_NAME: String = "TestImportExportFiles"
- private val TABLE_DATA_PATH: String = "src/test/resources/TestImportExportFiles.csv"
+ private val TABLE_DATA_PATH: String = "/TestImportExportFiles.csv"
test("Spark Import Export") {
val schema: Schema = {
@@ -51,12 +51,14 @@ class TestImportExportFiles extends FunSuite with TestContext with Matchers {
.setNumReplicas(1)
kuduClient.createTable(TABLE_NAME, schema, tableOptions)
- val dataPath = new File(TABLE_DATA_PATH).getAbsolutePath
+ // Get the absolute path of the resource file.
+ val schemaResource = classOf[TestImportExportFiles].getResource(TABLE_DATA_PATH)
+ val dataPath = Paths.get(schemaResource.toURI).toAbsolutePath
ImportExportFiles.testMain(Array("--operation=import",
"--format=csv",
s"--master-addrs=${miniCluster.getMasterAddresses}",
- s"--path=$TABLE_DATA_PATH",
+ s"--path=$dataPath",
s"--table-name=$TABLE_NAME",
"--delimiter=,",
"--header=true",
[2/2] kudu git commit: Test for backup/restore multiple tables
Posted by gr...@apache.org.
Test for backup/restore multiple tables
This is a prerequisite for testing parallel backup/restore and error
handling.
- Extracted `createTable` method so it can be re-used
- Added `tableName` argument to `insertRows` so it can be re-used
Change-Id: I439e0d1c6904395ad382e3f059846b3b03a79af4
Reviewed-on: http://gerrit.cloudera.org:8080/10899
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Reviewed-by: Grant Henke <gr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f62bbdd6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f62bbdd6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f62bbdd6
Branch: refs/heads/master
Commit: f62bbdd64d1c039b91edbd1d55a1607abd023f43
Parents: fff3cf0
Author: Tony Foerster <to...@phdata.io>
Authored: Tue Jul 10 07:08:52 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Thu Jul 12 19:17:09 2018 +0000
----------------------------------------------------------------------
.../org/apache/kudu/backup/TestKuduBackup.scala | 28 +++++++++++++++++---
.../kudu/spark/kudu/DefaultSourceTest.scala | 2 +-
.../kudu/spark/kudu/KuduContextTest.scala | 4 +--
.../apache/kudu/spark/kudu/KuduRDDTest.scala | 2 +-
.../apache/kudu/spark/kudu/TestContext.scala | 24 ++++++++++-------
5 files changed, 42 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/f62bbdd6/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
index 2e2e589..1c166bd 100644
--- a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
+++ b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
@@ -42,7 +42,7 @@ class TestKuduBackup extends FunSuite with TestContext with Matchers {
val log: Logger = LoggerFactory.getLogger(getClass)
test("Simple Backup and Restore") {
- insertRows(100) // Insert data into the default test table.
+ insertRows(table, 100) // Insert data into the default test table.
backupAndRestore(tableName)
@@ -93,6 +93,26 @@ class TestKuduBackup extends FunSuite with TestContext with Matchers {
assertTrue(partitionSchemasMatch(tA.getPartitionSchema, tB.getPartitionSchema))
}
+ test("Backup and Restore Multiple Tables") {
+ val numRows = 1
+ val table1Name = "table1"
+ val table2Name = "table2"
+
+ val table1 = kuduClient.createTable(table1Name, schema, tableOptions)
+ val table2 = kuduClient.createTable(table2Name, schema, tableOptions)
+
+ insertRows(table1, numRows)
+ insertRows(table2, numRows)
+
+ backupAndRestore(table1Name, table2Name)
+
+ val rdd1 = kuduContext.kuduRDD(ss.sparkContext, s"$table1Name-restore", List("key"))
+ assertResult(numRows)(rdd1.count())
+
+ val rdd2 = kuduContext.kuduRDD(ss.sparkContext, s"$table2Name-restore", List("key"))
+ assertResult(numRows)(rdd2.count())
+ }
+
// TODO: Move to a Schema equals/equivalent method.
def schemasMatch(before: Schema, after: Schema): Boolean = {
if (before eq after) return true
@@ -299,14 +319,14 @@ class TestKuduBackup extends FunSuite with TestContext with Matchers {
}
}
- def backupAndRestore(tableName: String): Unit = {
+ def backupAndRestore(tableNames: String*): Unit = {
val dir = Files.createTempDirectory("backup")
val path = dir.toUri.toString
- val backupOptions = new KuduBackupOptions(Seq(tableName), path, miniCluster.getMasterAddresses)
+ val backupOptions = new KuduBackupOptions(tableNames, path, miniCluster.getMasterAddresses)
KuduBackup.run(backupOptions, ss)
- val restoreOptions = new KuduRestoreOptions(Seq(tableName), path, miniCluster.getMasterAddresses)
+ val restoreOptions = new KuduRestoreOptions(tableNames, path, miniCluster.getMasterAddresses)
KuduRestore.run(restoreOptions, ss)
FileUtils.deleteDirectory(dir.toFile)
http://git-wip-us.apache.org/repos/asf/kudu/blob/f62bbdd6/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
index 4e7d5a4..ec4d37c 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -41,7 +41,7 @@ class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfterEac
override def beforeEach(): Unit = {
super.beforeEach()
- rows = insertRows(rowCount)
+ rows = insertRows(table, rowCount)
sqlContext = ss.sqlContext
http://git-wip-us.apache.org/repos/asf/kudu/blob/f62bbdd6/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
index ce1059b..4915002 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala
@@ -60,7 +60,7 @@ class KuduContextTest extends FunSuite with TestContext with Matchers {
}
test("Test basic kuduRDD") {
- val rows = insertRows(rowCount)
+ val rows = insertRows(table, rowCount)
val scanList = kuduContext.kuduRDD(ss.sparkContext, "test", Seq("key", "c1_i", "c2_s", "c3_double",
"c4_long", "c5_bool", "c6_short", "c7_float", "c8_binary", "c9_unixtime_micros", "c10_byte",
"c11_decimal32", "c12_decimal64", "c13_decimal128"))
@@ -87,7 +87,7 @@ class KuduContextTest extends FunSuite with TestContext with Matchers {
}
test("Test kudu-spark DataFrame") {
- insertRows(rowCount)
+ insertRows(table, rowCount)
val sqlContext = ss.sqlContext
val dataDF = sqlContext.read.options(Map("kudu.master" -> miniCluster.getMasterAddresses,
"kudu.table" -> "test")).kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/f62bbdd6/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala
index fbd7bbb..56ff412 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduRDDTest.scala
@@ -25,7 +25,7 @@ import org.scalatest.{BeforeAndAfter, FunSuite}
class KuduRDDTest extends FunSuite with TestContext with BeforeAndAfter {
test("collect rows") {
- insertRows(100)
+ insertRows(table, 100)
val rdd = kuduContext.kuduRDD(ss.sparkContext, tableName, List("key"))
assert(rdd.collect.length == 100)
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/f62bbdd6/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
index 3ef0e45..a3247da 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/TestContext.scala
@@ -98,23 +98,27 @@ trait TestContext extends BeforeAndAfterEach { self: Suite =>
kuduContext = new KuduContext(miniCluster.getMasterAddresses, ss.sparkContext)
+ table = kuduClient.createTable(tableName, schema, tableOptions)
+
+
+ val simpleTableOptions = new CreateTableOptions()
+ .setRangePartitionColumns(List("key").asJava)
+ .setNumReplicas(1)
+
+ kuduClient.createTable(simpleTableName, simpleSchema, simpleTableOptions)
+ }
+
+ val tableOptions: CreateTableOptions = {
val bottom = schema.newPartialRow() // Unbounded.
val middle = schema.newPartialRow()
middle.addInt("key", 50)
val top = schema.newPartialRow() // Unbounded.
- val tableOptions = new CreateTableOptions()
+ new CreateTableOptions()
.setRangePartitionColumns(List("key").asJava)
.addRangePartition(bottom, middle)
.addRangePartition(middle, top)
.setNumReplicas(1)
- table = kuduClient.createTable(tableName, schema, tableOptions)
-
- val simpleTableOptions = new CreateTableOptions()
- .setRangePartitionColumns(List("key").asJava)
- .setNumReplicas(1)
-
- kuduClient.createTable(simpleTableName, simpleSchema, simpleTableOptions)
}
override def afterEach() {
@@ -130,11 +134,11 @@ trait TestContext extends BeforeAndAfterEach { self: Suite =>
kuduSession.apply(delete)
}
- def insertRows(rowCount: Integer): IndexedSeq[(Int, Int, String, Long)] = {
+ def insertRows(targetTable: KuduTable, rowCount: Integer): IndexedSeq[(Int, Int, String, Long)] = {
val kuduSession = kuduClient.newSession()
val rows = Range(0, rowCount).map { i =>
- val insert = table.newInsert
+ val insert = targetTable.newInsert
val row = insert.getRow
row.addInt(0, i)
row.addInt(1, i)