You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/07/30 20:32:30 UTC
[3/3] kudu git commit: Scala code formatting with Scalafmt
Scala code formatting with Scalafmt
Scalafmt https://scalameta.org/scalafmt/#Scalafmt is a code formatting
tool, scalafmt.conf is the main configuration file.
Scalafmt is added using the Gradle Scalafmt plugin:
https://github.com/alenkacz/gradle-scalafmt and Maven scalafmt plugin:
https://github.com/SimonJPegg/mvn_scalafmt. The plugin is configured to run on
compile or can invoked with 'gradle scalafmt' or 'gradle testScalafmt' to
format test code. It will run during the verification stage of a Maven build.
Set gradle endoding globally because 'testScalafmt' target would otherwise
choke on some special characters.
Change-Id: Iac96383d88394084e19712177d05f9fc63de766c
Reviewed-on: http://gerrit.cloudera.org:8080/11030
Tested-by: Kudu Jenkins
Reviewed-by: Tony Foerster <an...@gmail.com>
Reviewed-by: Grant Henke <gr...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/79166955
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/79166955
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/79166955
Branch: refs/heads/master
Commit: 79166955f96a50a3ba76afe95d1b52d1b866053e
Parents: 349b810
Author: Tony Foerster <to...@phdata.io>
Authored: Thu Jul 26 10:44:12 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Mon Jul 30 20:31:55 2018 +0000
----------------------------------------------------------------------
java/.gitignore | 1 +
java/.scalafmt.conf | 10 +
java/buildSrc/build.gradle | 1 +
java/gradle.properties | 2 +
java/gradle/quality.gradle | 12 +
java/kudu-backup/pom.xml | 7 +
.../org/apache/kudu/backup/KuduBackup.scala | 21 +-
.../apache/kudu/backup/KuduBackupOptions.scala | 115 ++---
.../org/apache/kudu/backup/KuduBackupRDD.scala | 80 ++--
.../org/apache/kudu/backup/KuduRestore.scala | 32 +-
.../apache/kudu/backup/KuduRestoreOptions.scala | 91 ++--
.../org/apache/kudu/backup/TableMetadata.scala | 137 ++++--
java/kudu-spark-tools/pom.xml | 7 +
.../kudu/spark/tools/ImportExportFiles.scala | 51 ++-
.../tools/IntegrationTestBigLinkedList.scala | 193 +++++----
java/kudu-spark/pom.xml | 7 +
.../apache/kudu/spark/kudu/DefaultSource.scala | 420 ++++++++++--------
.../apache/kudu/spark/kudu/KuduContext.scala | 421 +++++++++++--------
.../org/apache/kudu/spark/kudu/KuduRDD.scala | 113 ++---
.../kudu/spark/kudu/KuduWriteOptions.scala | 23 +-
.../apache/kudu/spark/kudu/OperationType.scala | 4 +-
.../org/apache/kudu/spark/kudu/SparkUtil.scala | 109 ++---
.../org/apache/kudu/spark/kudu/package.scala | 8 +-
java/pom.xml | 19 +
24 files changed, 1117 insertions(+), 767 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/.gitignore
----------------------------------------------------------------------
diff --git a/java/.gitignore b/java/.gitignore
index 865e93d..d09c9c6 100644
--- a/java/.gitignore
+++ b/java/.gitignore
@@ -36,3 +36,4 @@ gradle/wrapper/*.jar
*.iml
.idea/
classes
+**/out
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/.scalafmt.conf
----------------------------------------------------------------------
diff --git a/java/.scalafmt.conf b/java/.scalafmt.conf
new file mode 100644
index 0000000..7247dc9
--- /dev/null
+++ b/java/.scalafmt.conf
@@ -0,0 +1,10 @@
+style = defaultWithAlign
+rewrite.rules = [prefercurlyfors, AvoidInfix]
+align = false
+docstrings=JavaDoc
+maxColumn=80
+spaces.inImportCurlyBraces=false
+unindentTopLevelOperators =true
+newlines.alwaysBeforeTopLevelStatements=true
+newlines.penalizeSingleSelectMultiArgList=false
+lineEndings=unix
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/buildSrc/build.gradle
----------------------------------------------------------------------
diff --git a/java/buildSrc/build.gradle b/java/buildSrc/build.gradle
index 0281f89..c0ecf8f 100644
--- a/java/buildSrc/build.gradle
+++ b/java/buildSrc/build.gradle
@@ -38,4 +38,5 @@ dependencies {
compile "net.ltgt.gradle:gradle-errorprone-plugin:0.0.14"
compile "ru.vyarus:gradle-animalsniffer-plugin:1.4.3"
compile 'com.google.code.gson:gson:2.8.5'
+ compile "cz.alenkacz:gradle-scalafmt:1.6.0"
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/gradle.properties
----------------------------------------------------------------------
diff --git a/java/gradle.properties b/java/gradle.properties
index 32f4dd2..64a26bf 100755
--- a/java/gradle.properties
+++ b/java/gradle.properties
@@ -58,3 +58,5 @@ org.gradle.daemon = true
# org.gradle.configureondemand = true
# org.gradle.parallel = true
# org.gradle.workers.max = 4
+
+systemProp.file.encoding = UTF-8
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/gradle/quality.gradle
----------------------------------------------------------------------
diff --git a/java/gradle/quality.gradle b/java/gradle/quality.gradle
index fae5ed7..0527726 100644
--- a/java/gradle/quality.gradle
+++ b/java/gradle/quality.gradle
@@ -22,6 +22,7 @@ apply plugin: "com.github.spotbugs" // Performs static code analysis to look f
apply plugin: "pmd" // Performs static code analysis to look for common code smells in Java code.
apply plugin: "com.github.ben-manes.versions" // Provides a task to determine which dependencies have updates.
apply plugin: "ru.vyarus.animalsniffer" // Ensures Java code uses APIs from a particular version of Java.
+apply plugin: "scalafmt"
checkstyle {
configFile = file("$rootDir/kudu_style.xml")
@@ -62,6 +63,17 @@ pmd {
ignoreFailures = true
}
+scalafmt {
+ configFilePath = "$rootDir/.scalafmt.conf"
+}
+
+// Run scalafmt on compile
+tasks.withType(ScalaCompile) {
+ if (!propertyExists("skipFormat")) {
+ dependsOn("scalafmtAll")
+ }
+}
+
// Create an aggregate pmd task.
// This simplifies running pmd on all the code by only needing one task instead of multiple in your command.
task pmd(dependsOn: [pmdMain, pmdTest, pmdIntegrationTest], group: "Verification") {
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-backup/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-backup/pom.xml b/java/kudu-backup/pom.xml
index 37aefc4..4200b5f 100644
--- a/java/kudu-backup/pom.xml
+++ b/java/kudu-backup/pom.xml
@@ -149,6 +149,9 @@
</dependencies>
<build>
+ <!-- set source dirs explicitly for the scalafmt plugin -->
+ <sourceDirectory>src/main/scala</sourceDirectory>
+ <testSourceDirectory>src/test/scala</testSourceDirectory>
<extensions>
<!-- Used in the protobuf plugin to find the right protoc artifact
with the property os.detected.classifier -->
@@ -223,6 +226,10 @@
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.antipathy</groupId>
+ <artifactId>mvn-scalafmt</artifactId>
+ </plugin>
</plugins>
<!-- This big ol' block of nonsense tells the m2e Eclipse plugin what
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala
index 1fa2899..db16055 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala
@@ -35,7 +35,8 @@ object KuduBackup {
val log: Logger = LoggerFactory.getLogger(getClass)
def run(options: KuduBackupOptions, session: SparkSession): Unit = {
- val context = new KuduContext(options.kuduMasterAddresses, session.sparkContext)
+ val context =
+ new KuduContext(options.kuduMasterAddresses, session.sparkContext)
val path = options.path
log.info(s"Backing up to path: $path")
@@ -45,7 +46,8 @@ object KuduBackup {
val tablePath = Paths.get(path).resolve(URLEncoder.encode(t, "UTF-8"))
val rdd = new KuduBackupRDD(table, options, context, session.sparkContext)
- val df = session.sqlContext.createDataFrame(rdd, sparkSchema(table.getSchema))
+ val df =
+ session.sqlContext.createDataFrame(rdd, sparkSchema(table.getSchema))
// TODO: Prefix path with the time? Maybe a backup "name" parameter defaulted to something?
// TODO: Take parameter for the SaveMode.
val writer = df.write.mode(SaveMode.ErrorIfExists)
@@ -59,7 +61,10 @@ object KuduBackup {
}
}
- private def writeTableMetadata(metadata: TableMetadataPB, path: Path, session: SparkSession): Unit = {
+ private def writeTableMetadata(
+ metadata: TableMetadataPB,
+ path: Path,
+ session: SparkSession): Unit = {
val conf = session.sparkContext.hadoopConfiguration
val hPath = new HPath(path.resolve(TableMetadata.MetadataFileName).toString)
val fs = hPath.getFileSystem(conf)
@@ -71,14 +76,16 @@ object KuduBackup {
}
def main(args: Array[String]): Unit = {
- val options = KuduBackupOptions.parse(args)
- .getOrElse(throw new IllegalArgumentException("could not parse the arguments"))
+ val options = KuduBackupOptions
+ .parse(args)
+ .getOrElse(
+ throw new IllegalArgumentException("could not parse the arguments"))
- val session = SparkSession.builder()
+ val session = SparkSession
+ .builder()
.appName("Kudu Table Backup")
.getOrCreate()
run(options, session)
}
}
-
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
index 04053cb..4d343ed 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
@@ -24,76 +24,81 @@ import scopt.OptionParser
@InterfaceAudience.Private
@InterfaceStability.Unstable
-case class KuduBackupOptions(tables: Seq[String],
- path: String,
- kuduMasterAddresses: String = InetAddress.getLocalHost.getCanonicalHostName,
- timestampMs: Long = System.currentTimeMillis(),
- format: String = KuduBackupOptions.DefaultFormat,
- scanBatchSize: Int = KuduBackupOptions.DefaultScanBatchSize,
- scanRequestTimeout: Long = KuduBackupOptions.DefaultScanRequestTimeout,
- scanPrefetching: Boolean = KuduBackupOptions.DefaultScanPrefetching)
+case class KuduBackupOptions(
+ tables: Seq[String],
+ path: String,
+ kuduMasterAddresses: String = InetAddress.getLocalHost.getCanonicalHostName,
+ timestampMs: Long = System.currentTimeMillis(),
+ format: String = KuduBackupOptions.DefaultFormat,
+ scanBatchSize: Int = KuduBackupOptions.DefaultScanBatchSize,
+ scanRequestTimeout: Long = KuduBackupOptions.DefaultScanRequestTimeout,
+ scanPrefetching: Boolean = KuduBackupOptions.DefaultScanPrefetching)
object KuduBackupOptions {
val DefaultFormat: String = "parquet"
- val DefaultScanBatchSize: Int = 1024*1024*20 // 20 MiB
- val DefaultScanRequestTimeout: Long = AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS // 30 seconds
- val DefaultScanPrefetching: Boolean = false // TODO: Add a test per KUDU-1260 and enable by default?
+ val DefaultScanBatchSize: Int = 1024 * 1024 * 20 // 20 MiB
+ val DefaultScanRequestTimeout: Long =
+ AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS // 30 seconds
+ val DefaultScanPrefetching
+ : Boolean = false // TODO: Add a test per KUDU-1260 and enable by default?
// TODO: clean up usage output.
// TODO: timeout configurations.
- private val parser: OptionParser[KuduBackupOptions] = new OptionParser[KuduBackupOptions]("KuduBackup") {
- opt[String]("path")
- .action((v, o) => o.copy(path = v))
- .text("The root path to output backup data. Accepts any Spark compatible path.")
- .optional()
+ private val parser: OptionParser[KuduBackupOptions] =
+ new OptionParser[KuduBackupOptions]("KuduBackup") {
+ opt[String]("path")
+ .action((v, o) => o.copy(path = v))
+ .text("The root path to output backup data. Accepts any Spark compatible path.")
+ .optional()
- opt[String]("kuduMasterAddresses")
- .action((v, o) => o.copy(kuduMasterAddresses = v))
- .text("Comma-separated addresses of Kudu masters.")
- .optional()
+ opt[String]("kuduMasterAddresses")
+ .action((v, o) => o.copy(kuduMasterAddresses = v))
+ .text("Comma-separated addresses of Kudu masters.")
+ .optional()
- opt[Long]("timestampMs")
- .action((v, o) => o.copy(timestampMs = v))
- // TODO: Document the limitations based on cluster configuration (ex: ancient history watermark).
- .text("A UNIX timestamp in milliseconds since the epoch to execute scans at.")
- .optional()
+ opt[Long]("timestampMs")
+ .action((v, o) => o.copy(timestampMs = v))
+ // TODO: Document the limitations based on cluster configuration (ex: ancient history watermark).
+ .text("A UNIX timestamp in milliseconds since the epoch to execute scans at.")
+ .optional()
- opt[String]("format")
- .action((v, o) => o.copy(format = v))
- .text("The file format to use when writing the data.")
- .optional()
+ opt[String]("format")
+ .action((v, o) => o.copy(format = v))
+ .text("The file format to use when writing the data.")
+ .optional()
- opt[Int]("scanBatchSize")
- .action((v, o) => o.copy(scanBatchSize = v))
- .text("The maximum number of bytes returned by the scanner, on each batch.")
- .optional()
+ opt[Int]("scanBatchSize")
+ .action((v, o) => o.copy(scanBatchSize = v))
+ .text(
+ "The maximum number of bytes returned by the scanner, on each batch.")
+ .optional()
- opt[Int]("scanRequestTimeout")
- .action((v, o) => o.copy(scanRequestTimeout = v))
- .text("Sets how long each scan request to a server can last.")
- .optional()
+ opt[Int]("scanRequestTimeout")
+ .action((v, o) => o.copy(scanRequestTimeout = v))
+ .text("Sets how long each scan request to a server can last.")
+ .optional()
- opt[Unit]("scanPrefetching")
- .action( (_, o) => o.copy(scanPrefetching = true) )
- .text("An experimental flag to enable pre-fetching data.")
- .optional()
+ opt[Unit]("scanPrefetching")
+ .action((_, o) => o.copy(scanPrefetching = true))
+ .text("An experimental flag to enable pre-fetching data.")
+ .optional()
- arg[String]("<table>...")
- .unbounded()
- .action( (v, o) => o.copy(tables = o.tables :+ v) )
- .text("A list of tables to be backed up.")
- }
+ arg[String]("<table>...")
+ .unbounded()
+ .action((v, o) => o.copy(tables = o.tables :+ v))
+ .text("A list of tables to be backed up.")
+ }
/**
- * Parses the passed arguments into Some[KuduBackupOptions].
- *
- * If the arguments are bad, an error message is displayed
- * and None is returned.
- *
- * @param args The arguments to parse.
- * @return Some[KuduBackupOptions] if parsing was successful, None if not.
- */
+ * Parses the passed arguments into Some[KuduBackupOptions].
+ *
+ * If the arguments are bad, an error message is displayed
+ * and None is returned.
+ *
+ * @param args The arguments to parse.
+ * @return Some[KuduBackupOptions] if parsing was successful, None if not.
+ */
def parse(args: Seq[String]): Option[KuduBackupOptions] = {
parser.parse(args, KuduBackupOptions(Seq(), null))
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
index e0acdaa..d116d35 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
@@ -32,11 +32,12 @@ import scala.collection.JavaConverters._
@InterfaceAudience.Private
@InterfaceStability.Unstable
-class KuduBackupRDD private[kudu](@transient val table: KuduTable,
- @transient val options: KuduBackupOptions,
- val kuduContext: KuduContext,
- @transient val sc: SparkContext
- ) extends RDD[Row](sc, Nil) {
+class KuduBackupRDD private[kudu] (
+ @transient val table: KuduTable,
+ @transient val options: KuduBackupOptions,
+ val kuduContext: KuduContext,
+ @transient val sc: SparkContext)
+ extends RDD[Row](sc, Nil) {
// TODO: Split large tablets into smaller scan tokens?
override protected def getPartitions: Array[Partition] = {
@@ -44,10 +45,12 @@ class KuduBackupRDD private[kudu](@transient val table: KuduTable,
// Set a hybrid time for the scan to ensure application consistency.
val timestampMicros = TimeUnit.MILLISECONDS.toMicros(options.timestampMs)
- val hybridTime = HybridTimeUtil.physicalAndLogicalToHTTimestamp(timestampMicros, 0)
+ val hybridTime =
+ HybridTimeUtil.physicalAndLogicalToHTTimestamp(timestampMicros, 0)
// Create the scan tokens for each partition.
- val tokens = client.newScanTokenBuilder(table)
+ val tokens = client
+ .newScanTokenBuilder(table)
.cacheBlocks(false)
// TODO: Use fault tolerant scans to get mostly.
// ordered results when KUDU-2466 is fixed.
@@ -72,11 +75,14 @@ class KuduBackupRDD private[kudu](@transient val table: KuduTable,
// TODO: Do we need a custom spark partitioner for any guarantees?
// override val partitioner = None
- override def compute(part: Partition, taskContext: TaskContext): Iterator[Row] = {
+ override def compute(
+ part: Partition,
+ taskContext: TaskContext): Iterator[Row] = {
val client: KuduClient = kuduContext.syncClient
val partition: KuduBackupPartition = part.asInstanceOf[KuduBackupPartition]
// TODO: Get deletes and updates for incremental backups.
- val scanner = KuduScanToken.deserializeIntoScanner(partition.scanToken, client)
+ val scanner =
+ KuduScanToken.deserializeIntoScanner(partition.scanToken, client)
new RowIterator(scanner)
}
@@ -85,19 +91,22 @@ class KuduBackupRDD private[kudu](@transient val table: KuduTable,
}
}
-private case class KuduBackupPartition(index: Int,
- scanToken: Array[Byte],
- locations: Array[String]) extends Partition
+private case class KuduBackupPartition(
+ index: Int,
+ scanToken: Array[Byte],
+ locations: Array[String])
+ extends Partition
/**
- * This iterator wraps a KuduScanner, converts the returned RowResults into a
- * Spark Row, and allows iterating over those scanned results.
- *
- * The Spark RDD abstraction has an abstract compute method, implemented in KuduBackupRDD,
- * that takes the job partitions and task context and expects to return an Iterator[Row].
- * This implementation facilitates that.
- */
-private class RowIterator(private val scanner: KuduScanner) extends Iterator[Row] {
+ * This iterator wraps a KuduScanner, converts the returned RowResults into a
+ * Spark Row, and allows iterating over those scanned results.
+ *
+ * The Spark RDD abstraction has an abstract compute method, implemented in KuduBackupRDD,
+ * that takes the job partitions and task context and expects to return an Iterator[Row].
+ * This implementation facilitates that.
+ */
+private class RowIterator(private val scanner: KuduScanner)
+ extends Iterator[Row] {
private var currentIterator: RowResultIterator = _
@@ -115,20 +124,23 @@ private class RowIterator(private val scanner: KuduScanner) extends Iterator[Row
// TODO: Use a more "raw" encoding for efficiency?
private def get(rowResult: RowResult, i: Int): Any = {
if (rowResult.isNull(i)) null
- else rowResult.getColumnType(i) match {
- case Type.BOOL => rowResult.getBoolean(i)
- case Type.INT8 => rowResult.getByte(i)
- case Type.INT16 => rowResult.getShort(i)
- case Type.INT32 => rowResult.getInt(i)
- case Type.INT64 => rowResult.getLong(i)
- case Type.UNIXTIME_MICROS => rowResult.getTimestamp(i)
- case Type.FLOAT => rowResult.getFloat(i)
- case Type.DOUBLE => rowResult.getDouble(i)
- case Type.STRING => rowResult.getString(i)
- case Type.BINARY => rowResult.getBinaryCopy(i)
- case Type.DECIMAL => rowResult.getDecimal(i)
- case _ => throw new RuntimeException(s"Unsupported column type: ${rowResult.getColumnType(i)}")
- }
+ else
+ rowResult.getColumnType(i) match {
+ case Type.BOOL => rowResult.getBoolean(i)
+ case Type.INT8 => rowResult.getByte(i)
+ case Type.INT16 => rowResult.getShort(i)
+ case Type.INT32 => rowResult.getInt(i)
+ case Type.INT64 => rowResult.getLong(i)
+ case Type.UNIXTIME_MICROS => rowResult.getTimestamp(i)
+ case Type.FLOAT => rowResult.getFloat(i)
+ case Type.DOUBLE => rowResult.getDouble(i)
+ case Type.STRING => rowResult.getString(i)
+ case Type.BINARY => rowResult.getBinaryCopy(i)
+ case Type.DECIMAL => rowResult.getDecimal(i)
+ case _ =>
+ throw new RuntimeException(
+ s"Unsupported column type: ${rowResult.getColumnType(i)}")
+ }
}
// TODO: There may be an old KuduRDD implementation where we did some
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
index 74de830..8555215 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
@@ -36,7 +36,8 @@ object KuduRestore {
val log: Logger = LoggerFactory.getLogger(getClass)
def run(options: KuduRestoreOptions, session: SparkSession): Unit = {
- val context = new KuduContext(options.kuduMasterAddresses, session.sparkContext)
+ val context =
+ new KuduContext(options.kuduMasterAddresses, session.sparkContext)
val path = options.path
log.info(s"Restoring from path: $path")
@@ -58,20 +59,29 @@ object KuduRestore {
}
// TODO: Restrict format option.
- val df = session.sqlContext.read.format(metadata.getDataFormat).load(tablePath.toString)
- val writeOptions = new KuduWriteOptions(ignoreDuplicateRowErrors = false, ignoreNull = false)
+ val df = session.sqlContext.read
+ .format(metadata.getDataFormat)
+ .load(tablePath.toString)
+ val writeOptions = new KuduWriteOptions(
+ ignoreDuplicateRowErrors = false,
+ ignoreNull = false)
// TODO: Use client directly for more control?
// (session timeout, consistency mode, flush interval, mutation buffer space)
context.insertRows(df, restoreName, writeOptions)
}
}
- private def getMetadataPath(tableName: String, options: KuduRestoreOptions): Path = {
- val rootPath = if (options.metadataPath.isEmpty) options.path else options.metadataPath
+ private def getMetadataPath(
+ tableName: String,
+ options: KuduRestoreOptions): Path = {
+ val rootPath =
+ if (options.metadataPath.isEmpty) options.path else options.metadataPath
Paths.get(rootPath).resolve(tableName)
}
- private def readTableMetadata(path: Path, session: SparkSession): TableMetadataPB = {
+ private def readTableMetadata(
+ path: Path,
+ session: SparkSession): TableMetadataPB = {
val conf = session.sparkContext.hadoopConfiguration
val hPath = new HPath(path.resolve(TableMetadata.MetadataFileName).toString)
val fs = hPath.getFileSystem(conf)
@@ -84,14 +94,16 @@ object KuduRestore {
}
def main(args: Array[String]): Unit = {
- val options = KuduRestoreOptions.parse(args)
- .getOrElse(throw new IllegalArgumentException("could not parse the arguments"))
+ val options = KuduRestoreOptions
+ .parse(args)
+ .getOrElse(
+ throw new IllegalArgumentException("could not parse the arguments"))
- val session = SparkSession.builder()
+ val session = SparkSession
+ .builder()
.appName("Kudu Table Restore")
.getOrCreate()
run(options, session)
}
}
-
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestoreOptions.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestoreOptions.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestoreOptions.scala
index 2cd1d4d..418c365 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestoreOptions.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestoreOptions.scala
@@ -23,12 +23,13 @@ import scopt.OptionParser
@InterfaceAudience.Private
@InterfaceStability.Unstable
-case class KuduRestoreOptions(tables: Seq[String],
- path: String,
- kuduMasterAddresses: String = InetAddress.getLocalHost.getCanonicalHostName,
- tableSuffix: String = KuduRestoreOptions.DefaultTableSuffix,
- createTables: Boolean = KuduRestoreOptions.DefaultCreateTables,
- metadataPath: String = "")
+case class KuduRestoreOptions(
+ tables: Seq[String],
+ path: String,
+ kuduMasterAddresses: String = InetAddress.getLocalHost.getCanonicalHostName,
+ tableSuffix: String = KuduRestoreOptions.DefaultTableSuffix,
+ createTables: Boolean = KuduRestoreOptions.DefaultCreateTables,
+ metadataPath: String = "")
object KuduRestoreOptions {
val DefaultTableSuffix: String = "-restore"
@@ -36,50 +37,54 @@ object KuduRestoreOptions {
// TODO: clean up usage output.
// TODO: timeout configurations.
- private val parser: OptionParser[KuduRestoreOptions] = new OptionParser[KuduRestoreOptions]("KuduRestore") {
- opt[String]("path")
- .action((v, o) => o.copy(path = v))
- .text("The root path to the backup data. Accepts any Spark compatible path.")
- .optional()
+ private val parser: OptionParser[KuduRestoreOptions] =
+ new OptionParser[KuduRestoreOptions]("KuduRestore") {
+ opt[String]("path")
+ .action((v, o) => o.copy(path = v))
+ .text(
+ "The root path to the backup data. Accepts any Spark compatible path.")
+ .optional()
- opt[String]("kuduMasterAddresses")
- .action((v, o) => o.copy(kuduMasterAddresses = v))
- .text("Comma-separated addresses of Kudu masters.")
- .optional()
+ opt[String]("kuduMasterAddresses")
+ .action((v, o) => o.copy(kuduMasterAddresses = v))
+ .text("Comma-separated addresses of Kudu masters.")
+ .optional()
- opt[Boolean]("createTables")
- .action( (v, o) => o.copy(createTables = v) )
- .text("true to create tables during restore, false if they already exist.")
- .optional()
+ opt[Boolean]("createTables")
+ .action((v, o) => o.copy(createTables = v))
+ .text(
+ "true to create tables during restore, false if they already exist.")
+ .optional()
- opt[String]("tableSuffix")
- .action((v, o) => o.copy(tableSuffix = v))
- .text("The suffix to add to the restored table names. Only used when createTables is true.")
- .optional()
+ opt[String]("tableSuffix")
+ .action((v, o) => o.copy(tableSuffix = v))
+ .text("The suffix to add to the restored table names. Only used when createTables is true.")
+ .optional()
- opt[String]("metadataPath")
- .action((v, o) => o.copy(metadataPath = v))
- .text("The root path to look for table metadata. This can be used to change the properties of " +
- "the tables created during restore. By default the backup path is used. " +
- "Only used when createTables is true.")
- .optional()
+ opt[String]("metadataPath")
+ .action((v, o) => o.copy(metadataPath = v))
+ .text(
+ "The root path to look for table metadata. This can be used to change the properties of " +
+ "the tables created during restore. By default the backup path is used. " +
+ "Only used when createTables is true.")
+ .optional()
- arg[String]("<table>...")
- .unbounded()
- .action( (v, o) => o.copy(tables = o.tables :+ v) )
- .text("A list of tables to be restored.")
- }
+ arg[String]("<table>...")
+ .unbounded()
+ .action((v, o) => o.copy(tables = o.tables :+ v))
+ .text("A list of tables to be restored.")
+ }
/**
- * Parses the passed arguments into Some[KuduRestoreOptions].
- *
- * If the arguments are bad, an error message is displayed
- * and None is returned.
- *
- * @param args The arguments to parse.
- * @return Some[KuduRestoreOptions] if parsing was successful, None if not.
- */
+ * Parses the passed arguments into Some[KuduRestoreOptions].
+ *
+ * If the arguments are bad, an error message is displayed
+ * and None is returned.
+ *
+ * @param args The arguments to parse.
+ * @return Some[KuduRestoreOptions] if parsing was successful, None if not.
+ */
def parse(args: Seq[String]): Option[KuduRestoreOptions] = {
parser.parse(args, KuduRestoreOptions(Seq(), null))
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
index 9574943..68e6f49 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
@@ -21,7 +21,11 @@ import java.math.BigDecimal
import com.google.protobuf.StringValue
import org.apache.commons.net.util.Base64
import org.apache.kudu.backup.Backup._
-import org.apache.kudu.ColumnSchema.{ColumnSchemaBuilder, CompressionAlgorithm, Encoding}
+import org.apache.kudu.ColumnSchema.{
+ ColumnSchemaBuilder,
+ CompressionAlgorithm,
+ Encoding
+}
import org.apache.kudu.ColumnTypeAttributes.ColumnTypeAttributesBuilder
import org.apache.kudu.client.{Bytes, CreateTableOptions, KuduTable, PartialRow}
import org.apache.kudu.{ColumnSchema, Schema, Type}
@@ -35,9 +39,12 @@ object TableMetadata {
val MetadataFileName = ".kudu-metadata.json"
- def getTableMetadata(table: KuduTable, options: KuduBackupOptions): TableMetadataPB = {
+ def getTableMetadata(
+ table: KuduTable,
+ options: KuduBackupOptions): TableMetadataPB = {
val columns = table.getSchema.getColumns.asScala.map { col =>
- val builder = ColumnMetadataPB.newBuilder()
+ val builder = ColumnMetadataPB
+ .newBuilder()
.setName(col.getName)
.setType(col.getType.name())
.setIsKey(col.isKey)
@@ -49,12 +56,14 @@ object TableMetadata {
builder.setTypeAttributes(getTypeAttributesMetadata(col))
}
if (col.getDefaultValue != null) {
- builder.setDefaultValue(StringValue.of(valueToString(col.getDefaultValue, col.getType)))
+ builder.setDefaultValue(
+ StringValue.of(valueToString(col.getDefaultValue, col.getType)))
}
builder.build()
}
- TableMetadataPB.newBuilder()
+ TableMetadataPB
+ .newBuilder()
.setFromMs(0) // TODO: fromMs is always zero until we support incremental backups
.setToMs(options.timestampMs)
.setDataFormat(options.format)
@@ -65,9 +74,11 @@ object TableMetadata {
.build()
}
- private def getTypeAttributesMetadata(col: ColumnSchema): ColumnTypeAttributesMetadataPB = {
+ private def getTypeAttributesMetadata(
+ col: ColumnSchema): ColumnTypeAttributesMetadataPB = {
val attributes = col.getTypeAttributes
- ColumnTypeAttributesMetadataPB.newBuilder()
+ ColumnTypeAttributesMetadataPB
+ .newBuilder()
.setPrecision(attributes.getPrecision)
.setScale(attributes.getScale)
.build()
@@ -76,20 +87,23 @@ object TableMetadata {
private def getPartitionMetadata(table: KuduTable): PartitionMetadataPB = {
val hashPartitions = getHashPartitionsMetadata(table)
val rangePartitions = getRangePartitionMetadata(table)
- PartitionMetadataPB.newBuilder()
+ PartitionMetadataPB
+ .newBuilder()
.addAllHashPartitions(hashPartitions.asJava)
.setRangePartitions(rangePartitions)
.build()
}
- private def getHashPartitionsMetadata(table: KuduTable): Seq[HashPartitionMetadataPB] = {
+ private def getHashPartitionsMetadata(
+ table: KuduTable): Seq[HashPartitionMetadataPB] = {
val tableSchema = table.getSchema
val partitionSchema = table.getPartitionSchema
partitionSchema.getHashBucketSchemas.asScala.map { hs =>
val columnNames = hs.getColumnIds.asScala.map { id =>
getColumnById(tableSchema, id).getName
}
- HashPartitionMetadataPB.newBuilder()
+ HashPartitionMetadataPB
+ .newBuilder()
.addAllColumnNames(columnNames.asJava)
.setNumBuckets(hs.getNumBuckets)
.setSeed(hs.getSeed)
@@ -97,23 +111,35 @@ object TableMetadata {
}
}
- private def getRangePartitionMetadata(table: KuduTable): RangePartitionMetadataPB = {
+ private def getRangePartitionMetadata(
+ table: KuduTable): RangePartitionMetadataPB = {
val tableSchema = table.getSchema
val partitionSchema = table.getPartitionSchema
- val columnNames = partitionSchema.getRangeSchema.getColumnIds.asScala.map { id =>
- getColumnById(tableSchema, id).getName
+ val columnNames = partitionSchema.getRangeSchema.getColumnIds.asScala.map {
+ id =>
+ getColumnById(tableSchema, id).getName
}
- val bounds = table.getRangePartitions(table.getAsyncClient.getDefaultOperationTimeoutMs)
- .asScala.map { p =>
- val lowerValues = getBoundValues(p.getDecodedRangeKeyStart(table), columnNames, tableSchema)
- val upperValues = getBoundValues(p.getDecodedRangeKeyEnd(table), columnNames, tableSchema)
- RangeBoundsMetadataPB.newBuilder()
+ val bounds = table
+ .getRangePartitions(table.getAsyncClient.getDefaultOperationTimeoutMs)
+ .asScala
+ .map { p =>
+ val lowerValues = getBoundValues(
+ p.getDecodedRangeKeyStart(table),
+ columnNames,
+ tableSchema)
+ val upperValues = getBoundValues(
+ p.getDecodedRangeKeyEnd(table),
+ columnNames,
+ tableSchema)
+ RangeBoundsMetadataPB
+ .newBuilder()
.addAllUpperBounds(upperValues.asJava)
.addAllLowerBounds(lowerValues.asJava)
.build()
}
- RangePartitionMetadataPB.newBuilder()
+ RangePartitionMetadataPB
+ .newBuilder()
.addAllColumnNames(columnNames.asJava)
.addAllBounds(bounds.asJava)
.build()
@@ -123,22 +149,32 @@ object TableMetadata {
schema.getColumnByIndex(schema.getColumnIndex(colId))
}
- private def getBoundValues(bound: PartialRow, columnNames: Seq[String], schema: Schema): Seq[ColumnValueMetadataPB] = {
+ private def getBoundValues(
+ bound: PartialRow,
+ columnNames: Seq[String],
+ schema: Schema): Seq[ColumnValueMetadataPB] = {
columnNames.filter(bound.isSet).map { col =>
val colType = schema.getColumn(col).getType
val value = getValue(bound, col, colType)
- ColumnValueMetadataPB.newBuilder()
+ ColumnValueMetadataPB
+ .newBuilder()
.setColumnName(col)
.setValue(valueToString(value, colType))
.build()
}
}
- private def getPartialRow(values: Seq[ColumnValueMetadataPB], schema: Schema): PartialRow = {
+ private def getPartialRow(
+ values: Seq[ColumnValueMetadataPB],
+ schema: Schema): PartialRow = {
val row = schema.newPartialRow()
values.foreach { v =>
- val colType = schema.getColumn(v.getColumnName).getType
- addValue(valueFromString(v.getValue, colType), row, v.getColumnName, colType)
+ val colType = schema.getColumn(v.getColumnName).getType
+ addValue(
+ valueFromString(v.getValue, colType),
+ row,
+ v.getColumnName,
+ colType)
}
row
}
@@ -172,7 +208,10 @@ object TableMetadata {
new Schema(columns.asJava)
}
- private def getValue(row: PartialRow, columnName: String, colType: Type): Any = {
+ private def getValue(
+ row: PartialRow,
+ columnName: String,
+ colType: Type): Any = {
colType match {
case Type.BOOL => row.getBoolean(columnName)
case Type.INT8 => row.getByte(columnName)
@@ -184,23 +223,32 @@ object TableMetadata {
case Type.STRING => row.getString(columnName)
case Type.BINARY => row.getBinary(columnName)
case Type.DECIMAL => row.getDecimal(columnName)
- case _ => throw new IllegalArgumentException(s"Unsupported column type: $colType")
+ case _ =>
+ throw new IllegalArgumentException(s"Unsupported column type: $colType")
}
}
- private def addValue(value: Any, row: PartialRow, columnName: String, colType: Type): Any = {
+ private def addValue(
+ value: Any,
+ row: PartialRow,
+ columnName: String,
+ colType: Type): Any = {
colType match {
case Type.BOOL => row.addBoolean(columnName, value.asInstanceOf[Boolean])
case Type.INT8 => row.addByte(columnName, value.asInstanceOf[Byte])
case Type.INT16 => row.addShort(columnName, value.asInstanceOf[Short])
case Type.INT32 => row.addInt(columnName, value.asInstanceOf[Int])
- case Type.INT64 | Type.UNIXTIME_MICROS => row.addLong(columnName, value.asInstanceOf[Long])
+ case Type.INT64 | Type.UNIXTIME_MICROS =>
+ row.addLong(columnName, value.asInstanceOf[Long])
case Type.FLOAT => row.addFloat(columnName, value.asInstanceOf[Float])
case Type.DOUBLE => row.addDouble(columnName, value.asInstanceOf[Double])
case Type.STRING => row.addString(columnName, value.asInstanceOf[String])
- case Type.BINARY => row.addBinary(columnName, value.asInstanceOf[Array[Byte]])
- case Type.DECIMAL => row.addDecimal(columnName, value.asInstanceOf[BigDecimal])
- case _ => throw new IllegalArgumentException(s"Unsupported column type: $colType")
+ case Type.BINARY =>
+ row.addBinary(columnName, value.asInstanceOf[Array[Byte]])
+ case Type.DECIMAL =>
+ row.addDecimal(columnName, value.asInstanceOf[BigDecimal])
+ case _ =>
+ throw new IllegalArgumentException(s"Unsupported column type: $colType")
}
}
@@ -225,8 +273,11 @@ object TableMetadata {
case Type.BINARY =>
Base64.encodeBase64String(value.asInstanceOf[Array[Byte]])
case Type.DECIMAL =>
- value.asInstanceOf[BigDecimal].toString // TODO: Explicitly control print format
- case _ => throw new IllegalArgumentException(s"Unsupported column type: $colType")
+ value
+ .asInstanceOf[BigDecimal]
+ .toString // TODO: Explicitly control print format
+ case _ =>
+ throw new IllegalArgumentException(s"Unsupported column type: $colType")
}
}
@@ -242,7 +293,8 @@ object TableMetadata {
case Type.STRING => value
case Type.BINARY => Base64.decodeBase64(value.asInstanceOf[Array[Byte]])
case Type.DECIMAL => new BigDecimal(value) // TODO: Explicitly pass scale
- case _ => throw new IllegalArgumentException(s"Unsupported column type: $colType")
+ case _ =>
+ throw new IllegalArgumentException(s"Unsupported column type: $colType")
}
}
@@ -251,16 +303,19 @@ object TableMetadata {
val options = new CreateTableOptions()
options.setNumReplicas(metadata.getNumReplicas)
metadata.getPartitions.getHashPartitionsList.asScala.foreach { hp =>
- options.addHashPartitions(hp.getColumnNamesList, hp.getNumBuckets, hp.getSeed)
+ options
+ .addHashPartitions(hp.getColumnNamesList, hp.getNumBuckets, hp.getSeed)
}
- val rangePartitionColumns = metadata.getPartitions.getRangePartitions.getColumnNamesList
+ val rangePartitionColumns =
+ metadata.getPartitions.getRangePartitions.getColumnNamesList
options.setRangePartitionColumns(rangePartitionColumns)
- metadata.getPartitions.getRangePartitions.getBoundsList.asScala.foreach { b =>
- val lower = getPartialRow(b.getLowerBoundsList.asScala, schema)
- val upper = getPartialRow(b.getUpperBoundsList.asScala, schema)
- options.addRangePartition(lower, upper)
+ metadata.getPartitions.getRangePartitions.getBoundsList.asScala.foreach {
+ b =>
+ val lower = getPartialRow(b.getLowerBoundsList.asScala, schema)
+ val upper = getPartialRow(b.getUpperBoundsList.asScala, schema)
+ options.addRangePartition(lower, upper)
}
options
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-spark-tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/pom.xml b/java/kudu-spark-tools/pom.xml
index 0fd7744..9924f77 100644
--- a/java/kudu-spark-tools/pom.xml
+++ b/java/kudu-spark-tools/pom.xml
@@ -138,6 +138,9 @@
</dependencies>
<build>
+ <!-- set source dirs explicitly for the scalafmt plugin -->
+ <sourceDirectory>src/main/scala</sourceDirectory>
+ <testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
@@ -191,6 +194,10 @@
</artifactSet>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.antipathy</groupId>
+ <artifactId>mvn-scalafmt</artifactId>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
index fc1ef3e..0625db4 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
@@ -53,16 +53,16 @@ object ImportExportKudu {
| header: header for csv import/export, default:false
""".stripMargin
- case class ArgsCls(operation: String = "import",
- format: String = "csv",
- masterAddrs: String = defaultMasterAddrs,
- path: String = "file://",
- tableName: String = "",
- columns: String = "*",
- delimiter: String = ",",
- header: String = "false",
- inferschema: String="false"
- )
+ case class ArgsCls(
+ operation: String = "import",
+ format: String = "csv",
+ masterAddrs: String = defaultMasterAddrs,
+ path: String = "file://",
+ tableName: String = "",
+ columns: String = "*",
+ delimiter: String = ",",
+ header: String = "false",
+ inferschema: String = "false")
object ArgsCls {
private def parseInner(options: ArgsCls, args: List[String]): ArgsCls = {
@@ -109,21 +109,25 @@ object ImportExportFiles {
fail(args.tableName + s" table doesn't exist")
}
- val kuduOptions = Map(
- "kudu.table" -> args.tableName,
- "kudu.master" -> args.masterAddrs)
+ val kuduOptions =
+ Map("kudu.table" -> args.tableName, "kudu.master" -> args.masterAddrs)
args.operation match {
case "import" =>
args.format match {
case "csv" =>
- val df = sqlContext.read.option("header", args.header).option("delimiter", args.delimiter).csv(args.path)
+ val df = sqlContext.read
+ .option("header", args.header)
+ .option("delimiter", args.delimiter)
+ .csv(args.path)
kc.upsertRows(df, args.tableName)
case "parquet" =>
val df = sqlContext.read.parquet(args.path)
kc.upsertRows(df, args.tableName)
case "avro" =>
- val df = sqlContext.read.format("com.databricks.spark.avro").load(args.path)
+ val df = sqlContext.read
+ .format("com.databricks.spark.avro")
+ .load(args.path)
kc.upsertRows(df, args.tableName)
case _ => fail(args.format + s"unknown argument given ")
}
@@ -131,8 +135,11 @@ object ImportExportFiles {
val df = sqlContext.read.options(kuduOptions).kudu.select(args.columns)
args.format match {
case "csv" =>
- df.write.format("com.databricks.spark.csv").option("header", args.header).option("delimiter",
- args.delimiter).save(args.path)
+ df.write
+ .format("com.databricks.spark.csv")
+ .option("header", args.header)
+ .option("delimiter", args.delimiter)
+ .save(args.path)
case "parquet" =>
df.write.parquet(args.path)
case "avro" =>
@@ -144,18 +151,18 @@ object ImportExportFiles {
}
/**
- * Entry point for testing. SparkContext is a singleton,
- * so tests must create and manage their own.
- */
+ * Entry point for testing. SparkContext is a singleton,
+ * so tests must create and manage their own.
+ */
@InterfaceAudience.LimitedPrivate(Array("Test"))
def testMain(args: Array[String], ss: SparkSession): Unit = {
run(ArgsCls.parse(args), ss)
}
def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("Import or Export CSV files from/to Kudu ")
+ val conf =
+ new SparkConf().setAppName("Import or Export CSV files from/to Kudu ")
val ss = SparkSession.builder().config(conf).getOrCreate()
testMain(args, ss)
}
}
-
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
index 0b2e4cd..47161da 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
@@ -21,7 +21,10 @@ import java.net.InetAddress
import org.apache.kudu.client.SessionConfiguration.FlushMode
import org.apache.kudu.client.{KuduClient, KuduSession, KuduTable}
-import org.apache.kudu.mapreduce.tools.BigLinkedListCommon.{Xoroshiro128PlusRandom, _}
+import org.apache.kudu.mapreduce.tools.BigLinkedListCommon.{
+ Xoroshiro128PlusRandom,
+ _
+}
import org.apache.kudu.spark.kudu.KuduContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, TaskContext}
@@ -31,21 +34,23 @@ import org.slf4j.{Logger, LoggerFactory}
import scala.util.Try
/**
- * Spark port of [[org.apache.kudu.mapreduce.tools.IntegrationTestBigLinkedList]].
- *
- * Major differences:
- * * Currently, only the generator and verifier jobs are implemented.
- * * The heads table is not written to during generate, and not used during verify.
- * * The generate job does not write in batches. Instead, it writes a head node,
- * followed by many tail nodes into the table, and then updates just the
- * head node to point at the final tail node. Writes use AUTO_FLUSH_BACKGROUND.
- * This is hopefully easier to understand, and has the advantage of stressing
- * slightly different code paths than the MR version.
- */
+ * Spark port of [[org.apache.kudu.mapreduce.tools.IntegrationTestBigLinkedList]].
+ *
+ * Major differences:
+ * * Currently, only the generator and verifier jobs are implemented.
+ * * The heads table is not written to during generate, and not used during verify.
+ * * The generate job does not write in batches. Instead, it writes a head node,
+ * followed by many tail nodes into the table, and then updates just the
+ * head node to point at the final tail node. Writes use AUTO_FLUSH_BACKGROUND.
+ * This is hopefully easier to understand, and has the advantage of stressing
+ * slightly different code paths than the MR version.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
object IntegrationTestBigLinkedList {
- val LOG: Logger = LoggerFactory.getLogger(IntegrationTestBigLinkedList.getClass)
+
+ val LOG: Logger =
+ LoggerFactory.getLogger(IntegrationTestBigLinkedList.getClass)
def usage: String =
s"""
@@ -65,11 +70,13 @@ object IntegrationTestBigLinkedList {
""".stripMargin
def parseIntFlag(flag: String, num: String): Int = {
- Try(num.toInt).getOrElse(fail(s"failed to parse $flag value as integer: $num"))
+ Try(num.toInt)
+ .getOrElse(fail(s"failed to parse $flag value as integer: $num"))
}
def parseLongFlag(flag: String, num: String): Long = {
- Try(num.toLong).getOrElse(fail(s"failed to parse $flag value as integer: $num"))
+ Try(num.toLong)
+ .getOrElse(fail(s"failed to parse $flag value as integer: $num"))
}
def fail(msg: String): Nothing = {
@@ -100,7 +107,13 @@ object IntegrationTestBigLinkedList {
}
object Generator {
- import IntegrationTestBigLinkedList.{LOG, defaultMasterAddrs, fail, nanosToHuman, parseIntFlag}
+ import IntegrationTestBigLinkedList.{
+ LOG,
+ defaultMasterAddrs,
+ fail,
+ nanosToHuman,
+ parseIntFlag
+ }
def usage: String =
s"""
@@ -118,14 +131,15 @@ object Generator {
| table-name: the name of the linked list table, default: $DEFAULT_TABLE_NAME
""".stripMargin
- case class Args(tasks: Int = 1,
- lists: Int = 1,
- nodes: Int = 10000000,
- hashPartitions: Int = 1,
- rangePartitions: Int = 1,
- replicas: Int = 1,
- masterAddrs: String = defaultMasterAddrs,
- tableName: String = DEFAULT_TABLE_NAME)
+ case class Args(
+ tasks: Int = 1,
+ lists: Int = 1,
+ nodes: Int = 10000000,
+ hashPartitions: Int = 1,
+ rangePartitions: Int = 1,
+ replicas: Int = 1,
+ masterAddrs: String = defaultMasterAddrs,
+ tableName: String = DEFAULT_TABLE_NAME)
object Args {
private def parseInner(options: Args, args: List[String]): Args = {
@@ -140,9 +154,12 @@ object Generator {
case "--tasks" => options.copy(tasks = parseIntFlag(flag, value))
case "--lists" => options.copy(lists = parseIntFlag(flag, value))
case "--nodes" => options.copy(nodes = parseIntFlag(flag, value))
- case "--hash-partitions" => options.copy(hashPartitions = parseIntFlag(flag, value))
- case "--range-partitions" => options.copy(rangePartitions = parseIntFlag(flag, value))
- case "--replicas" => options.copy(replicas = parseIntFlag(flag, value))
+ case "--hash-partitions" =>
+ options.copy(hashPartitions = parseIntFlag(flag, value))
+ case "--range-partitions" =>
+ options.copy(rangePartitions = parseIntFlag(flag, value))
+ case "--replicas" =>
+ options.copy(replicas = parseIntFlag(flag, value))
case "--master-addrs" => options.copy(masterAddrs = value)
case "--table-name" => options.copy(tableName = value)
case _ => fail(s"unknown generate flag $flag")
@@ -163,40 +180,43 @@ object Generator {
val client: KuduClient = kc.syncClient
if (!client.tableExists(args.tableName)) {
val schema = getTableSchema
- val options = getCreateTableOptions(schema, args.replicas,
- args.rangePartitions, args.hashPartitions)
+ val options = getCreateTableOptions(
+ schema,
+ args.replicas,
+ args.rangePartitions,
+ args.hashPartitions)
client.createTable(args.tableName, getTableSchema, options)
}
// Run the generate tasks
- ss.sparkContext.makeRDD(0 until args.tasks, args.tasks)
+ ss.sparkContext
+ .makeRDD(0 until args.tasks, args.tasks)
.foreach(_ => generate(args, applicationId, kc))
}
def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("Integration Test Big Linked List Generator")
+ val conf =
+ new SparkConf().setAppName("Integration Test Big Linked List Generator")
val ss = SparkSession.builder().config(conf).getOrCreate()
run(Args.parse(args), ss)
}
/**
- * Entry point for testing. SparkContext is a singleton,
- * so tests must create and manage their own.
- */
+ * Entry point for testing. SparkContext is a singleton,
+ * so tests must create and manage their own.
+ */
@InterfaceAudience.LimitedPrivate(Array("Test"))
def testMain(args: Array[String], ss: SparkSession): Unit = {
run(Args.parse(args), ss)
}
- def generate(args: Args,
- applicationId: String,
- kc: KuduContext): Unit = {
+ def generate(args: Args, applicationId: String, kc: KuduContext): Unit = {
val taskContext = TaskContext.get()
val clientId = s"$applicationId-${taskContext.partitionId()}"
val rand = new Xoroshiro128PlusRandom()
- val client:KuduClient = kc.syncClient
+ val client: KuduClient = kc.syncClient
val table: KuduTable = client.openTable(args.tableName)
val session: KuduSession = client.newSession()
@@ -205,19 +225,21 @@ object Generator {
for (_ <- 0 until args.lists) {
val start = System.nanoTime()
insertList(clientId, args, table, session, rand)
- LOG.info(s"$clientId inserted ${args.nodes} node linked list in {}",
- nanosToHuman(System.nanoTime() - start))
+ LOG.info(
+ s"$clientId inserted ${args.nodes} node linked list in {}",
+ nanosToHuman(System.nanoTime() - start))
}
} finally {
session.close()
}
}
- def insertList(clientId: String,
- args: Args,
- table: KuduTable,
- session: KuduSession,
- rand: Xoroshiro128PlusRandom): Unit = {
+ def insertList(
+ clientId: String,
+ args: Args,
+ table: KuduTable,
+ session: KuduSession,
+ rand: Xoroshiro128PlusRandom): Unit = {
// Write the head node to the table.
val headKeyOne = rand.nextLong()
@@ -265,9 +287,10 @@ object Generator {
session.flush()
val errors = session.getPendingErrors
if (errors.getRowErrors.length > 0) {
- throw new RuntimeException(errors.getRowErrors
- .map(_.getErrorStatus.toString)
- .mkString("Row errors: [", ", ", "]"))
+ throw new RuntimeException(
+ errors.getRowErrors
+ .map(_.getErrorStatus.toString)
+ .mkString("Row errors: [", ", ", "]"))
}
}
}
@@ -284,9 +307,10 @@ object Verifier {
| table-name: the name of the linked list table, default: $DEFAULT_TABLE_NAME
""".stripMargin
- case class Args(nodes: Option[Long] = None,
- masterAddrs: String = defaultMasterAddrs,
- tableName: String = DEFAULT_TABLE_NAME)
+ case class Args(
+ nodes: Option[Long] = None,
+ masterAddrs: String = defaultMasterAddrs,
+ tableName: String = DEFAULT_TABLE_NAME)
object Args {
private def parseInner(options: Args, args: List[String]): Args = {
@@ -298,7 +322,8 @@ object Verifier {
case flag :: Nil => fail(s"flag $flag has no value\n$usage")
case flag :: value :: tail =>
val newOptions = flag match {
- case "--nodes" => options.copy(nodes = Some(parseLongFlag(flag, value)))
+ case "--nodes" =>
+ options.copy(nodes = Some(parseLongFlag(flag, value)))
case "--master-addrs" => options.copy(masterAddrs = value)
case "--table-name" => options.copy(tableName = value)
case _ => fail(s"unknown verify flag $flag")
@@ -312,21 +337,23 @@ object Verifier {
}
}
- case class Counts(referenced: Long,
- unreferenced: Long,
- extrareferences: Long,
- undefined: Long)
+ case class Counts(
+ referenced: Long,
+ unreferenced: Long,
+ extrareferences: Long,
+ undefined: Long)
/**
- * Verifies the expected count against the count of nodes from a verification run.
- * @param expected the expected node count
- * @param counts the node counts returned by the verification job
- * @return an error message, if the verification fails
- */
+ * Verifies the expected count against the count of nodes from a verification run.
+ * @param expected the expected node count
+ * @param counts the node counts returned by the verification job
+ * @return an error message, if the verification fails
+ */
def verify(expected: Option[Long], counts: Counts): Option[String] = {
if (expected.exists(_ != counts.referenced)) {
- Some(s"Found ${counts.referenced} referenced nodes, " +
- s"which does not match the expected count of ${expected.get} nodes")
+ Some(
+ s"Found ${counts.referenced} referenced nodes, " +
+ s"which does not match the expected count of ${expected.get} nodes")
} else if (counts.unreferenced > 0) {
Some(s"Found ${counts.unreferenced} unreferenced nodes")
} else if (counts.undefined > 0) {
@@ -342,14 +369,13 @@ object Verifier {
val sql = ss.sqlContext
sql.read
- .option("kudu.master", args.masterAddrs)
- .option("kudu.table", args.tableName)
- .kudu
- .createOrReplaceTempView("nodes")
+ .option("kudu.master", args.masterAddrs)
+ .option("kudu.table", args.tableName)
+ .kudu
+ .createOrReplaceTempView("nodes")
// Get a table of all nodes and their ref count
- sql.sql(
- s"""
+ sql.sql(s"""
| SELECT (SELECT COUNT(*)
| FROM nodes t2
| WHERE t1.$COLUMN_KEY_ONE = t2.$COLUMN_PREV_ONE
@@ -359,23 +385,20 @@ object Verifier {
// Compress the ref counts down to 0, 1, or 2. 0 Indicates no references,
// 1 indicates a single reference, and 2 indicates more than 1 reference.
- sql.sql(
- s"""
+ sql.sql(s"""
| SELECT (CASE WHEN ref_count > 1 THEN 2 ELSE ref_count END) as ref_count
| FROM ref_counts
""".stripMargin).createOrReplaceTempView("ref_counts")
// Aggregate the ref counts
- sql.sql(
- s"""
+ sql.sql(s"""
| SELECT ref_count, COUNT(*) as nodes
| FROM ref_counts
| GROUP BY ref_count
""".stripMargin).createOrReplaceTempView("ref_counts")
// Transform the ref count to a state.
- sql.sql(
- s"""
+ sql.sql(s"""
| SELECT CASE WHEN ref_count = 0 THEN "UNREFERENCED"
| WHEN ref_count = 1 THEN "REFERENCED"
| ELSE "EXTRAREFERENCES" END as state,
@@ -384,8 +407,7 @@ object Verifier {
""".stripMargin).createOrReplaceTempView("ref_counts")
// Find all referenced but undefined nodes.
- sql.sql(
- s"""
+ sql.sql(s"""
| SELECT $COLUMN_CLIENT as list, "UNDEFINED" as state, COUNT(*) as nodes
| FROM nodes t1
| WHERE $COLUMN_PREV_ONE IS NOT NULL
@@ -398,8 +420,7 @@ object Verifier {
""".stripMargin).createOrReplaceTempView("undefined")
// Combine the ref counts and undefined counts tables.
- val rows = sql.sql(
- s"""
+ val rows = sql.sql(s"""
| SELECT state, nodes FROM ref_counts
| UNION ALL
| SELECT state, nodes FROM undefined
@@ -425,7 +446,8 @@ object Verifier {
def main(arguments: Array[String]): Unit = {
val args = Args.parse(arguments)
- val conf = new SparkConf().setAppName("Integration Test Big Linked List Generator")
+ val conf =
+ new SparkConf().setAppName("Integration Test Big Linked List Generator")
val ss = SparkSession.builder().config(conf).getOrCreate()
val counts = run(Args.parse(arguments), ss)
@@ -435,13 +457,15 @@ object Verifier {
object Looper {
import IntegrationTestBigLinkedList.{LOG, fail}
+
def main(args: Array[String]): Unit = {
- val conf = new SparkConf().setAppName("Integration Test Big Linked List Looper")
+ val conf =
+ new SparkConf().setAppName("Integration Test Big Linked List Looper")
val ss = SparkSession.builder().config(conf).getOrCreate()
val genArgs = Generator.Args.parse(args)
- var verifyArgs = Verifier.Args(masterAddrs = genArgs.masterAddrs,
- tableName = genArgs.tableName)
+ var verifyArgs = Verifier
+ .Args(masterAddrs = genArgs.masterAddrs, tableName = genArgs.tableName)
val nodesPerLoop = genArgs.tasks * genArgs.lists * genArgs.nodes
for (n <- Stream.from(1)) {
@@ -449,7 +473,8 @@ object Looper {
val count = Verifier.run(verifyArgs, ss)
val expected = verifyArgs.nodes.map(_ + nodesPerLoop)
Verifier.verify(expected, count).map(fail)
- verifyArgs = verifyArgs.copy(nodes = Some(expected.getOrElse(nodesPerLoop)))
+ verifyArgs =
+ verifyArgs.copy(nodes = Some(expected.getOrElse(nodesPerLoop)))
LOG.info("*************************************************")
LOG.info(s"Completed $n loops. Nodes verified: ${count.referenced}")
LOG.info("*************************************************")
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-spark/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-spark/pom.xml b/java/kudu-spark/pom.xml
index 51b564e..7665c2e 100644
--- a/java/kudu-spark/pom.xml
+++ b/java/kudu-spark/pom.xml
@@ -95,6 +95,9 @@
</dependencies>
<build>
+ <!-- Set source dirs explicitly for the scalafmt plugin. -->
+ <sourceDirectory>src/main/scala</sourceDirectory>
+ <testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -152,6 +155,10 @@
</artifactSet>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.antipathy</groupId>
+ <artifactId>mvn-scalafmt</artifactId>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/kudu/blob/79166955/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index 090e5fb..7970459 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -33,16 +33,18 @@ import org.apache.kudu.client._
import org.apache.kudu.spark.kudu.SparkUtil._
/**
- * Data source for integration with Spark's [[DataFrame]] API.
- *
- * Serves as a factory for [[KuduRelation]] instances for Spark. Spark will
- * automatically look for a [[RelationProvider]] implementation named
- * `DefaultSource` when the user specifies the path of a source during DDL
- * operations through [[org.apache.spark.sql.DataFrameReader.format]].
- */
+ * Data source for integration with Spark's [[DataFrame]] API.
+ *
+ * Serves as a factory for [[KuduRelation]] instances for Spark. Spark will
+ * automatically look for a [[RelationProvider]] implementation named
+ * `DefaultSource` when the user specifies the path of a source during DDL
+ * operations through [[org.apache.spark.sql.DataFrameReader.format]].
+ */
@InterfaceStability.Unstable
-class DefaultSource extends RelationProvider with CreatableRelationProvider
- with SchemaRelationProvider {
+class DefaultSource
+ extends RelationProvider
+ with CreatableRelationProvider
+ with SchemaRelationProvider {
val TABLE_KEY = "kudu.table"
val KUDU_MASTER = "kudu.master"
@@ -65,67 +67,103 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
}
/**
- * Construct a BaseRelation using the provided context and parameters.
- *
- * @param sqlContext SparkSQL context
- * @param parameters parameters given to us from SparkSQL
- * @return a BaseRelation Object
- */
- override def createRelation(sqlContext: SQLContext,
- parameters: Map[String, String]): BaseRelation = {
- val tableName = parameters.getOrElse(TABLE_KEY,
+ * Construct a BaseRelation using the provided context and parameters.
+ *
+ * @param sqlContext SparkSQL context
+ * @param parameters parameters given to us from SparkSQL
+ * @return a BaseRelation Object
+ */
+ override def createRelation(
+ sqlContext: SQLContext,
+ parameters: Map[String, String]): BaseRelation = {
+ val tableName = parameters.getOrElse(
+ TABLE_KEY,
throw new IllegalArgumentException(
s"Kudu table name must be specified in create options using key '$TABLE_KEY'"))
val kuduMaster = parameters.getOrElse(KUDU_MASTER, defaultMasterAddrs)
- val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
- val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
- .getOrElse(false)
- val scanLocality = getScanLocalityType(parameters.getOrElse(SCAN_LOCALITY, "closest_replica"))
- val ignoreDuplicateRowErrors = Try(parameters(IGNORE_DUPLICATE_ROW_ERRORS).toBoolean).getOrElse(false) ||
- Try(parameters(OPERATION) == "insert-ignore").getOrElse(false)
- val ignoreNull = Try(parameters.getOrElse(IGNORE_NULL, "false").toBoolean).getOrElse(false)
- val writeOptions = new KuduWriteOptions(ignoreDuplicateRowErrors, ignoreNull)
+ val operationType = getOperationType(
+ parameters.getOrElse(OPERATION, "upsert"))
+ val faultTolerantScanner =
+ Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
+ .getOrElse(false)
+ val scanLocality = getScanLocalityType(
+ parameters.getOrElse(SCAN_LOCALITY, "closest_replica"))
+ val ignoreDuplicateRowErrors = Try(
+ parameters(IGNORE_DUPLICATE_ROW_ERRORS).toBoolean).getOrElse(false) ||
+ Try(parameters(OPERATION) == "insert-ignore").getOrElse(false)
+ val ignoreNull =
+ Try(parameters.getOrElse(IGNORE_NULL, "false").toBoolean).getOrElse(false)
+ val writeOptions =
+ new KuduWriteOptions(ignoreDuplicateRowErrors, ignoreNull)
- new KuduRelation(tableName, kuduMaster, faultTolerantScanner,
- scanLocality, getScanRequestTimeoutMs(parameters), getSocketReadTimeoutMs(parameters),
- operationType, None, writeOptions)(sqlContext)
+ new KuduRelation(
+ tableName,
+ kuduMaster,
+ faultTolerantScanner,
+ scanLocality,
+ getScanRequestTimeoutMs(parameters),
+ getSocketReadTimeoutMs(parameters),
+ operationType,
+ None,
+ writeOptions
+ )(sqlContext)
}
/**
- * Creates a relation and inserts data to specified table.
- *
- * @param sqlContext
- * @param mode Only Append mode is supported. It will upsert or insert data
- * to an existing table, depending on the upsert parameter
- * @param parameters Necessary parameters for kudu.table, kudu.master, etc..
- * @param data Dataframe to save into kudu
- * @return returns populated base relation
- */
- override def createRelation(sqlContext: SQLContext, mode: SaveMode,
- parameters: Map[String, String], data: DataFrame): BaseRelation = {
+ * Creates a relation and inserts data to specified table.
+ *
+ * @param sqlContext
+ * @param mode Only Append mode is supported. It will upsert or insert data
+ * to an existing table, depending on the upsert parameter
+ * @param parameters Necessary parameters for kudu.table, kudu.master, etc..
+ * @param data Dataframe to save into kudu
+ * @return returns populated base relation
+ */
+ override def createRelation(
+ sqlContext: SQLContext,
+ mode: SaveMode,
+ parameters: Map[String, String],
+ data: DataFrame): BaseRelation = {
val kuduRelation = createRelation(sqlContext, parameters)
mode match {
- case SaveMode.Append => kuduRelation.asInstanceOf[KuduRelation].insert(data, false)
- case _ => throw new UnsupportedOperationException("Currently, only Append is supported")
+ case SaveMode.Append =>
+ kuduRelation.asInstanceOf[KuduRelation].insert(data, false)
+ case _ =>
+ throw new UnsupportedOperationException(
+ "Currently, only Append is supported")
}
kuduRelation
}
- override def createRelation(sqlContext: SQLContext, parameters: Map[String, String],
- schema: StructType): BaseRelation = {
- val tableName = parameters.getOrElse(TABLE_KEY,
- throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
- s"using key '$TABLE_KEY'"))
+ override def createRelation(
+ sqlContext: SQLContext,
+ parameters: Map[String, String],
+ schema: StructType): BaseRelation = {
+ val tableName = parameters.getOrElse(
+ TABLE_KEY,
+ throw new IllegalArgumentException(
+ s"Kudu table name must be specified in create options " +
+ s"using key '$TABLE_KEY'"))
val kuduMaster = parameters.getOrElse(KUDU_MASTER, defaultMasterAddrs)
- val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
- val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
- .getOrElse(false)
- val scanLocality = getScanLocalityType(parameters.getOrElse(SCAN_LOCALITY, "closest_replica"))
+ val operationType = getOperationType(
+ parameters.getOrElse(OPERATION, "upsert"))
+ val faultTolerantScanner =
+ Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
+ .getOrElse(false)
+ val scanLocality = getScanLocalityType(
+ parameters.getOrElse(SCAN_LOCALITY, "closest_replica"))
- new KuduRelation(tableName, kuduMaster, faultTolerantScanner,
- scanLocality, getScanRequestTimeoutMs(parameters), getSocketReadTimeoutMs(parameters),
- operationType, Some(schema))(sqlContext)
+ new KuduRelation(
+ tableName,
+ kuduMaster,
+ faultTolerantScanner,
+ scanLocality,
+ getScanRequestTimeoutMs(parameters),
+ getSocketReadTimeoutMs(parameters),
+ operationType,
+ Some(schema)
+ )(sqlContext)
}
private def getOperationType(opParam: String): OperationType = {
@@ -135,7 +173,9 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
case "upsert" => Upsert
case "update" => Update
case "delete" => Delete
- case _ => throw new IllegalArgumentException(s"Unsupported operation type '$opParam'")
+ case _ =>
+ throw new IllegalArgumentException(
+ s"Unsupported operation type '$opParam'")
}
}
@@ -143,43 +183,46 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
opParam.toLowerCase match {
case "leader_only" => ReplicaSelection.LEADER_ONLY
case "closest_replica" => ReplicaSelection.CLOSEST_REPLICA
- case _ => throw new IllegalArgumentException(s"Unsupported replica selection type '$opParam'")
+ case _ =>
+ throw new IllegalArgumentException(
+ s"Unsupported replica selection type '$opParam'")
}
}
}
/**
- * Implementation of Spark BaseRelation.
- *
- * @param tableName Kudu table that we plan to read from
- * @param masterAddrs Kudu master addresses
- * @param faultTolerantScanner scanner type to be used. Fault tolerant if true,
- * otherwise, use non fault tolerant one
- * @param scanLocality If true scan locality is enabled, so that the scan will
- * take place at the closest replica.
- * @param scanRequestTimeoutMs Maximum time allowed per scan request, in milliseconds
- * @param operationType The default operation type to perform when writing to the relation
- * @param userSchema A schema used to select columns for the relation
- * @param writeOptions Kudu write options
- * @param sqlContext SparkSQL context
- */
+ * Implementation of Spark BaseRelation.
+ *
+ * @param tableName Kudu table that we plan to read from
+ * @param masterAddrs Kudu master addresses
+ * @param faultTolerantScanner scanner type to be used. Fault tolerant if true,
+ * otherwise, use non fault tolerant one
+ * @param scanLocality If true scan locality is enabled, so that the scan will
+ * take place at the closest replica.
+ * @param scanRequestTimeoutMs Maximum time allowed per scan request, in milliseconds
+ * @param operationType The default operation type to perform when writing to the relation
+ * @param userSchema A schema used to select columns for the relation
+ * @param writeOptions Kudu write options
+ * @param sqlContext SparkSQL context
+ */
@InterfaceStability.Unstable
-class KuduRelation(private val tableName: String,
- private val masterAddrs: String,
- private val faultTolerantScanner: Boolean,
- private val scanLocality: ReplicaSelection,
- private[kudu] val scanRequestTimeoutMs: Option[Long],
- private[kudu] val socketReadTimeoutMs: Option[Long],
- private val operationType: OperationType,
- private val userSchema: Option[StructType],
- private val writeOptions: KuduWriteOptions = new KuduWriteOptions)(
- val sqlContext: SQLContext)
- extends BaseRelation
+class KuduRelation(
+ private val tableName: String,
+ private val masterAddrs: String,
+ private val faultTolerantScanner: Boolean,
+ private val scanLocality: ReplicaSelection,
+ private[kudu] val scanRequestTimeoutMs: Option[Long],
+ private[kudu] val socketReadTimeoutMs: Option[Long],
+ private val operationType: OperationType,
+ private val userSchema: Option[StructType],
+ private val writeOptions: KuduWriteOptions = new KuduWriteOptions)(
+ val sqlContext: SQLContext)
+ extends BaseRelation
with PrunedFilteredScan
with InsertableRelation {
- private val context: KuduContext = new KuduContext(masterAddrs, sqlContext.sparkContext,
- socketReadTimeoutMs)
+ private val context: KuduContext =
+ new KuduContext(masterAddrs, sqlContext.sparkContext, socketReadTimeoutMs)
private val table: KuduTable = context.syncClient.openTable(tableName)
@@ -187,36 +230,47 @@ class KuduRelation(private val tableName: String,
filters.filterNot(KuduRelation.supportsFilter)
/**
- * Generates a SparkSQL schema object so SparkSQL knows what is being
- * provided by this BaseRelation.
- *
- * @return schema generated from the Kudu table's schema
- */
+ * Generates a SparkSQL schema object so SparkSQL knows what is being
+ * provided by this BaseRelation.
+ *
+ * @return schema generated from the Kudu table's schema
+ */
override def schema: StructType = {
sparkSchema(table.getSchema, userSchema.map(_.fieldNames))
}
/**
- * Build the RDD to scan rows.
- *
- * @param requiredColumns columns that are being requested by the requesting query
- * @param filters filters that are being applied by the requesting query
- * @return RDD will all the results from Kudu
- */
- override def buildScan(requiredColumns: Array[String], filters: Array[Filter]): RDD[Row] = {
+ * Build the RDD to scan rows.
+ *
+ * @param requiredColumns columns that are being requested by the requesting query
+ * @param filters filters that are being applied by the requesting query
+ * @return RDD will all the results from Kudu
+ */
+ override def buildScan(
+ requiredColumns: Array[String],
+ filters: Array[Filter]): RDD[Row] = {
val predicates = filters.flatMap(filterToPredicate)
- new KuduRDD(context, 1024 * 1024 * 20, requiredColumns, predicates,
- table, faultTolerantScanner, scanLocality, scanRequestTimeoutMs,
- socketReadTimeoutMs, sqlContext.sparkContext)
+ new KuduRDD(
+ context,
+ 1024 * 1024 * 20,
+ requiredColumns,
+ predicates,
+ table,
+ faultTolerantScanner,
+ scanLocality,
+ scanRequestTimeoutMs,
+ socketReadTimeoutMs,
+ sqlContext.sparkContext
+ )
}
/**
- * Converts a Spark [[Filter]] to a Kudu [[KuduPredicate]].
- *
- * @param filter the filter to convert
- * @return the converted filter
- */
- private def filterToPredicate(filter : Filter) : Array[KuduPredicate] = {
+ * Converts a Spark [[Filter]] to a Kudu [[KuduPredicate]].
+ *
+ * @param filter the filter to convert
+ * @return the converted filter
+ */
+ private def filterToPredicate(filter: Filter): Array[KuduPredicate] = {
filter match {
case EqualTo(column, value) =>
Array(comparisonPredicate(column, ComparisonOp.EQUAL, value))
@@ -232,25 +286,29 @@ class KuduRelation(private val tableName: String,
Array(inListPredicate(column, values))
case StringStartsWith(column, prefix) =>
prefixInfimum(prefix) match {
- case None => Array(comparisonPredicate(column, ComparisonOp.GREATER_EQUAL, prefix))
+ case None =>
+ Array(
+ comparisonPredicate(column, ComparisonOp.GREATER_EQUAL, prefix))
case Some(inf) =>
- Array(comparisonPredicate(column, ComparisonOp.GREATER_EQUAL, prefix),
- comparisonPredicate(column, ComparisonOp.LESS, inf))
+ Array(
+ comparisonPredicate(column, ComparisonOp.GREATER_EQUAL, prefix),
+ comparisonPredicate(column, ComparisonOp.LESS, inf))
}
case IsNull(column) => Array(isNullPredicate(column))
case IsNotNull(column) => Array(isNotNullPredicate(column))
- case And(left, right) => filterToPredicate(left) ++ filterToPredicate(right)
+ case And(left, right) =>
+ filterToPredicate(left) ++ filterToPredicate(right)
case _ => Array()
}
}
/**
- * Returns the smallest string s such that, if p is a prefix of t,
- * then t < s, if one exists.
- *
- * @param p the prefix
- * @return Some(the prefix infimum), or None if none exists.
- */
+ * Returns the smallest string s such that, if p is a prefix of t,
+ * then t < s, if one exists.
+ *
+ * @param p the prefix
+ * @return Some(the prefix infimum), or None if none exists.
+ */
private def prefixInfimum(p: String): Option[String] = {
p.reverse.dropWhile(_ == Char.MaxValue).reverse match {
case "" => None
@@ -259,72 +317,88 @@ class KuduRelation(private val tableName: String,
}
/**
- * Creates a new comparison predicate for the column, comparison operator, and comparison value.
- *
- * @param column the column name
- * @param operator the comparison operator
- * @param value the comparison value
- * @return the comparison predicate
- */
- private def comparisonPredicate(column: String,
- operator: ComparisonOp,
- value: Any): KuduPredicate = {
+ * Creates a new comparison predicate for the column, comparison operator, and comparison value.
+ *
+ * @param column the column name
+ * @param operator the comparison operator
+ * @param value the comparison value
+ * @return the comparison predicate
+ */
+ private def comparisonPredicate(
+ column: String,
+ operator: ComparisonOp,
+ value: Any): KuduPredicate = {
val columnSchema = table.getSchema.getColumn(column)
value match {
- case value: Boolean => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Byte => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Short => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Int => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Long => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Timestamp => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Float => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Double => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: String => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: Array[Byte] => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
- case value: BigDecimal => KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Boolean =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Byte =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Short =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Int =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Long =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Timestamp =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Float =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Double =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: String =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: Array[Byte] =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
+ case value: BigDecimal =>
+ KuduPredicate.newComparisonPredicate(columnSchema, operator, value)
}
}
/**
- * Creates a new in list predicate for the column and values.
- *
- * @param column the column name
- * @param values the values
- * @return the in list predicate
- */
- private def inListPredicate(column: String, values: Array[Any]): KuduPredicate = {
- KuduPredicate.newInListPredicate(table.getSchema.getColumn(column), values.toList.asJava)
+ * Creates a new in list predicate for the column and values.
+ *
+ * @param column the column name
+ * @param values the values
+ * @return the in list predicate
+ */
+ private def inListPredicate(
+ column: String,
+ values: Array[Any]): KuduPredicate = {
+ KuduPredicate.newInListPredicate(
+ table.getSchema.getColumn(column),
+ values.toList.asJava)
}
/**
- * Creates a new `IS NULL` predicate for the column.
- *
- * @param column the column name
- * @return the `IS NULL` predicate
- */
+ * Creates a new `IS NULL` predicate for the column.
+ *
+ * @param column the column name
+ * @return the `IS NULL` predicate
+ */
private def isNullPredicate(column: String): KuduPredicate = {
KuduPredicate.newIsNullPredicate(table.getSchema.getColumn(column))
}
/**
- * Creates a new `IS NULL` predicate for the column.
- *
- * @param column the column name
- * @return the `IS NULL` predicate
- */
+ * Creates a new `IS NULL` predicate for the column.
+ *
+ * @param column the column name
+ * @return the `IS NULL` predicate
+ */
private def isNotNullPredicate(column: String): KuduPredicate = {
KuduPredicate.newIsNotNullPredicate(table.getSchema.getColumn(column))
}
/**
- * Writes data into an existing Kudu table.
- *
- * If the `kudu.operation` parameter is set, the data will use that operation
- * type. If the parameter is unset, the data will be upserted.
- *
- * @param data [[DataFrame]] to be inserted into Kudu
- * @param overwrite must be false; otherwise, throws [[UnsupportedOperationException]]
- */
+ * Writes data into an existing Kudu table.
+ *
+ * If the `kudu.operation` parameter is set, the data will use that operation
+ * type. If the parameter is unset, the data will be upserted.
+ *
+ * @param data [[DataFrame]] to be inserted into Kudu
+ * @param overwrite must be false; otherwise, throws [[UnsupportedOperationException]]
+ */
override def insert(data: DataFrame, overwrite: Boolean): Unit = {
if (overwrite) {
throw new UnsupportedOperationException("overwrite is not yet supported")
@@ -334,22 +408,20 @@ class KuduRelation(private val tableName: String,
}
private[spark] object KuduRelation {
+
/**
- * Returns `true` if the filter is able to be pushed down to Kudu.
- *
- * @param filter the filter to test
- */
+ * Returns `true` if the filter is able to be pushed down to Kudu.
+ *
+ * @param filter the filter to test
+ */
+ // formatter: off
private def supportsFilter(filter: Filter): Boolean = filter match {
- case EqualTo(_, _)
- | GreaterThan(_, _)
- | GreaterThanOrEqual(_, _)
- | LessThan(_, _)
- | LessThanOrEqual(_, _)
- | In(_, _)
- | StringStartsWith(_, _)
- | IsNull(_)
- | IsNotNull(_) => true
+ case EqualTo(_, _) | GreaterThan(_, _) | GreaterThanOrEqual(_, _) |
+ LessThan(_, _) | LessThanOrEqual(_, _) | In(_, _) |
+ StringStartsWith(_, _) | IsNull(_) | IsNotNull(_) =>
+ true
case And(left, right) => supportsFilter(left) && supportsFilter(right)
case _ => false
}
+ // formatter: on
}