You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2018/10/30 23:29:52 UTC
[1/2] kudu git commit: [backup] Port keepAlive implementation to
KuduBackupRDD
Repository: kudu
Updated Branches:
refs/heads/master a2decad12 -> 606488495
[backup] Port keepAlive implementation to KuduBackupRDD
This copies the keepAlive implementation from
KuduRDD over to the KuduBackupRDD.
Change-Id: I10d7645781d2b18eb89034e1b4f55045b01ca0da
Reviewed-on: http://gerrit.cloudera.org:8080/11817
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7d972a8a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7d972a8a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7d972a8a
Branch: refs/heads/master
Commit: 7d972a8ae7313c8a6be414a3272f3b297b5c66db
Parents: a2decad
Author: Grant Henke <gr...@apache.org>
Authored: Mon Oct 29 13:36:33 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Tue Oct 30 21:17:10 2018 +0000
----------------------------------------------------------------------
.../apache/kudu/backup/KuduBackupOptions.scala | 10 +++++++++-
.../org/apache/kudu/backup/KuduBackupRDD.scala | 21 ++++++++++++++++++--
2 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/7d972a8a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
index 02bfa28..c594b1a 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala
@@ -33,7 +33,8 @@ case class KuduBackupOptions(
format: String = KuduBackupOptions.DefaultFormat,
scanBatchSize: Int = KuduBackupOptions.DefaultScanBatchSize,
scanRequestTimeout: Long = KuduBackupOptions.DefaultScanRequestTimeout,
- scanPrefetching: Boolean = KuduBackupOptions.DefaultScanPrefetching)
+ scanPrefetching: Boolean = KuduBackupOptions.DefaultScanPrefetching,
+ keepAlivePeriodMs: Long = KuduBackupOptions.defaultKeepAlivePeriodMs)
object KuduBackupOptions {
val DefaultFormat: String = "parquet"
@@ -42,6 +43,7 @@ object KuduBackupOptions {
AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS // 30 seconds
val DefaultScanPrefetching
: Boolean = false // TODO: Add a test per KUDU-1260 and enable by default?
+ val defaultKeepAlivePeriodMs: Long = 15000 // 25% of the default scanner ttl.
// TODO: clean up usage output.
// TODO: timeout configurations.
@@ -83,6 +85,12 @@ object KuduBackupOptions {
.text("An experimental flag to enable pre-fetching data.")
.optional()
+ opt[Long]("keepAlivePeriodMs")
+ .action((v, o) => o.copy(keepAlivePeriodMs = v))
+ .text("Sets the period at which to send keep-alive requests to the tablet server to ensure" +
+ " that scanners do not time out")
+ .optional()
+
arg[String]("<table>...")
.unbounded()
.action((v, o) => o.copy(tables = o.tables :+ v))
http://git-wip-us.apache.org/repos/asf/kudu/blob/7d972a8a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
----------------------------------------------------------------------
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
index 740dfb9..f42b369 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala
@@ -42,6 +42,9 @@ class KuduBackupRDD private[kudu] (
@transient val sc: SparkContext)
extends RDD[Row](sc, Nil) {
+ // Defined here because the options are transient.
+ private val keepAlivePeriodMs = options.keepAlivePeriodMs
+
// TODO: Split large tablets into smaller scan tokens?
override protected def getPartitions: Array[Partition] = {
val client = kuduContext.syncClient
@@ -84,7 +87,7 @@ class KuduBackupRDD private[kudu] (
// TODO: Get deletes and updates for incremental backups.
val scanner =
KuduScanToken.deserializeIntoScanner(partition.scanToken, client)
- new RowIterator(scanner)
+ new RowIterator(scanner, keepAlivePeriodMs)
}
override def getPreferredLocations(partition: Partition): Seq[String] = {
@@ -103,9 +106,22 @@ private case class KuduBackupPartition(index: Int, scanToken: Array[Byte], locat
* that takes the job partitions and task context and expects to return an Iterator[Row].
* This implementation facilitates that.
*/
-private class RowIterator(private val scanner: KuduScanner) extends Iterator[Row] {
+private class RowIterator(private val scanner: KuduScanner, val keepAlivePeriodMs: Long)
+ extends Iterator[Row] {
private var currentIterator: RowResultIterator = RowResultIterator.empty
+ private var lastKeepAliveTimeMs = System.currentTimeMillis()
+
+ /**
+ * Calls the keepAlive API on the current scanner if the keepAlivePeriodMs has passed.
+ */
+ private def KeepKuduScannerAlive(): Unit = {
+ val now = System.currentTimeMillis
+ if (now >= lastKeepAliveTimeMs + keepAlivePeriodMs && !scanner.isClosed) {
+ scanner.keepAlive()
+ lastKeepAliveTimeMs = now
+ }
+ }
override def hasNext: Boolean = {
while (!currentIterator.hasNext && scanner.hasMoreRows) {
@@ -114,6 +130,7 @@ private class RowIterator(private val scanner: KuduScanner) extends Iterator[Row
}
currentIterator = scanner.nextRows()
}
+ KeepKuduScannerAlive()
currentIterator.hasNext
}
[2/2] kudu git commit: [docs] site.sh: treat doxygen warnings as
errors
Posted by al...@apache.org.
[docs] site.sh: treat doxygen warnings as errors
With this patch, doxygen treats warnings as errors while generating
Kudu C++ client API docs for the project Web site. By default, when
just building the 'doxygen' target without any special flags in the
development environment, doxygen does not treat warnings as errors.
The idea behind this patch is to make it easier to spot various
documentation-related issues in the Kudu C++ client API when generating
public-facing documentation.
Change-Id: I8badede45f1f54788bbb05ebcb80bfb0b61120cb
Reviewed-on: http://gerrit.cloudera.org:8080/11831
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/60648849
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/60648849
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/60648849
Branch: refs/heads/master
Commit: 6064884950f7bb5cb5ca1c4767ac0f34e1c631c9
Parents: 7d972a8
Author: Alexey Serbin <al...@apache.org>
Authored: Tue Oct 30 15:31:15 2018 -0700
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Tue Oct 30 23:24:00 2018 +0000
----------------------------------------------------------------------
CMakeLists.txt | 9 +++++++--
README.adoc | 11 ++++++-----
docs/support/doxygen/client_api.doxy.in | 4 ++++
docs/support/scripts/make_site.sh | 4 ++++
4 files changed, 21 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/60648849/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 388eaad..4ae7190 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1408,6 +1408,11 @@ if (UNIX)
if (NOT (DOXYGEN_FOUND AND DOXYGEN_DOT_FOUND))
message(WARNING "Doxygen with Dot support (graphviz) not found: 'doxygen' target is not available")
else ()
+ if (DOXYGEN_WARN_AS_ERROR)
+ set(DOXY_CLIENT_API_WARN_AS_ERROR YES)
+ else()
+ set(DOXY_CLIENT_API_WARN_AS_ERROR NO)
+ endif()
set(DOXY_SUBDIR ${CMAKE_CURRENT_BINARY_DIR}/docs/doxygen)
set(DOXY_CLIENT_DESTDIR ${DOXY_SUBDIR}/tmp.client)
set(DOXY_CLIENT_API_CFG ${DOXY_SUBDIR}/client_api.doxy)
@@ -1415,8 +1420,8 @@ if (UNIX)
set(DOXY_CLIENT_API_OUTDIR ${DOXY_SUBDIR}/client_api)
list(APPEND DOXY_CLIENT_API_EXCLUDE
"share/doc/kuduClient/examples/example.cc")
- # NOTE: DOXY_CLIENT_API_OUTDIR and DOXY_CLIENT_API_FOOTER are used
- # in client_api.doxy.in template file
+ # NOTE: DOXY_CLIENT_API_EXCLUDE, DOXY_CLIENT_API_OUTDIR, and
+ # DOXY_CLIENT_API_FOOTER are used in client_api.doxy.in template file
configure_file(docs/support/doxygen/client_api.doxy.in ${DOXY_CLIENT_API_CFG} @ONLY)
configure_file(docs/support/doxygen/client_api.footer.in ${DOXY_CLIENT_API_FOOTER} @ONLY)
add_custom_target(doxy_install_client_alt_destdir
http://git-wip-us.apache.org/repos/asf/kudu/blob/60648849/README.adoc
----------------------------------------------------------------------
diff --git a/README.adoc b/README.adoc
index 04215a4..79d338e 100644
--- a/README.adoc
+++ b/README.adoc
@@ -299,7 +299,8 @@ dependencies into `$HOME/.gems` using http://bundler.io/[bundler].
=== Updating the Kudu web site documentation
To update the documentation that is integrated into the Kudu web site,
-including Javadoc documentation, you may run the following command:
+including Java and C++ client API documentation, you may run the following
+command:
[source,bash]
----
@@ -308,10 +309,10 @@ $ ./docs/support/scripts/make_site.sh
This script will use your local Git repository to check out a shallow clone of
the 'gh-pages' branch and use `make_docs.sh` to generate the HTML documentation
-for the web site. It will also build the Javadoc documentation. These will be
-placed inside the checked-out web site, along with a tarball containing only
-the generated documentation (the _docs/_ and _apidocs/_ paths on the web site).
-Everything can be found in the _build/site_ subdirectory.
+for the web site. It will also build the Javadoc and Doxygen documentation.
+These will be placed inside the checked-out web site, along with a tarball
+containing only the generated documentation (the _docs/_ and _apidocs/_ paths
+on the web site). Everything can be found in the _build/site_ subdirectory.
You can proceed to commit the changes in the pages repository and send a code
review for your changes. In the future, this step may be automated whenever
http://git-wip-us.apache.org/repos/asf/kudu/blob/60648849/docs/support/doxygen/client_api.doxy.in
----------------------------------------------------------------------
diff --git a/docs/support/doxygen/client_api.doxy.in b/docs/support/doxygen/client_api.doxy.in
index 33aea70..f7e862c 100644
--- a/docs/support/doxygen/client_api.doxy.in
+++ b/docs/support/doxygen/client_api.doxy.in
@@ -51,6 +51,10 @@ HTML_FOOTER = @DOXY_CLIENT_API_FOOTER@
# the Kudu C++ client API documentation.
OUTPUT_DIRECTORY = @DOXY_CLIENT_API_OUTDIR@
+# Whether to treat warnings as errors. It makes sense to be more strict while
+# generating documentation targeted for broader audience (e.g., for releases).
+WARN_AS_ERROR = @DOXY_CLIENT_API_WARN_AS_ERROR@
+
# Do not be too verbose: QUIET is set to YES, so informational messages are off.
QUIET = YES
http://git-wip-us.apache.org/repos/asf/kudu/blob/60648849/docs/support/scripts/make_site.sh
----------------------------------------------------------------------
diff --git a/docs/support/scripts/make_site.sh b/docs/support/scripts/make_site.sh
index c675813..e6cd7f3 100755
--- a/docs/support/scripts/make_site.sh
+++ b/docs/support/scripts/make_site.sh
@@ -75,10 +75,14 @@ echo "Successfully built third-party dependencies."
mkdir -p "$BUILD_ROOT"
cd "$BUILD_ROOT"
rm -rf CMakeCache CMakeFiles/
+if [ -n "$OPT_DOXYGEN" ]; then
+ DOXYGEN_FLAGS="-DDOXYGEN_WARN_AS_ERROR=1"
+fi
$SOURCE_ROOT/build-support/enable_devtoolset.sh \
$SOURCE_ROOT/thirdparty/installed/common/bin/cmake \
-DNO_TESTS=1 \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+ $DOXYGEN_FLAGS \
$SOURCE_ROOT
MAKE_TARGETS="kudu kudu-tserver kudu-master"
if [ -n "$OPT_DOXYGEN" ]; then