You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2025/09/15 11:11:17 UTC
(kudu) branch master updated: Deflake testRandomBackupAndRestore
This is an automated email from the ASF dual-hosted git repository.
mgreber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 724911497 Deflake testRandomBackupAndRestore
724911497 is described below
commit 724911497f7b2b2d6bc53f1c510ac0b82889bbb3
Author: Marton Greber <gr...@gmail.com>
AuthorDate: Thu Aug 28 12:16:19 2025 +0200
Deflake testRandomBackupAndRestore
The backup/restore tests were intermittently failing. This wasn’t
obvious at first because dist-test archives contained empty
_ARCHIVE_TOO_BIG_-* files.
Root cause was a Base64 dependency mismatch:
java.lang.NoClassDefFoundError: org/apache/commons/net/util/Base64
Switching from org.apache.commons.net.util.Base64 to java.util.Base64
resolves the failure. The issue is reproduced by the new test
testBinaryColumnBackupReproducesBase64Issue.
Running the repro test without the fix results in this stacktrace:
13:45:05.130 [Test worker] ERROR org.apache.kudu.test.junit.RetryRule -- org.apache.kudu.backup.TestKuduBackup.testBinaryColumnBackupReproducesBase64Issue: failed attempt 1
java.lang.NoClassDefFoundError: org/apache/commons/net/util/Base64
at org.apache.kudu.backup.TableMetadata$.valueToString(TableMetadata.scala:351)
at org.apache.kudu.backup.TableMetadata$.$anonfun$getTableMetadata$1(TableMetadata.scala:78)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at org.apache.kudu.backup.TableMetadata$.getTableMetadata(TableMetadata.scala:61)
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:102)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.net.util.Base64
at java.net.URLClassLoader.findClass(URLClassLoader.java:387)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 56 common frames omitted
During investigation I found another bug with BINARY range-partition
boundaries causing:
java.lang.ClassCastException: class java.nio.HeapByteBuffer cannot be
cast to class [B
This is fixed in the same valueToString switch; covered by
testBinaryPartitionBoundariesBase64Issue.
Running the repro test without the fix results in this stacktrace:
13:39:47.053 [ForkJoinPool-2-worker-1] ERROR org.apache.kudu.backup.KuduBackup$ -- Failed to back up table binary-partition-base64-issue
java.lang.ClassCastException: java.nio.HeapByteBuffer cannot be cast to [B
at org.apache.kudu.backup.TableMetadata$.valueToString(TableMetadata.scala:351)
at org.apache.kudu.backup.TableMetadata$.$anonfun$getBoundValues$2(TableMetadata.scala:225)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at org.apache.kudu.backup.TableMetadata$.getBoundValues(TableMetadata.scala:219)
at org.apache.kudu.backup.TableMetadata$.$anonfun$getRangePartitionMetadata$2(TableMetadata.scala:196)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at org.apache.kudu.backup.TableMetadata$.getRangePartitionMetadata(TableMetadata.scala:195)
at org.apache.kudu.backup.TableMetadata$.getPartitionSchemaMetadata(TableMetadata.scala:125)
at org.apache.kudu.backup.TableMetadata$.getTableMetadata(TableMetadata.scala:106)
at org.apache.kudu.backup.KuduBackup$.doBackup(KuduBackup.scala:102)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$6(KuduBackup.scala:146)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.Try$.apply(Try.scala:213)
at org.apache.kudu.backup.KuduBackup$.$anonfun$run$5(KuduBackup.scala:146)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1064)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1061)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
at java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:968)
at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:963)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
at java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Before these fixes, running org.apache.kudu.backup.TestKuduBackup 100
times yielded 43 failures / 57 passes. After the fixes: 100/100 passes.
The random nature of test failures is because the problematic test:
testRandomBackupAndRestore uses randomized schema and data.
This is not a full RCA I wanted to deflake with reasonable solution;
I’ve opened KUDU-3694 to track root-cause follow-ups and additional
action items.
Change-Id: Ibbf05b23cb6c922ee880f4f1dda262be5e97452b
Reviewed-on: http://gerrit.cloudera.org:8080/23358
Reviewed-by: Abhishek Chennaka <ac...@cloudera.com>
Reviewed-by: Alexey Serbin <al...@apache.org>
Reviewed-by: Gabriella Lotz <lo...@gmail.com>
Tested-by: Marton Greber <gr...@gmail.com>
---
.../org/apache/kudu/backup/TableMetadata.scala | 24 +++-
.../org/apache/kudu/backup/TestKuduBackup.scala | 133 +++++++++++++++++++++
2 files changed, 153 insertions(+), 4 deletions(-)
diff --git a/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
index b52592f7d..d532e744e 100644
--- a/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
+++ b/java/kudu-backup-common/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
@@ -18,12 +18,13 @@
package org.apache.kudu.backup
import java.math.BigDecimal
+import java.nio.ByteBuffer
import java.sql.Date
import java.util
import com.google.protobuf.ByteString
import com.google.protobuf.StringValue
-import org.apache.commons.net.util.Base64
+import java.util.Base64
import org.apache.kudu.backup.Backup._
import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
import org.apache.kudu.ColumnSchema.CompressionAlgorithm
@@ -294,7 +295,12 @@ object TableMetadata {
case Type.DOUBLE => row.getDouble(columnName)
case Type.VARCHAR => row.getVarchar(columnName)
case Type.STRING => row.getString(columnName)
- case Type.BINARY => row.getBinary(columnName)
+ case Type.BINARY => {
+ val byteBuffer = row.getBinary(columnName)
+ val byteArray = new Array[Byte](byteBuffer.remaining())
+ byteBuffer.get(byteArray)
+ byteArray
+ }
case Type.DECIMAL => row.getDecimal(columnName)
case Type.DATE => row.getDate(columnName)
case _ =>
@@ -348,7 +354,17 @@ object TableMetadata {
case Type.STRING =>
value.asInstanceOf[String]
case Type.BINARY =>
- Base64.encodeBase64String(value.asInstanceOf[Array[Byte]])
+ val byteArray = value match {
+ case arr: Array[Byte] => arr
+ case buf: java.nio.ByteBuffer => {
+ val array = new Array[Byte](buf.remaining())
+ buf.get(array)
+ array
+ }
+ case _ =>
+ throw new IllegalArgumentException(s"Unsupported binary value type: ${value.getClass}")
+ }
+ Base64.getEncoder.encodeToString(byteArray)
case Type.DECIMAL =>
value
.asInstanceOf[BigDecimal]
@@ -371,7 +387,7 @@ object TableMetadata {
case Type.DOUBLE => value.toDouble
case Type.VARCHAR => value
case Type.STRING => value
- case Type.BINARY => Base64.decodeBase64(value)
+ case Type.BINARY => Base64.getDecoder.decode(value)
case Type.DECIMAL => new BigDecimal(value) // TODO: Explicitly pass scale
case Type.DATE => Date.valueOf(value)
case _ =>
diff --git a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
index ea13fcf62..cdf69811e 100644
--- a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
+++ b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
@@ -39,6 +39,7 @@ import org.apache.kudu.util.HybridTimeUtil
import org.apache.kudu.util.SchemaGenerator.SchemaGeneratorBuilder
import org.apache.spark.scheduler.SparkListener
import org.apache.spark.scheduler.SparkListenerJobEnd
+
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotEquals
@@ -335,6 +336,138 @@ class TestKuduBackup extends KuduTestSuite {
restoreAndValidateTable(tableName, 0)
}
+ @Test
+ def testBinaryColumnBackupReproducesBase64Issue(): Unit = {
+ // This test reproduces the latent Base64 issue.
+ // Before the fix, this would fail with:
+ // java.lang.NoClassDefFoundError: org/apache/commons/net/util/Base64
+ // The issue occurs when backup metadata processing encounters BINARY data
+ // and tries to serialize it.
+
+ val tableName = "binary-table-base64-issue"
+
+ // Create a schema with BINARY column that has a default value
+ // This triggers TableMetadata.valueToString():
+ // builder.setDefaultValue(StringValue.of(valueToString(col.getDefaultValue, col.getType)))
+ val binaryDefaultValue = Array[Byte](0x01.toByte, 0x02.toByte, 0x03.toByte, 0x04.toByte)
+ val binaryColumnSchema = new ColumnSchemaBuilder("binary_data", Type.BINARY)
+ .defaultValue(binaryDefaultValue) // This triggers the Base64 issue!
+ .build()
+
+ val keyColumnSchema = new ColumnSchemaBuilder("key", Type.INT32)
+ .key(true)
+ .build()
+
+ val binaryTableSchema = new Schema(util.Arrays.asList(keyColumnSchema, binaryColumnSchema))
+
+ val tableOptions = new CreateTableOptions()
+ .setRangePartitionColumns(List("key").asJava)
+ .setNumReplicas(1)
+
+ val binaryTable = kuduClient.createTable(tableName, binaryTableSchema, tableOptions)
+
+ val session = kuduClient.newSession()
+ for (i <- 0 until 5) {
+ val insert = binaryTable.newInsert()
+ val row = insert.getRow
+ row.addInt("key", i)
+ row.addBinary("binary_data", Array[Byte](i.toByte, (i + 1).toByte, (i + 2).toByte))
+ session.apply(insert)
+ }
+ session.flush()
+
+ // This backup operation would have failed with NoClassDefFoundError before the Base64 fix
+ // because processing the BINARY column's default value calls:
+ // TableMetadata.valueToString(col.getDefaultValue, col.getType)
+ // which calls Base64.encodeBase64String() for Type.BINARY
+ val backupOptions = createBackupOptions(Seq(tableName))
+
+ // The key assertion for ENCODE path: this should NOT throw NoClassDefFoundError
+ // Tests: TableMetadata.valueToString() -> Base64.encodeBase64String()
+ assertTrue(s"Backup failed - Base64 encode issue!", runBackup(backupOptions))
+
+ // Now test the DECODE path: restore the backup with BINARY default values
+ // Tests: TableMetadata.valueFromString() -> Base64.decodeBase64()
+ val restoreOptions = createRestoreOptions(Seq(tableName))
+ assertTrue(s"Restore failed - Base64 decode issue!", runRestore(restoreOptions))
+
+ // Validate the restored table has the correct data (verifies round-trip integrity)
+ validateTablesMatch(tableName, s"$tableName-restore")
+ }
+
+ @Test
+ def testBinaryPartitionBoundariesBase64Issue(): Unit = {
+ // This test reproduces the Base64 issue via a different code path:
+ // BINARY columns used in range partition boundaries trigger valueToString()
+ // in getBoundValues(): .setValue(valueToString(value, colType))
+
+ val tableName = "binary-partition-base64-issue"
+
+ val binaryKeySchema = new ColumnSchemaBuilder("binary_key", Type.BINARY)
+ .key(true)
+ .build()
+
+ val valueColumnSchema = new ColumnSchemaBuilder("value", Type.STRING)
+ .nullable(true)
+ .build()
+
+ val binaryPartitionSchema = new Schema(util.Arrays.asList(binaryKeySchema, valueColumnSchema))
+
+ val tableOptions = new CreateTableOptions()
+ .setRangePartitionColumns(List("binary_key").asJava)
+ .setNumReplicas(1)
+
+ // Add specific range partitions with BINARY boundaries
+ // This will create partition metadata that needs to be serialized during backup
+ val lowerBound1 = binaryPartitionSchema.newPartialRow()
+ lowerBound1.addBinary("binary_key", Array[Byte](0x00.toByte, 0x00.toByte))
+ val upperBound1 = binaryPartitionSchema.newPartialRow()
+ upperBound1.addBinary("binary_key", Array[Byte](0x10.toByte, 0x00.toByte))
+ tableOptions.addRangePartition(lowerBound1, upperBound1)
+
+ val lowerBound2 = binaryPartitionSchema.newPartialRow()
+ lowerBound2.addBinary("binary_key", Array[Byte](0x10.toByte, 0x00.toByte))
+ val upperBound2 = binaryPartitionSchema.newPartialRow()
+ upperBound2.addBinary("binary_key", Array[Byte](0x20.toByte, 0x00.toByte))
+ tableOptions.addRangePartition(lowerBound2, upperBound2)
+
+ val binaryPartitionTable =
+ kuduClient.createTable(tableName, binaryPartitionSchema, tableOptions)
+
+ val session = kuduClient.newSession()
+
+ // Insert into first partition
+ val insert1 = binaryPartitionTable.newInsert()
+ insert1.getRow.addBinary("binary_key", Array[Byte](0x05.toByte, 0x00.toByte))
+ insert1.getRow.addString("value", "partition1-data")
+ session.apply(insert1)
+
+ // Insert into second partition
+ val insert2 = binaryPartitionTable.newInsert()
+ insert2.getRow.addBinary("binary_key", Array[Byte](0x15.toByte, 0x00.toByte))
+ insert2.getRow.addString("value", "partition2-data")
+ session.apply(insert2)
+
+ session.flush()
+
+ // This backup would have failed because getRangePartitionMetadata() calls getBoundValues()
+ // which calls valueToString() for BINARY partition boundary values
+ // The stack trace would be:
+ // getRangePartitionMetadata() -> getBoundValues() ->
+ // -> valueToString() -> Base64.encodeBase64String()
+ val backupOptions = createBackupOptions(Seq(tableName))
+
+ assertTrue(
+ s"Backup failed - Base64 partition boundary issue not fixed!",
+ runBackup(backupOptions))
+
+ // Verify restore also works
+ val restoreOptions = createRestoreOptions(Seq(tableName))
+ assertTrue(s"Restore failed", runRestore(restoreOptions))
+
+ validateTablesMatch(tableName, s"$tableName-restore")
+ }
+
@Test
def testRandomBackupAndRestore() {
val table = createRandomTable()