You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2023/11/19 23:57:02 UTC
(iceberg) branch main updated: Spark: Fix metadata delete check with branches (#9102)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new c193de9e8f Spark: Fix metadata delete check with branches (#9102)
c193de9e8f is described below
commit c193de9e8f6ef8195506eeebe4a3721cf9884bd9
Author: Amogh Jahagirdar <am...@tabular.io>
AuthorDate: Sun Nov 19 15:56:55 2023 -0800
Spark: Fix metadata delete check with branches (#9102)
---
.../iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java | 2 +-
.../src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java | 3 +--
.../src/main/java/org/apache/iceberg/spark/source/SparkTable.java | 2 +-
.../spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java | 2 +-
4 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
index 16ab98f442..e4605a7658 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
@@ -135,7 +135,7 @@ public abstract class SparkRowLevelOperationsTestBase extends SparkExtensionsTes
true,
WRITE_DISTRIBUTION_MODE_NONE,
false,
- null,
+ "test",
DISTRIBUTED
},
{
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java
index bdc4be08b4..9bc46d0571 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java
@@ -326,9 +326,8 @@ public abstract class TestDelete extends SparkRowLevelOperationsTestBase {
public void testDeleteFileThenMetadataDelete() throws Exception {
Assume.assumeFalse("Avro does not support metadata delete", fileFormat.equals("avro"));
createAndInitUnpartitionedTable();
-
- sql("INSERT INTO TABLE %s VALUES (1, 'hr'), (2, 'hardware'), (null, 'hr')", tableName);
createBranchIfNeeded();
+ sql("INSERT INTO TABLE %s VALUES (1, 'hr'), (2, 'hardware'), (null, 'hr')", commitTarget());
// MOR mode: writes a delete file as null cannot be deleted by metadata
sql("DELETE FROM %s AS t WHERE t.id IS NULL", commitTarget());
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
index b540165b2e..e200bee03e 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
@@ -330,7 +330,7 @@ public class SparkTable
.ignoreResiduals();
if (branch != null) {
- scan.useRef(branch);
+ scan = scan.useRef(branch);
}
try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java
index ca9d14f05e..8e6b576ddf 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java
@@ -806,7 +806,7 @@ public class TestHelpers {
public static List<DataFile> dataFiles(Table table, String branch) {
TableScan scan = table.newScan();
if (branch != null) {
- scan.useRef(branch);
+ scan = scan.useRef(branch);
}
CloseableIterable<FileScanTask> tasks = scan.includeColumnStats().planFiles();