You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2023/04/18 05:02:30 UTC
[iceberg] branch master updated: Spark: Simplify checks of output-spec-id in SparkWriteConf (#7348)
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 2a087875bc Spark: Simplify checks of output-spec-id in SparkWriteConf (#7348)
2a087875bc is described below
commit 2a087875bc63a0d151575863ef47f4125fa8dbf6
Author: Gustavo Torres <gu...@users.noreply.github.com>
AuthorDate: Tue Apr 18 01:02:24 2023 -0400
Spark: Simplify checks of output-spec-id in SparkWriteConf (#7348)
---
.../main/java/org/apache/iceberg/spark/SparkWriteConf.java | 11 ++++-------
.../main/java/org/apache/iceberg/spark/SparkWriteConf.java | 9 ++-------
.../main/java/org/apache/iceberg/spark/SparkWriteConf.java | 9 ++-------
3 files changed, 8 insertions(+), 21 deletions(-)
diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
index e964aae1ac..179a51faa6 100644
--- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
+++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
@@ -20,7 +20,6 @@ package org.apache.iceberg.spark;
import java.util.Locale;
import java.util.Map;
-import java.util.Set;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.SnapshotSummary;
import org.apache.iceberg.Table;
@@ -50,18 +49,16 @@ import org.apache.spark.sql.SparkSession;
*/
public class SparkWriteConf {
+ private final Table table;
private final RuntimeConfig sessionConf;
private final Map<String, String> writeOptions;
private final SparkConfParser confParser;
- private final int currentSpecId;
- private final Set<Integer> partitionSpecIds;
public SparkWriteConf(SparkSession spark, Table table, Map<String, String> writeOptions) {
+ this.table = table;
this.sessionConf = spark.conf();
this.writeOptions = writeOptions;
this.confParser = new SparkConfParser(spark, table, writeOptions);
- this.currentSpecId = table.spec().specId();
- this.partitionSpecIds = table.specs().keySet();
}
public boolean checkNullability() {
@@ -126,10 +123,10 @@ public class SparkWriteConf {
confParser
.intConf()
.option(SparkWriteOptions.OUTPUT_SPEC_ID)
- .defaultValue(currentSpecId)
+ .defaultValue(table.spec().specId())
.parse();
Preconditions.checkArgument(
- partitionSpecIds.contains(outputSpecId),
+ table.specs().containsKey(outputSpecId),
"Output spec id %s is not a valid spec id for table",
outputSpecId);
return outputSpecId;
diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
index 5d5c31b722..63945e3c16 100644
--- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
+++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
@@ -24,7 +24,6 @@ import static org.apache.iceberg.DistributionMode.RANGE;
import java.util.Locale;
import java.util.Map;
-import java.util.Set;
import org.apache.iceberg.DistributionMode;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.IsolationLevel;
@@ -60,16 +59,12 @@ public class SparkWriteConf {
private final RuntimeConfig sessionConf;
private final Map<String, String> writeOptions;
private final SparkConfParser confParser;
- private final int currentSpecId;
- private final Set<Integer> partitionSpecIds;
public SparkWriteConf(SparkSession spark, Table table, Map<String, String> writeOptions) {
this.table = table;
this.sessionConf = spark.conf();
this.writeOptions = writeOptions;
this.confParser = new SparkConfParser(spark, table, writeOptions);
- this.currentSpecId = table.spec().specId();
- this.partitionSpecIds = table.specs().keySet();
}
public boolean checkNullability() {
@@ -134,10 +129,10 @@ public class SparkWriteConf {
confParser
.intConf()
.option(SparkWriteOptions.OUTPUT_SPEC_ID)
- .defaultValue(currentSpecId)
+ .defaultValue(table.spec().specId())
.parse();
Preconditions.checkArgument(
- partitionSpecIds.contains(outputSpecId),
+ table.specs().containsKey(outputSpecId),
"Output spec id %s is not a valid spec id for table",
outputSpecId);
return outputSpecId;
diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
index 40d67717d1..41777c5155 100644
--- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
+++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
@@ -24,7 +24,6 @@ import static org.apache.iceberg.DistributionMode.RANGE;
import java.util.Locale;
import java.util.Map;
-import java.util.Set;
import org.apache.iceberg.DistributionMode;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.IsolationLevel;
@@ -63,8 +62,6 @@ public class SparkWriteConf {
private final RuntimeConfig sessionConf;
private final Map<String, String> writeOptions;
private final SparkConfParser confParser;
- private final int currentSpecId;
- private final Set<Integer> partitionSpecIds;
public SparkWriteConf(SparkSession spark, Table table, Map<String, String> writeOptions) {
this(spark, table, null, writeOptions);
@@ -77,8 +74,6 @@ public class SparkWriteConf {
this.sessionConf = spark.conf();
this.writeOptions = writeOptions;
this.confParser = new SparkConfParser(spark, table, writeOptions);
- this.currentSpecId = table.spec().specId();
- this.partitionSpecIds = table.specs().keySet();
}
public boolean checkNullability() {
@@ -152,10 +147,10 @@ public class SparkWriteConf {
confParser
.intConf()
.option(SparkWriteOptions.OUTPUT_SPEC_ID)
- .defaultValue(currentSpecId)
+ .defaultValue(table.spec().specId())
.parse();
Preconditions.checkArgument(
- partitionSpecIds.contains(outputSpecId),
+ table.specs().containsKey(outputSpecId),
"Output spec id %s is not a valid spec id for table",
outputSpecId);
return outputSpecId;