You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/12/28 21:55:20 UTC
[iceberg] branch master updated: Spark: Use constants for DF read
and write options (#1933)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new cbb2447 Spark: Use constants for DF read and write options (#1933)
cbb2447 is described below
commit cbb244759ae218b85a7e68a1568fddaf815aab3c
Author: Karuppayya <ka...@gmail.com>
AuthorDate: Mon Dec 28 13:55:09 2020 -0800
Spark: Use constants for DF read and write options (#1933)
---
site/docs/configuration.md | 4 ++
.../org/apache/iceberg/spark/SparkReadOptions.java | 50 ++++++++++++++++
.../apache/iceberg/spark/SparkWriteOptions.java | 18 ++++++
.../iceberg/spark/source/TestDataFrameWrites.java | 4 +-
.../spark/source/TestDataSourceOptions.java | 18 +++---
.../spark/source/TestIdentityPartitionData.java | 5 +-
.../iceberg/spark/source/TestPartitionPruning.java | 3 +-
.../iceberg/spark/source/TestPartitionValues.java | 37 ++++++------
.../spark/source/TestSnapshotSelection.java | 9 +--
.../iceberg/spark/source/TestSparkDataWrite.java | 67 +++++++++++-----------
.../spark/source/TestSparkReadProjection.java | 3 +-
.../spark/source/TestWriteMetricsConfig.java | 22 +++----
.../orc/IcebergSourceFlatORCDataReadBenchmark.java | 5 +-
.../IcebergSourceNestedORCDataReadBenchmark.java | 5 +-
.../apache/iceberg/spark/source/IcebergSource.java | 5 +-
.../org/apache/iceberg/spark/source/Reader.java | 16 +++---
.../org/apache/iceberg/spark/source/Writer.java | 4 +-
.../iceberg/spark/source/TestFilteredScan.java | 7 ++-
.../java/org/apache/iceberg/spark/Spark3Util.java | 4 +-
.../iceberg/spark/source/SparkBatchQueryScan.java | 15 ++---
.../iceberg/spark/source/SparkMergeScan.java | 7 ++-
.../apache/iceberg/spark/source/SparkWrite.java | 4 +-
.../iceberg/spark/source/SparkWriteBuilder.java | 3 +-
.../iceberg/spark/source/TestFilteredScan.java | 7 ++-
24 files changed, 210 insertions(+), 112 deletions(-)
diff --git a/site/docs/configuration.md b/site/docs/configuration.md
index 40727fb..8c75f04 100644
--- a/site/docs/configuration.md
+++ b/site/docs/configuration.md
@@ -146,6 +146,8 @@ spark.read
| split-size | As per table property | Overrides this table's read.split.target-size and read.split.metadata-target-size |
| lookback | As per table property | Overrides this table's read.split.planning-lookback |
| file-open-cost | As per table property | Overrides this table's read.split.open-file-cost |
+| vectorization-enabled | As per table property | Overrides this table's read.parquet.vectorization.enabled |
+| batch-size | As per table property | Overrides this table's read.parquet.vectorization.batch-size |
### Write options
@@ -155,6 +157,7 @@ Spark write options are passed when configuring the DataFrameWriter, like this:
// write with Avro instead of Parquet
df.write
.option("write-format", "avro")
+ .option("snapshot-property.key", "value")
.insertInto("catalog.db.table")
```
@@ -165,4 +168,5 @@ df.write
| check-nullability | true | Sets the nullable check on fields |
| snapshot-property._custom-key_ | null | Adds an entry with custom-key and corresponding value in the snapshot summary |
| fanout-enabled | false | Overrides this table's write.spark.fanout.enabled |
+| check-ordering | true | Checks if input schema and table schema are same |
diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java b/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java
new file mode 100644
index 0000000..e0242e0
--- /dev/null
+++ b/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark;
+
+/**
+ * Spark DF read options
+ */
+public class SparkReadOptions {
+
+ private SparkReadOptions() {
+ }
+
+ // Snapshot ID of the table snapshot to read
+ public static final String SNAPSHOT_ID = "snapshot-id";
+
+ // A timestamp in milliseconds; the snapshot used will be the snapshot current at this time.
+ public static final String AS_OF_TIMESTAMP = "as-of-timestamp";
+
+ // Overrides the table's read.split.target-size and read.split.metadata-target-size
+ public static final String SPLIT_SIZE = "split-size";
+
+ // Overrides the table's read.split.planning-lookback
+ public static final String LOOKBACK = "lookback";
+
+ // Overrides the table's read.split.open-file-cost
+ public static final String FILE_OPEN_COST = "file-open-cost";
+
+ // Overrides the table's read.split.open-file-cost
+ public static final String VECTORIZATION_ENABLED = "vectorization-enabled";
+
+ // Overrides the table's read.parquet.vectorization.batch-size
+ public static final String VECTORIZATION_BATCH_SIZE = "batch-size";
+}
diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java b/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
index 4844c3c..9dfa082 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
@@ -27,6 +27,24 @@ public class SparkWriteOptions {
private SparkWriteOptions() {
}
+ // Fileformat for write operations(default: Table write.format.default )
+ public static final String WRITE_FORMAT = "write-format";
+
+ // Overrides this table's write.target-file-size-bytes
+ public static final String TARGET_FILE_SIZE_BYTES = "target-file-size-bytes";
+
+ // Sets the nullable check on fields(default: true)
+ public static final String CHECK_NULLABILITY = "check-nullability";
+
+ // Adds an entry with custom-key and corresponding value in the snapshot summary
+ // ex: df.write().format(iceberg)
+ // .option(SparkWriteOptions.SNAPSHOT_PROPERTY_PREFIX."key1", "value1")
+ // .save(location)
+ public static final String SNAPSHOT_PROPERTY_PREFIX = "snapshot-property";
+
// Overrides table property write.spark.fanout.enabled(default: false)
public static final String FANOUT_ENABLED = "fanout-enabled";
+
+ // Checks if input schema and table schema are same(default: true)
+ public static final String CHECK_ORDERING = "check-ordering";
}
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java
index a25cf7d..015b819 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java
@@ -42,6 +42,7 @@ import org.apache.iceberg.io.FileAppender;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.SparkSchemaUtil;
+import org.apache.iceberg.spark.SparkWriteOptions;
import org.apache.iceberg.spark.data.AvroDataTest;
import org.apache.iceberg.spark.data.RandomData;
import org.apache.iceberg.spark.data.SparkAvroReader;
@@ -295,7 +296,8 @@ public abstract class TestDataFrameWrites extends AvroDataTest {
// read from parquet and append to iceberg w/ nullability check disabled
spark
.read().schema(SparkSchemaUtil.convert(icebergSchema)).parquet(sourcePath)
- .write().format("iceberg").option("check-nullability", false).mode(SaveMode.Append).save(targetPath);
+ .write().format("iceberg").option(SparkWriteOptions.CHECK_NULLABILITY, false)
+ .mode(SaveMode.Append).save(targetPath);
// read all data
List<Row> rows = spark.read().format("iceberg").load(targetPath).collectAsList();
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java
index cf1b8c9..aea1687 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java
@@ -35,11 +35,14 @@ import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.spark.SparkReadOptions;
+import org.apache.iceberg.spark.SparkWriteOptions;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.SnapshotUtil;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -92,8 +95,8 @@ public abstract class TestDataSourceOptions {
Dataset<Row> df = spark.createDataFrame(expectedRecords, SimpleRecord.class);
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", "parquet")
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, "parquet")
+ .mode(SaveMode.Append)
.save(tableLocation);
try (CloseableIterable<FileScanTask> tasks = table.newScan().planFiles()) {
@@ -196,7 +199,7 @@ public abstract class TestDataSourceOptions {
Dataset<Row> resultDf = spark.read()
.format("iceberg")
- .option("split-size", String.valueOf(611)) // 611 bytes is the size of SimpleRecord(1,"a")
+ .option(SparkReadOptions.SPLIT_SIZE, String.valueOf(611)) // 611 bytes is the size of SimpleRecord(1,"a")
.load(tableLocation);
Assert.assertEquals("Spark partitions should match", 2, resultDf.javaRDD().getNumPartitions());
@@ -247,7 +250,8 @@ public abstract class TestDataSourceOptions {
() -> {
spark.read()
.format("iceberg")
- .option("as-of-timestamp", Long.toString(table.snapshot(snapshotIds.get(3)).timestampMillis()))
+ .option(SparkReadOptions.AS_OF_TIMESTAMP,
+ Long.toString(table.snapshot(snapshotIds.get(3)).timestampMillis()))
.option("end-snapshot-id", snapshotIds.get(2).toString())
.load(tableLocation).explain();
});
@@ -328,7 +332,7 @@ public abstract class TestDataSourceOptions {
// override the table property using options
entriesDf = spark.read()
.format("iceberg")
- .option("split-size", String.valueOf(128 * 1024 * 1024))
+ .option(SparkReadOptions.SPLIT_SIZE, String.valueOf(128 * 1024 * 1024))
.load(tableLocation + "#entries");
Assert.assertEquals("Num partitions must match", 1, entriesDf.javaRDD().getNumPartitions());
}
@@ -379,8 +383,8 @@ public abstract class TestDataSourceOptions {
originalDf.select("id", "data").write()
.format("iceberg")
.mode("append")
- .option("snapshot-property.extra-key", "someValue")
- .option("snapshot-property.another-key", "anotherValue")
+ .option(SparkWriteOptions.SNAPSHOT_PROPERTY_PREFIX + ".extra-key", "someValue")
+ .option(SparkWriteOptions.SNAPSHOT_PROPERTY_PREFIX + ".another-key", "anotherValue")
.save(tableLocation);
Table table = tables.load(tableLocation);
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java
index 88c8b93..2e02a7c 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java
@@ -31,6 +31,7 @@ import org.apache.iceberg.TableProperties;
import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.spark.SparkTableUtil;
import org.apache.iceberg.spark.SparkTestBase;
@@ -139,7 +140,7 @@ public abstract class TestIdentityPartitionData extends SparkTestBase {
public void testFullProjection() {
List<Row> expected = logs.orderBy("id").collectAsList();
List<Row> actual = spark.read().format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(table.location()).orderBy("id")
.select("id", "date", "level", "message")
.collectAsList();
@@ -174,7 +175,7 @@ public abstract class TestIdentityPartitionData extends SparkTestBase {
List<Row> expected = logs.select("id", ordering).orderBy("id").collectAsList();
List<Row> actual = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(table.location())
.select("id", ordering).orderBy("id")
.collectAsList();
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
index 9159563..79e6603 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
@@ -47,6 +47,7 @@ import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
@@ -254,7 +255,7 @@ public abstract class TestPartitionPruning {
List<Row> actual = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(table.location())
.select("id", "date", "level", "message", "timestamp")
.filter(filterCond)
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
index 7747281..3cee81e 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
@@ -33,6 +33,8 @@ import org.apache.iceberg.avro.Avro;
import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.io.FileAppender;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.SparkReadOptions;
+import org.apache.iceberg.spark.SparkWriteOptions;
import org.apache.iceberg.spark.data.RandomData;
import org.apache.iceberg.spark.data.TestHelpers;
import org.apache.iceberg.types.Types;
@@ -40,6 +42,7 @@ import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
@@ -143,12 +146,12 @@ public abstract class TestPartitionValues {
df.select("id", "data").write()
.format("iceberg")
- .mode("append")
+ .mode(SaveMode.Append)
.save(location.toString());
Dataset<Row> result = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(location.toString());
List<SimpleRecord> actual = result
@@ -182,13 +185,13 @@ public abstract class TestPartitionValues {
df.select("data", "id").write()
.format("iceberg")
- .mode("append")
- .option("check-ordering", "false")
+ .mode(SaveMode.Append)
+ .option(SparkWriteOptions.CHECK_ORDERING, "false")
.save(location.toString());
Dataset<Row> result = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(location.toString());
List<SimpleRecord> actual = result
@@ -222,14 +225,14 @@ public abstract class TestPartitionValues {
df.select("data", "id").write()
.format("iceberg")
- .mode("append")
- .option("check-ordering", "false")
- .option("check-nullability", "false")
+ .mode(SaveMode.Append)
+ .option(SparkWriteOptions.CHECK_ORDERING, "false")
+ .option(SparkWriteOptions.CHECK_NULLABILITY, "false")
.save(location.toString());
Dataset<Row> result = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(location.toString());
List<SimpleRecord> actual = result
@@ -272,7 +275,7 @@ public abstract class TestPartitionValues {
.commit();
Dataset<Row> sourceDF = spark.read().format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(sourceLocation);
for (String column : columnNames) {
@@ -290,12 +293,12 @@ public abstract class TestPartitionValues {
sourceDF.write()
.format("iceberg")
- .mode("append")
+ .mode(SaveMode.Append)
.save(location.toString());
List<Row> actual = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(location.toString())
.collectAsList();
@@ -340,7 +343,7 @@ public abstract class TestPartitionValues {
.commit();
Dataset<Row> sourceDF = spark.read().format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(sourceLocation);
for (String column : columnNames) {
@@ -358,12 +361,12 @@ public abstract class TestPartitionValues {
sourceDF.write()
.format("iceberg")
- .mode("append")
+ .mode(SaveMode.Append)
.save(location.toString());
List<Row> actual = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(location.toString())
.collectAsList();
@@ -416,13 +419,13 @@ public abstract class TestPartitionValues {
// write into iceberg
sourceDF.write()
.format("iceberg")
- .mode("append")
+ .mode(SaveMode.Append)
.save(baseLocation);
// verify
List<Row> actual = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(baseLocation)
.collectAsList();
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java
index 9ea7d26..8aadfb8 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java
@@ -29,6 +29,7 @@ import org.apache.iceberg.Table;
import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.types.Types;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
@@ -167,7 +168,7 @@ public abstract class TestSnapshotSelection {
// verify records in the previous snapshot
Dataset<Row> previousSnapshotResult = spark.read()
.format("iceberg")
- .option("as-of-timestamp", firstSnapshotTimestamp)
+ .option(SparkReadOptions.AS_OF_TIMESTAMP, firstSnapshotTimestamp)
.load(tableLocation);
List<SimpleRecord> previousSnapshotRecords = previousSnapshotResult.orderBy("id")
.as(Encoders.bean(SimpleRecord.class))
@@ -202,7 +203,7 @@ public abstract class TestSnapshotSelection {
Dataset<Row> df = spark.read()
.format("iceberg")
- .option("as-of-timestamp", timestamp)
+ .option(SparkReadOptions.AS_OF_TIMESTAMP, timestamp)
.load(tableLocation);
df.collectAsList();
@@ -228,8 +229,8 @@ public abstract class TestSnapshotSelection {
long snapshotId = table.currentSnapshot().snapshotId();
Dataset<Row> df = spark.read()
.format("iceberg")
- .option("snapshot-id", snapshotId)
- .option("as-of-timestamp", timestamp)
+ .option(SparkReadOptions.SNAPSHOT_ID, snapshotId)
+ .option(SparkReadOptions.AS_OF_TIMESTAMP, timestamp)
.load(tableLocation);
df.collectAsList();
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
index 942c5b0..34caab4 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java
@@ -39,6 +39,7 @@ import org.apache.iceberg.types.Types;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -106,8 +107,8 @@ public abstract class TestSparkDataWrite {
// TODO: incoming columns must be ordered according to the table's schema
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
table.refresh();
@@ -166,14 +167,14 @@ public abstract class TestSparkDataWrite {
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
df.withColumn("id", df.col("id").plus(3)).select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
table.refresh();
@@ -214,15 +215,15 @@ public abstract class TestSparkDataWrite {
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
// overwrite with 2*id to replace record 2, append 4 and 6
df.withColumn("id", df.col("id").multiply(2)).select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("overwrite")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Overwrite)
.option("overwrite-mode", "dynamic")
.save(location.toString());
@@ -256,15 +257,15 @@ public abstract class TestSparkDataWrite {
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
// overwrite with the same data; should not produce two copies
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("overwrite")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Overwrite)
.save(location.toString());
table.refresh();
@@ -300,8 +301,8 @@ public abstract class TestSparkDataWrite {
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
table.refresh();
@@ -365,8 +366,8 @@ public abstract class TestSparkDataWrite {
df.select("id").write() // select only id column
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
table.refresh();
@@ -408,8 +409,8 @@ public abstract class TestSparkDataWrite {
df.select("c1", "c3").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
table.refresh();
@@ -442,8 +443,8 @@ public abstract class TestSparkDataWrite {
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
Dataset<Row> query = spark.read()
@@ -461,8 +462,8 @@ public abstract class TestSparkDataWrite {
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
.save(location.toString());
List<SimpleRecord> actual2 = spark.table("tmp").as(Encoders.bean(SimpleRecord.class)).collectAsList();
@@ -497,26 +498,26 @@ public abstract class TestSparkDataWrite {
case NONE:
df.select("id", "data").sort("data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
- .option("target-file-size-bytes", 4) // ~4 bytes; low enough to trigger
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
+ .option(SparkWriteOptions.TARGET_FILE_SIZE_BYTES, 4) // ~4 bytes; low enough to trigger
.save(location.toString());
break;
case TABLE:
table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit();
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
- .option("target-file-size-bytes", 4) // ~4 bytes; low enough to trigger
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
+ .option(SparkWriteOptions.TARGET_FILE_SIZE_BYTES, 4) // ~4 bytes; low enough to trigger
.save(location.toString());
break;
case JOB:
df.select("id", "data").write()
.format("iceberg")
- .option("write-format", format.toString())
- .mode("append")
- .option("target-file-size-bytes", 4) // ~4 bytes; low enough to trigger
+ .option(SparkWriteOptions.WRITE_FORMAT, format.toString())
+ .mode(SaveMode.Append)
+ .option(SparkWriteOptions.TARGET_FILE_SIZE_BYTES, 4) // ~4 bytes; low enough to trigger
.option(SparkWriteOptions.FANOUT_ENABLED, true)
.save(location.toString());
break;
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java
index 8a289db..2af2097 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java
@@ -37,6 +37,7 @@ import org.apache.iceberg.io.FileAppender;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.SparkValueConverter;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
@@ -144,7 +145,7 @@ public abstract class TestSparkReadProjection extends TestReadProjection {
Dataset<Row> df = spark.read()
.format("org.apache.iceberg.spark.source.TestIcebergSource")
.option("iceberg.table.name", desc)
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load();
return SparkValueConverter.convert(readSchema, df.collectAsList().get(0));
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java
index 0782120..2cf1bb6 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java
@@ -33,6 +33,7 @@ import org.apache.iceberg.TableProperties;
import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.spark.SparkWriteOptions;
import org.apache.iceberg.spark.data.RandomData;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.ByteBuffers;
@@ -40,6 +41,7 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.catalyst.InternalRow;
import org.junit.AfterClass;
@@ -109,8 +111,8 @@ public abstract class TestWriteMetricsConfig {
.coalesce(1)
.write()
.format("iceberg")
- .option("write-format", "parquet")
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, "parquet")
+ .mode(SaveMode.Append)
.save(tableLocation);
for (FileScanTask task : table.newScan().includeColumnStats().planFiles()) {
@@ -142,8 +144,8 @@ public abstract class TestWriteMetricsConfig {
.coalesce(1)
.write()
.format("iceberg")
- .option("write-format", "parquet")
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, "parquet")
+ .mode(SaveMode.Append)
.save(tableLocation);
for (FileScanTask task : table.newScan().includeColumnStats().planFiles()) {
@@ -175,8 +177,8 @@ public abstract class TestWriteMetricsConfig {
.coalesce(1)
.write()
.format("iceberg")
- .option("write-format", "parquet")
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, "parquet")
+ .mode(SaveMode.Append)
.save(tableLocation);
for (FileScanTask task : table.newScan().includeColumnStats().planFiles()) {
@@ -209,8 +211,8 @@ public abstract class TestWriteMetricsConfig {
.coalesce(1)
.write()
.format("iceberg")
- .option("write-format", "parquet")
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, "parquet")
+ .mode(SaveMode.Append)
.save(tableLocation);
Schema schema = table.schema();
@@ -247,8 +249,8 @@ public abstract class TestWriteMetricsConfig {
df.coalesce(1).write()
.format("iceberg")
- .option("write-format", "parquet")
- .mode("append")
+ .option(SparkWriteOptions.WRITE_FORMAT, "parquet")
+ .mode(SaveMode.Append)
.save(tableLocation);
Schema schema = table.schema();
diff --git a/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java b/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java
index b69b6a1..0ed8b6b 100644
--- a/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java
+++ b/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java
@@ -22,6 +22,7 @@ package org.apache.iceberg.spark.source.orc;
import java.io.IOException;
import java.util.Map;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.internal.SQLConf;
@@ -83,7 +84,7 @@ public class IcebergSourceFlatORCDataReadBenchmark extends IcebergSourceFlatORCD
tableProperties.put(SPLIT_OPEN_FILE_COST, Integer.toString(128 * 1024 * 1024));
withTableProperties(tableProperties, () -> {
String tableLocation = table().location();
- Dataset<Row> df = spark().read().option("vectorization-enabled", "true")
+ Dataset<Row> df = spark().read().option(SparkReadOptions.VECTORIZATION_ENABLED, "true")
.format("iceberg").load(tableLocation);
materialize(df);
});
@@ -132,7 +133,7 @@ public class IcebergSourceFlatORCDataReadBenchmark extends IcebergSourceFlatORCD
tableProperties.put(SPLIT_OPEN_FILE_COST, Integer.toString(128 * 1024 * 1024));
withTableProperties(tableProperties, () -> {
String tableLocation = table().location();
- Dataset<Row> df = spark().read().option("vectorization-enabled", "true")
+ Dataset<Row> df = spark().read().option(SparkReadOptions.VECTORIZATION_ENABLED, "true")
.format("iceberg").load(tableLocation).select("longCol");
materialize(df);
});
diff --git a/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java b/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java
index b6c6235..bfbf4cb 100644
--- a/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java
+++ b/spark2/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java
@@ -22,6 +22,7 @@ package org.apache.iceberg.spark.source.orc;
import java.io.IOException;
import java.util.Map;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.source.IcebergSourceNestedDataBenchmark;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -85,7 +86,7 @@ public class IcebergSourceNestedORCDataReadBenchmark extends IcebergSourceNested
tableProperties.put(SPLIT_OPEN_FILE_COST, Integer.toString(128 * 1024 * 1024));
withTableProperties(tableProperties, () -> {
String tableLocation = table().location();
- Dataset<Row> df = spark().read().option("vectorization-enabled", "true")
+ Dataset<Row> df = spark().read().option(SparkReadOptions.VECTORIZATION_ENABLED, "true")
.format("iceberg").load(tableLocation);
materialize(df);
});
@@ -122,7 +123,7 @@ public class IcebergSourceNestedORCDataReadBenchmark extends IcebergSourceNested
tableProperties.put(SPLIT_OPEN_FILE_COST, Integer.toString(128 * 1024 * 1024));
withTableProperties(tableProperties, () -> {
String tableLocation = table().location();
- Dataset<Row> df = spark().read().option("vectorization-enabled", "true")
+ Dataset<Row> df = spark().read().option(SparkReadOptions.VECTORIZATION_ENABLED, "true")
.format("iceberg").load(tableLocation).selectExpr("nested.col3");
materialize(df);
});
diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java b/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java
index 589bbad..94c9c00 100644
--- a/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java
+++ b/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java
@@ -33,6 +33,7 @@ import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.spark.SparkUtil;
+import org.apache.iceberg.spark.SparkWriteOptions;
import org.apache.iceberg.types.TypeUtil;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
@@ -186,14 +187,14 @@ public class IcebergSource implements DataSourceV2, ReadSupport, WriteSupport, D
private boolean checkNullability(DataSourceOptions options) {
boolean sparkCheckNullability = Boolean.parseBoolean(lazySpark.conf()
.get("spark.sql.iceberg.check-nullability", "true"));
- boolean dataFrameCheckNullability = options.getBoolean("check-nullability", true);
+ boolean dataFrameCheckNullability = options.getBoolean(SparkWriteOptions.CHECK_NULLABILITY, true);
return sparkCheckNullability && dataFrameCheckNullability;
}
private boolean checkOrdering(DataSourceOptions options) {
boolean sparkCheckOrdering = Boolean.parseBoolean(lazySpark.conf()
.get("spark.sql.iceberg.check-ordering", "true"));
- boolean dataFrameCheckOrdering = options.getBoolean("check-ordering", true);
+ boolean dataFrameCheckOrdering = options.getBoolean(SparkWriteOptions.CHECK_ORDERING, true);
return sparkCheckOrdering && dataFrameCheckOrdering;
}
}
diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java b/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java
index 6ee4c75..4b0723e 100644
--- a/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java
+++ b/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java
@@ -49,6 +49,7 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.SparkFilters;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.util.TableScanUtil;
@@ -106,8 +107,8 @@ class Reader implements DataSourceReader, SupportsScanColumnarBatch, SupportsPus
Reader(Table table, Broadcast<FileIO> io, Broadcast<EncryptionManager> encryptionManager,
boolean caseSensitive, DataSourceOptions options) {
this.table = table;
- this.snapshotId = options.get("snapshot-id").map(Long::parseLong).orElse(null);
- this.asOfTimestamp = options.get("as-of-timestamp").map(Long::parseLong).orElse(null);
+ this.snapshotId = options.get(SparkReadOptions.SNAPSHOT_ID).map(Long::parseLong).orElse(null);
+ this.asOfTimestamp = options.get(SparkReadOptions.AS_OF_TIMESTAMP).map(Long::parseLong).orElse(null);
if (snapshotId != null && asOfTimestamp != null) {
throw new IllegalArgumentException(
"Cannot scan using both snapshot-id and as-of-timestamp to select the table snapshot");
@@ -128,9 +129,9 @@ class Reader implements DataSourceReader, SupportsScanColumnarBatch, SupportsPus
}
// look for split behavior overrides in options
- this.splitSize = options.get("split-size").map(Long::parseLong).orElse(null);
- this.splitLookback = options.get("lookback").map(Integer::parseInt).orElse(null);
- this.splitOpenFileCost = options.get("file-open-cost").map(Long::parseLong).orElse(null);
+ this.splitSize = options.get(SparkReadOptions.SPLIT_SIZE).map(Long::parseLong).orElse(null);
+ this.splitLookback = options.get(SparkReadOptions.LOOKBACK).map(Integer::parseInt).orElse(null);
+ this.splitOpenFileCost = options.get(SparkReadOptions.FILE_OPEN_COST).map(Long::parseLong).orElse(null);
if (io.getValue() instanceof HadoopFileIO) {
String fsscheme = "no_exist";
@@ -161,11 +162,12 @@ class Reader implements DataSourceReader, SupportsScanColumnarBatch, SupportsPus
if (batchReadsSessionConf != null) {
this.batchReadsEnabled = Boolean.valueOf(batchReadsSessionConf);
} else {
- this.batchReadsEnabled = options.get("vectorization-enabled").map(Boolean::parseBoolean).orElseGet(() ->
+ this.batchReadsEnabled = options.get(SparkReadOptions.VECTORIZATION_ENABLED)
+ .map(Boolean::parseBoolean).orElseGet(() ->
PropertyUtil.propertyAsBoolean(table.properties(), TableProperties.PARQUET_VECTORIZATION_ENABLED,
TableProperties.PARQUET_VECTORIZATION_ENABLED_DEFAULT));
}
- this.batchSize = options.get("batch-size").map(Integer::parseInt).orElseGet(() ->
+ this.batchSize = options.get(SparkReadOptions.VECTORIZATION_BATCH_SIZE).map(Integer::parseInt).orElseGet(() ->
PropertyUtil.propertyAsInt(table.properties(),
TableProperties.PARQUET_BATCH_SIZE, TableProperties.PARQUET_BATCH_SIZE_DEFAULT));
}
diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java b/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java
index 594215f..51ec595 100644
--- a/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java
+++ b/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java
@@ -116,7 +116,7 @@ class Writer implements DataSourceWriter {
long tableTargetFileSize = PropertyUtil.propertyAsLong(
table.properties(), WRITE_TARGET_FILE_SIZE_BYTES, WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT);
- this.targetFileSize = options.getLong("target-file-size-bytes", tableTargetFileSize);
+ this.targetFileSize = options.getLong(SparkWriteOptions.TARGET_FILE_SIZE_BYTES, tableTargetFileSize);
boolean tablePartitionedFanoutEnabled = PropertyUtil.propertyAsBoolean(
table.properties(), SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT);
@@ -124,7 +124,7 @@ class Writer implements DataSourceWriter {
}
private FileFormat getFileFormat(Map<String, String> tableProperties, DataSourceOptions options) {
- Optional<String> formatOption = options.get("write-format");
+ Optional<String> formatOption = options.get(SparkWriteOptions.WRITE_FORMAT);
String formatString = formatOption
.orElseGet(() -> tableProperties.getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
return FileFormat.valueOf(formatString.toUpperCase(Locale.ENGLISH));
diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
index 60d044a..ad26910 100644
--- a/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
+++ b/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
@@ -41,6 +41,7 @@ import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.io.FileAppender;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.data.GenericsHelpers;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
@@ -487,7 +488,7 @@ public class TestFilteredScan {
public void testUnpartitionedStartsWith() {
Dataset<Row> df = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(unpartitioned.toString());
List<String> matchedData = df.select("data")
@@ -554,7 +555,7 @@ public class TestFilteredScan {
// copy the unpartitioned table into the partitioned table to produce the partitioned data
Dataset<Row> allRows = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(unpartitioned.toString());
allRows
@@ -591,7 +592,7 @@ public class TestFilteredScan {
private static List<Row> read(String table, boolean vectorized, String expr, String select0, String... selectN) {
Dataset<Row> dataset = spark.read().format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(table).filter(expr)
.select(select0, selectN);
return dataset.collectAsList();
diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 73eb751..8a73df9 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -416,13 +416,13 @@ public class Spark3Util {
if (batchReadsSessionConf != null) {
return Boolean.valueOf(batchReadsSessionConf);
}
- return readOptions.getBoolean("vectorization-enabled",
+ return readOptions.getBoolean(SparkReadOptions.VECTORIZATION_ENABLED,
PropertyUtil.propertyAsBoolean(properties,
TableProperties.PARQUET_VECTORIZATION_ENABLED, TableProperties.PARQUET_VECTORIZATION_ENABLED_DEFAULT));
}
public static int batchSize(Map<String, String> properties, CaseInsensitiveStringMap readOptions) {
- return readOptions.getInt("batch-size",
+ return readOptions.getInt(SparkReadOptions.VECTORIZATION_BATCH_SIZE,
PropertyUtil.propertyAsInt(properties,
TableProperties.PARQUET_BATCH_SIZE, TableProperties.PARQUET_BATCH_SIZE_DEFAULT));
}
diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java
index d6e0a3f..76d873e 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java
@@ -34,6 +34,7 @@ import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.Spark3Util;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
@@ -55,8 +56,8 @@ class SparkBatchQueryScan extends SparkBatchScan {
super(table, io, encryption, caseSensitive, expectedSchema, filters, options);
- this.snapshotId = Spark3Util.propertyAsLong(options, "snapshot-id", null);
- this.asOfTimestamp = Spark3Util.propertyAsLong(options, "as-of-timestamp", null);
+ this.snapshotId = Spark3Util.propertyAsLong(options, SparkReadOptions.SNAPSHOT_ID, null);
+ this.asOfTimestamp = Spark3Util.propertyAsLong(options, SparkReadOptions.AS_OF_TIMESTAMP, null);
if (snapshotId != null && asOfTimestamp != null) {
throw new IllegalArgumentException(
@@ -68,17 +69,17 @@ class SparkBatchQueryScan extends SparkBatchScan {
if (snapshotId != null || asOfTimestamp != null) {
if (startSnapshotId != null || endSnapshotId != null) {
throw new IllegalArgumentException(
- "Cannot specify start-snapshot-id and end-snapshot-id to do incremental scan when either snapshot-id or " +
- "as-of-timestamp is specified");
+ "Cannot specify start-snapshot-id and end-snapshot-id to do incremental scan when either " +
+ SparkReadOptions.SNAPSHOT_ID + " or " + SparkReadOptions.AS_OF_TIMESTAMP + " is specified");
}
} else if (startSnapshotId == null && endSnapshotId != null) {
throw new IllegalArgumentException("Cannot only specify option end-snapshot-id to do incremental scan");
}
// look for split behavior overrides in options
- this.splitSize = Spark3Util.propertyAsLong(options, "split-size", null);
- this.splitLookback = Spark3Util.propertyAsInt(options, "lookback", null);
- this.splitOpenFileCost = Spark3Util.propertyAsLong(options, "file-open-cost", null);
+ this.splitSize = Spark3Util.propertyAsLong(options, SparkReadOptions.SPLIT_SIZE, null);
+ this.splitLookback = Spark3Util.propertyAsInt(options, SparkReadOptions.LOOKBACK, null);
+ this.splitOpenFileCost = Spark3Util.propertyAsLong(options, SparkReadOptions.FILE_OPEN_COST, null);
}
@Override
diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java
index 662e8ef..9892ec1 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.Spark3Util;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.util.TableScanUtil;
import org.apache.spark.broadcast.Broadcast;
@@ -82,13 +83,13 @@ class SparkMergeScan extends SparkBatchScan implements SupportsFileFilter {
Map<String, String> props = table.properties();
long tableSplitSize = PropertyUtil.propertyAsLong(props, SPLIT_SIZE, SPLIT_SIZE_DEFAULT);
- this.splitSize = Spark3Util.propertyAsLong(options, "split-size", tableSplitSize);
+ this.splitSize = Spark3Util.propertyAsLong(options, SparkReadOptions.SPLIT_SIZE, tableSplitSize);
int tableSplitLookback = PropertyUtil.propertyAsInt(props, SPLIT_LOOKBACK, SPLIT_LOOKBACK_DEFAULT);
- this.splitLookback = Spark3Util.propertyAsInt(options, "lookback", tableSplitLookback);
+ this.splitLookback = Spark3Util.propertyAsInt(options, SparkReadOptions.LOOKBACK, tableSplitLookback);
long tableOpenFileCost = PropertyUtil.propertyAsLong(props, SPLIT_OPEN_FILE_COST, SPLIT_OPEN_FILE_COST_DEFAULT);
- this.splitOpenFileCost = Spark3Util.propertyAsLong(options, "file-open-cost", tableOpenFileCost);
+ this.splitOpenFileCost = Spark3Util.propertyAsLong(options, SparkReadOptions.FILE_OPEN_COST, tableOpenFileCost);
Preconditions.checkArgument(!options.containsKey("snapshot-id"), "Cannot have snapshot-id in options");
Snapshot currentSnapshot = table.currentSnapshot();
diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
index 7fbe459..c5c3faa 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
@@ -122,7 +122,7 @@ class SparkWrite {
long tableTargetFileSize = PropertyUtil.propertyAsLong(
table.properties(), WRITE_TARGET_FILE_SIZE_BYTES, WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT);
- this.targetFileSize = writeInfo.options().getLong("target-file-size-bytes", tableTargetFileSize);
+ this.targetFileSize = writeInfo.options().getLong(SparkWriteOptions.TARGET_FILE_SIZE_BYTES, tableTargetFileSize);
boolean tablePartitionedFanoutEnabled = PropertyUtil.propertyAsBoolean(
table.properties(), SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT);
@@ -155,7 +155,7 @@ class SparkWrite {
}
private FileFormat getFileFormat(Map<String, String> tableProperties, Map<String, String> options) {
- Optional<String> formatOption = Optional.ofNullable(options.get("write-format"));
+ Optional<String> formatOption = Optional.ofNullable(options.get(SparkWriteOptions.WRITE_FORMAT));
String formatString = formatOption
.orElseGet(() -> tableProperties.getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
return FileFormat.valueOf(formatString.toUpperCase(Locale.ENGLISH));
diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
index 2111552..7530afd 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java
@@ -31,6 +31,7 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.spark.SparkFilters;
import org.apache.iceberg.spark.SparkSchemaUtil;
import org.apache.iceberg.spark.SparkUtil;
+import org.apache.iceberg.spark.SparkWriteOptions;
import org.apache.iceberg.types.TypeUtil;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
@@ -183,7 +184,7 @@ class SparkWriteBuilder implements WriteBuilder, SupportsDynamicOverwrite, Suppo
private static boolean checkOrdering(SparkSession spark, CaseInsensitiveStringMap options) {
boolean sparkCheckOrdering = Boolean.parseBoolean(spark.conf()
.get("spark.sql.iceberg.check-ordering", "true"));
- boolean dataFrameCheckOrdering = options.getBoolean("check-ordering", true);
+ boolean dataFrameCheckOrdering = options.getBoolean(SparkWriteOptions.CHECK_ORDERING, true);
return sparkCheckOrdering && dataFrameCheckOrdering;
}
}
diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
index 5824dca..d4d9eaf 100644
--- a/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
+++ b/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.io.FileAppender;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.data.GenericsHelpers;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
@@ -439,7 +440,7 @@ public class TestFilteredScan {
public void testUnpartitionedStartsWith() {
Dataset<Row> df = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(unpartitioned.toString());
List<String> matchedData = df.select("data")
@@ -506,7 +507,7 @@ public class TestFilteredScan {
// copy the unpartitioned table into the partitioned table to produce the partitioned data
Dataset<Row> allRows = spark.read()
.format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(unpartitioned.toString());
allRows
@@ -545,7 +546,7 @@ public class TestFilteredScan {
private static List<Row> read(String table, boolean vectorized, String expr, String select0, String... selectN) {
Dataset<Row> dataset = spark.read().format("iceberg")
- .option("vectorization-enabled", String.valueOf(vectorized))
+ .option(SparkReadOptions.VECTORIZATION_ENABLED, String.valueOf(vectorized))
.load(table).filter(expr)
.select(select0, selectN);
return dataset.collectAsList();