You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2023/04/27 16:50:05 UTC
[iceberg] branch master updated: Spark: Refactor SparkReadConf to use primitive type (#7429)
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new faddf308ed Spark: Refactor SparkReadConf to use primitive type (#7429)
faddf308ed is described below
commit faddf308eda97228119bb3cdaecec6faf2307ae4
Author: Prashant Singh <35...@users.noreply.github.com>
AuthorDate: Thu Apr 27 09:49:58 2023 -0700
Spark: Refactor SparkReadConf to use primitive type (#7429)
Co-authored-by: Prashant Singh <ps...@amazon.com>
---
.../main/java/org/apache/iceberg/spark/SparkReadConf.java | 2 +-
.../apache/iceberg/spark/source/SparkMicroBatchStream.java | 8 ++++----
.../main/java/org/apache/iceberg/spark/SparkReadConf.java | 2 +-
.../apache/iceberg/spark/source/SparkMicroBatchStream.java | 8 ++++----
.../main/java/org/apache/iceberg/spark/SparkReadConf.java | 6 +++---
.../apache/iceberg/spark/source/SparkMicroBatchStream.java | 12 ++++++------
.../main/java/org/apache/iceberg/spark/SparkReadConf.java | 2 +-
.../apache/iceberg/spark/source/SparkMicroBatchStream.java | 8 ++++----
8 files changed, 24 insertions(+), 24 deletions(-)
diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
index 2ade02bbd0..d5833a9d07 100644
--- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
+++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
@@ -201,7 +201,7 @@ public class SparkReadConf {
.parse();
}
- public Long streamFromTimestamp() {
+ public long streamFromTimestamp() {
return confParser
.longConf()
.option(SparkReadOptions.STREAM_FROM_TIMESTAMP)
diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
index 1ed3b64f02..ebac79ebcb 100644
--- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
+++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
@@ -69,14 +69,14 @@ public class SparkMicroBatchStream implements MicroBatchStream {
private final boolean caseSensitive;
private final String expectedSchema;
private final Broadcast<Table> tableBroadcast;
- private final Long splitSize;
- private final Integer splitLookback;
- private final Long splitOpenFileCost;
+ private final long splitSize;
+ private final int splitLookback;
+ private final long splitOpenFileCost;
private final boolean localityPreferred;
private final StreamingOffset initialOffset;
private final boolean skipDelete;
private final boolean skipOverwrite;
- private final Long fromTimestamp;
+ private final long fromTimestamp;
SparkMicroBatchStream(
JavaSparkContext sparkContext,
diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
index 8fbd11c9d9..fda807431a 100644
--- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
+++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
@@ -201,7 +201,7 @@ public class SparkReadConf {
.parse();
}
- public Long streamFromTimestamp() {
+ public long streamFromTimestamp() {
return confParser
.longConf()
.option(SparkReadOptions.STREAM_FROM_TIMESTAMP)
diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
index 816e3d2bf8..c4de2ddfe6 100644
--- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
+++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
@@ -70,14 +70,14 @@ public class SparkMicroBatchStream implements MicroBatchStream {
private final boolean caseSensitive;
private final String expectedSchema;
private final Broadcast<Table> tableBroadcast;
- private final Long splitSize;
- private final Integer splitLookback;
- private final Long splitOpenFileCost;
+ private final long splitSize;
+ private final int splitLookback;
+ private final long splitOpenFileCost;
private final boolean localityPreferred;
private final StreamingOffset initialOffset;
private final boolean skipDelete;
private final boolean skipOverwrite;
- private final Long fromTimestamp;
+ private final long fromTimestamp;
SparkMicroBatchStream(
JavaSparkContext sparkContext,
diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
index dbd2613dde..0f24844414 100644
--- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
+++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
@@ -239,7 +239,7 @@ public class SparkReadConf {
.parse();
}
- public Long streamFromTimestamp() {
+ public long streamFromTimestamp() {
return confParser
.longConf()
.option(SparkReadOptions.STREAM_FROM_TIMESTAMP)
@@ -255,7 +255,7 @@ public class SparkReadConf {
return confParser.longConf().option(SparkReadOptions.END_TIMESTAMP).parseOptional();
}
- public Integer maxFilesPerMicroBatch() {
+ public int maxFilesPerMicroBatch() {
return confParser
.intConf()
.option(SparkReadOptions.STREAMING_MAX_FILES_PER_MICRO_BATCH)
@@ -263,7 +263,7 @@ public class SparkReadConf {
.parse();
}
- public Integer maxRecordsPerMicroBatch() {
+ public int maxRecordsPerMicroBatch() {
return confParser
.intConf()
.option(SparkReadOptions.STREAMING_MAX_ROWS_PER_MICRO_BATCH)
diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
index e1a8ef5912..4019fedcbb 100644
--- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
+++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
@@ -77,16 +77,16 @@ public class SparkMicroBatchStream implements MicroBatchStream, SupportsAdmissio
private final boolean caseSensitive;
private final String expectedSchema;
private final Broadcast<Table> tableBroadcast;
- private final Long splitSize;
- private final Integer splitLookback;
- private final Long splitOpenFileCost;
+ private final long splitSize;
+ private final int splitLookback;
+ private final long splitOpenFileCost;
private final boolean localityPreferred;
private final StreamingOffset initialOffset;
private final boolean skipDelete;
private final boolean skipOverwrite;
- private final Long fromTimestamp;
- private final Integer maxFilesPerMicroBatch;
- private final Integer maxRecordsPerMicroBatch;
+ private final long fromTimestamp;
+ private final int maxFilesPerMicroBatch;
+ private final int maxRecordsPerMicroBatch;
SparkMicroBatchStream(
JavaSparkContext sparkContext,
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
index 1c1182c4da..b91270f166 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java
@@ -239,7 +239,7 @@ public class SparkReadConf {
.parse();
}
- public Long streamFromTimestamp() {
+ public long streamFromTimestamp() {
return confParser
.longConf()
.option(SparkReadOptions.STREAM_FROM_TIMESTAMP)
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
index 6e03dd69a8..728acbe646 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
@@ -72,14 +72,14 @@ public class SparkMicroBatchStream implements MicroBatchStream {
private final boolean caseSensitive;
private final String expectedSchema;
private final Broadcast<Table> tableBroadcast;
- private final Long splitSize;
- private final Integer splitLookback;
- private final Long splitOpenFileCost;
+ private final long splitSize;
+ private final int splitLookback;
+ private final long splitOpenFileCost;
private final boolean localityPreferred;
private final StreamingOffset initialOffset;
private final boolean skipDelete;
private final boolean skipOverwrite;
- private final Long fromTimestamp;
+ private final long fromTimestamp;
SparkMicroBatchStream(
JavaSparkContext sparkContext,