[iceberg] branch master updated: Spark: Fix file-open-cost in DSv2 streams (#2743)

[bl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new a100d2d5 Spark: Fix file-open-cost in DSv2 streams (#2743)
a100d2d5 is described below

commit a100d2d5a06153674cdf97940039f3df68b5f9b8
Author: Sreeram Garlapati <Sr...@users.noreply.github.com>
AuthorDate: Mon Jun 28 10:06:08 2021 -0700

    Spark: Fix file-open-cost in DSv2 streams (#2743)
---
 .../java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
index 1a7f217..84fdbd2 100644
--- a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
+++ b/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java
@@ -96,7 +96,8 @@ public class SparkMicroBatchStream implements MicroBatchStream {
 
     long tableSplitOpenFileCost = PropertyUtil.propertyAsLong(
         table.properties(), SPLIT_OPEN_FILE_COST, SPLIT_OPEN_FILE_COST_DEFAULT);
-    this.splitOpenFileCost = Spark3Util.propertyAsLong(options, SPLIT_OPEN_FILE_COST, tableSplitOpenFileCost);
+    this.splitOpenFileCost = Spark3Util.propertyAsLong(
+        options, SparkReadOptions.FILE_OPEN_COST, tableSplitOpenFileCost);
 
     InitialOffsetStore initialOffsetStore = new InitialOffsetStore(table, checkpointLocation);
     this.initialOffset = initialOffsetStore.initialOffset();