You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/07/31 07:59:49 UTC
[incubator-paimon] branch master updated: [core] Sort spill threhold can not be 1 (#1687)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a59596eed [core] Sort spill threhold can not be 1 (#1687)
a59596eed is described below
commit a59596eedb291bafa7f909bb75fa09803c93a308
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Jul 31 15:59:44 2023 +0800
[core] Sort spill threhold can not be 1 (#1687)
---
paimon-common/src/main/java/org/apache/paimon/CoreOptions.java | 2 ++
.../src/test/java/org/apache/paimon/spark/SparkWriteITCase.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index 2711ccbca..a56d26d7c 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -47,6 +47,7 @@ import java.util.stream.Collectors;
import static org.apache.paimon.options.ConfigOptions.key;
import static org.apache.paimon.options.description.TextElement.text;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
/** Core options for paimon. */
public class CoreOptions implements Serializable {
@@ -936,6 +937,7 @@ public class CoreOptions implements Serializable {
int stopNum = numSortedRunStopTrigger();
maxSortedRunNum = Math.max(stopNum, stopNum + 1);
}
+ checkArgument(maxSortedRunNum > 1, "The sort spill threshold cannot be smaller than 2.");
return maxSortedRunNum;
}
diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java
index d5302fdee..bbe85c11b 100644
--- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java
+++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java
@@ -83,7 +83,7 @@ public class SparkWriteITCase {
public void testSortSpill() {
spark.sql(
"CREATE TABLE T (a INT, b INT, c STRING) TBLPROPERTIES"
- + " ('primary-key'='a', 'bucket'='4', 'file.format'='avro', 'sort-spill-threshold'='1')");
+ + " ('primary-key'='a', 'bucket'='4', 'file.format'='avro', 'sort-spill-threshold'='2')");
innerSimpleWrite();
}