You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by gh...@apache.org on 2025/01/27 17:04:36 UTC
(kafka) branch trunk updated: KAFKA-17792: Efficiently parse decimals with large exponents in Connect Values (#17510)
This is an automated email from the ASF dual-hosted git repository.
gharris pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new d001b47093c KAFKA-17792: Efficiently parse decimals with large exponents in Connect Values (#17510)
d001b47093c is described below
commit d001b47093c8ca45c53bd8220d3c735e77f57999
Author: Martin Sillence <ma...@luminoussheep.net>
AuthorDate: Mon Jan 27 17:04:29 2025 +0000
KAFKA-17792: Efficiently parse decimals with large exponents in Connect Values (#17510)
Reviewers: Greg Harris <gr...@aiven.io>, Mickael Maison <mi...@gmail.com>
---
.../java/org/apache/kafka/connect/data/Values.java | 20 ++++++++++++++------
.../org/apache/kafka/connect/data/ValuesTest.java | 11 +++++++++++
2 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
index cd332d2a8b3..f23f1f88a75 100644
--- a/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
+++ b/connect/api/src/main/java/org/apache/kafka/connect/data/Values.java
@@ -26,7 +26,6 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
-import java.math.RoundingMode;
import java.nio.ByteBuffer;
import java.text.CharacterIterator;
import java.text.DateFormat;
@@ -1012,6 +1011,7 @@ public class Values {
return parseAsTemporal(token);
}
+
private static SchemaAndValue parseAsNumber(String token) {
// Try to parse as a number ...
BigDecimal decimal = new BigDecimal(token);
@@ -1032,12 +1032,20 @@ public class Values {
}
}
+ private static boolean isWholeNumber(BigDecimal bd) {
+ return bd.signum() == 0 || bd.scale() <= 0 || bd.stripTrailingZeros().scale() <= 0;
+ }
+
+ private static final BigDecimal BIGGER_THAN_LONG = new BigDecimal("1e19");
+
private static SchemaAndValue parseAsExactDecimal(BigDecimal decimal) {
- BigDecimal ceil = decimal.setScale(0, RoundingMode.CEILING);
- BigDecimal floor = decimal.setScale(0, RoundingMode.FLOOR);
- if (ceil.equals(floor)) {
- BigInteger num = ceil.toBigIntegerExact();
- if (ceil.precision() >= 19 && (num.compareTo(LONG_MIN) < 0 || num.compareTo(LONG_MAX) > 0)) {
+ BigDecimal abs = decimal.abs();
+ if (abs.compareTo(BIGGER_THAN_LONG) > 0 || (abs.compareTo(BigDecimal.ONE) < 0 && abs.compareTo(BigDecimal.ZERO) != 0)) {
+ return null;
+ }
+ if (isWholeNumber(decimal)) {
+ BigInteger num = decimal.toBigIntegerExact();
+ if (num.compareTo(LONG_MIN) < 0 || num.compareTo(LONG_MAX) > 0) {
return null;
}
long integral = num.longValue();
diff --git a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
index e81b1f8a06e..ac6eef6fa68 100644
--- a/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
+++ b/connect/api/src/test/java/org/apache/kafka/connect/data/ValuesTest.java
@@ -1181,6 +1181,17 @@ public class ValuesTest {
assertEquals(new SchemaAndValue(Schema.FLOAT32_SCHEMA, 66000.0008f), Values.parseString("66000.0008"));
}
+ @Test
+ public void avoidCpuAndMemoryIssuesConvertingExtremeBigDecimals() {
+ String parsingBig = "1e+100000000"; // new BigDecimal().setScale(0, RoundingMode.FLOOR) takes around two minutes and uses 3GB;
+ BigDecimal valueBig = new BigDecimal(parsingBig);
+ assertEquals(new SchemaAndValue(Decimal.schema(-100000000), valueBig), Values.parseString(parsingBig), "parsing number that's too big");
+
+ String parsingSmall = "1e-100000000";
+ BigDecimal valueSmall = new BigDecimal(parsingSmall);
+ assertEquals(new SchemaAndValue(Schema.FLOAT32_SCHEMA, (float) valueSmall.doubleValue()), Values.parseString(parsingSmall), "parsing number that's too big, strictly this should return a bigdecimal");
+ }
+
protected void assertParsed(String input) {
assertParsed(input, input);
}