You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ho...@apache.org on 2026/09/20 03:38:51 UTC
(paimon) 20/20: [common] Fix JSON round-trip of temporal and decimal predicate literals (#9640)
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 2a5cbe906c512028488896a1fc6318347fe5ae8c
Author: YangJie <ya...@baidu.com>
AuthorDate: Sun Sep 13 22:35:40 2026 -0400
[common] Fix JSON round-trip of temporal and decimal predicate literals (#9640)
---
.../requests/ListPartitionsByFilterRequest.java | 10 +++-
.../org/apache/paimon/predicate/LeafPredicate.java | 55 +++++++++++++++++++++-
.../paimon/predicate/PredicateJsonSerdeTest.java | 44 +++++++++++++++++
.../org/apache/paimon/rest/RESTApiJsonTest.java | 52 ++++++++++++++++++++
4 files changed, 158 insertions(+), 3 deletions(-)
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/ListPartitionsByFilterRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/ListPartitionsByFilterRequest.java
index 102b51e11d..8af21306f0 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/ListPartitionsByFilterRequest.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/ListPartitionsByFilterRequest.java
@@ -40,7 +40,15 @@ public class ListPartitionsByFilterRequest implements RESTRequest {
private static final String FIELD_MAX_RESULTS = "maxResults";
private static final String FIELD_PAGE_TOKEN = "pageToken";
- /** JSON serialization of a Paimon {@code Predicate} tree over the partition columns. */
+ /**
+ * JSON serialization of a Paimon {@code Predicate} tree over the partition columns.
+ *
+ * <p>Wire encoding of the literals a server has to parse: DATE, TIME, TIMESTAMP and
+ * TIMESTAMP_LTZ are ISO-8601 strings (e.g. {@code "2026-01-15"}, {@code "12:34:56.789"}, {@code
+ * "2026-01-15T12:34:56.789"}, {@code "2026-01-15T04:34:56.789Z"}); DECIMAL is a plain
+ * (non-scientific) decimal string. Older clients emitted these as JSON arrays and numbers that
+ * no Paimon server could read back, so no previously working request encoding changes meaning.
+ */
@JsonProperty(FIELD_FILTER)
private final String filter;
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
index 1383837f93..85a0485539 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java
@@ -35,6 +35,11 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonPro
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -262,11 +267,30 @@ public class LeafPredicate implements Predicate {
}
List<Object> serialized = new ArrayList<>(literals.size());
for (Object lit : literals) {
- serialized.add(PredicateBuilder.convertToJavaObject(type, lit));
+ serialized.add(toJsonFriendly(PredicateBuilder.convertToJavaObject(type, lit)));
}
return serialized;
}
+ /**
+ * Temporal and decimal literals are carried as strings: reading the JSON back into the untyped
+ * literal list materializes Jackson's JavaTimeModule array/number forms as {@code List}/{@code
+ * Double}, which {@link PredicateBuilder#convertJavaObject} rejects, and the double
+ * materialization of a decimal is lossy beyond ~15 significant digits.
+ */
+ private static Object toJsonFriendly(Object literal) {
+ if (literal instanceof LocalDate
+ || literal instanceof LocalTime
+ || literal instanceof LocalDateTime
+ || literal instanceof Instant) {
+ return literal.toString();
+ }
+ if (literal instanceof BigDecimal) {
+ return ((BigDecimal) literal).toPlainString();
+ }
+ return literal;
+ }
+
protected static List<Object> deserializeLiterals(DataType type, List<Object> literals) {
if (literals == null) {
return null;
@@ -277,8 +301,35 @@ public class LeafPredicate implements Predicate {
converted.add(literal);
continue;
}
- converted.add(PredicateBuilder.convertJavaObject(type, literal));
+ converted.add(
+ PredicateBuilder.convertJavaObject(type, parseJsonLiteral(type, literal)));
}
return converted;
}
+
+ /**
+ * Converts a literal materialized from JSON back into the object {@link
+ * PredicateBuilder#convertJavaObject} accepts, undoing {@link #toJsonFriendly}. Anything else
+ * is returned unchanged so that convertJavaObject reports it.
+ */
+ private static Object parseJsonLiteral(DataType type, Object literal) {
+ if (!(literal instanceof String)) {
+ return literal;
+ }
+ String text = (String) literal;
+ switch (type.getTypeRoot()) {
+ case DATE:
+ return LocalDate.parse(text);
+ case TIME_WITHOUT_TIME_ZONE:
+ return LocalTime.parse(text);
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ return LocalDateTime.parse(text);
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return Instant.parse(text);
+ case DECIMAL:
+ return new BigDecimal(text);
+ default:
+ return literal;
+ }
+ }
}
diff --git a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java
index 362f34f917..4a62f0008e 100644
--- a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateJsonSerdeTest.java
@@ -19,16 +19,23 @@
package org.apache.paimon.predicate;
import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.Decimal;
+import org.apache.paimon.data.Timestamp;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.JsonSerdeUtil;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import javax.annotation.Nullable;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -272,6 +279,43 @@ class PredicateJsonSerdeTest {
}
}
+ @Test
+ void testTemporalAndDecimalLiteralsRoundTrip() {
+ Predicate predicate =
+ PredicateBuilder.and(
+ temporalBuilder().equal(0, (int) LocalDate.of(2026, 1, 15).toEpochDay()),
+ temporalBuilder().equal(1, 45_296_789), // 12:34:56.789
+ temporalBuilder()
+ .equal(
+ 2,
+ Timestamp.fromLocalDateTime(
+ LocalDateTime.of(
+ 2026, 1, 15, 12, 34, 56, 789_000_000))),
+ temporalBuilder()
+ .equal(
+ 3,
+ Timestamp.fromInstant(
+ Instant.parse("2026-01-15T04:34:56.789Z"))),
+ temporalBuilder()
+ .equal(
+ 4,
+ Decimal.fromBigDecimal(
+ new BigDecimal("12345678901234567.891"), 20, 3)));
+
+ assertThat(parse(toJson(predicate))).isEqualTo(predicate);
+ }
+
+ private static PredicateBuilder temporalBuilder() {
+ return new PredicateBuilder(
+ RowType.of(
+ DataTypes.DATE(),
+ DataTypes.TIME(3),
+ DataTypes.TIMESTAMP(6),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(9),
+ DataTypes.DECIMAL(20, 3),
+ DataTypes.STRING()));
+ }
+
private static PredicateBuilder newBuilder() {
return new PredicateBuilder(
RowType.of(
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java
index e16233ece5..6ed35b84d4 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java
@@ -18,8 +18,12 @@
package org.apache.paimon.rest;
+import org.apache.paimon.data.Decimal;
+import org.apache.paimon.data.Timestamp;
import org.apache.paimon.function.FunctionChange;
import org.apache.paimon.partition.PartitionStatistics;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.rest.requests.AlterDatabaseRequest;
import org.apache.paimon.rest.requests.AlterFunctionRequest;
import org.apache.paimon.rest.requests.AlterTableRequest;
@@ -53,6 +57,8 @@ import org.apache.paimon.table.Instant;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.JsonSerdeUtil;
import org.apache.paimon.view.ViewChange;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
@@ -292,6 +298,52 @@ public class RESTApiJsonTest {
Map.class));
}
+ @Test
+ public void listPartitionsByFilterRequestPreservesTemporalAndDecimalLiterals()
+ throws Exception {
+ // A partition filter reaches the server as JsonSerdeUtil.toFlatJson(predicate) carried in
+ // the request's filter field. DATE/TIME/TIMESTAMP/TIMESTAMP_LTZ/DECIMAL literals travel as
+ // strings; assert they survive the full request round-trip that a server parses, including
+ // a decimal with more significant digits than a double can hold.
+ PredicateBuilder builder =
+ new PredicateBuilder(
+ RowType.of(
+ DataTypes.DATE(),
+ DataTypes.TIME(3),
+ DataTypes.TIMESTAMP(6),
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(9),
+ DataTypes.DECIMAL(38, 18)));
+ Predicate predicate =
+ PredicateBuilder.and(
+ builder.equal(0, (int) java.time.LocalDate.of(2026, 1, 15).toEpochDay()),
+ builder.equal(1, 45_296_789), // 12:34:56.789
+ builder.equal(
+ 2,
+ Timestamp.fromLocalDateTime(
+ java.time.LocalDateTime.of(
+ 2026, 1, 15, 12, 34, 56, 789_000_000))),
+ builder.equal(
+ 3,
+ Timestamp.fromInstant(
+ java.time.Instant.parse("2026-01-15T04:34:56.789Z"))),
+ builder.equal(
+ 4,
+ Decimal.fromBigDecimal(
+ new java.math.BigDecimal(
+ "12345678901234567890.123456789012345678"),
+ 38,
+ 18)));
+
+ ListPartitionsByFilterRequest request =
+ new ListPartitionsByFilterRequest(
+ JsonSerdeUtil.toFlatJson(predicate), "dt=2026%", 2, null);
+ ListPartitionsByFilterRequest parsed =
+ RESTApi.fromJson(RESTApi.toJson(request), ListPartitionsByFilterRequest.class);
+ Predicate serverSide = JsonSerdeUtil.fromJson(parsed.getFilter(), Predicate.class);
+
+ assertEquals(predicate, serverSide);
+ }
+
@Test
public void createPartitionsResponseParseTest() throws Exception {
Map<String, String> created = new HashMap<>();