You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2026/09/11 15:35:10 UTC
(iceberg) branch main updated: Spark, Flink: Remove Joda from `SparkUtil.java` and tests (#17579)
This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new baece5e6fe Spark, Flink: Remove Joda from `SparkUtil.java` and tests (#17579)
baece5e6fe is described below
commit baece5e6fee4ddcc5bcba3791fd5c1b4c9e6a577
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Fri Sep 11 17:35:03 2026 +0200
Spark, Flink: Remove Joda from `SparkUtil.java` and tests (#17579)
* Spark, Flink: Remove Joda from tests
* Spark, Flink: Remove Joda from tests
* Include Spark 4.2
* Make spotless happy
* Fix import
* Add Flink 2.3
* Include timezones as well
* Thanks Eduard
---
.../org/apache/iceberg/flink/DataGenerators.java | 26 +++----
.../org/apache/iceberg/flink/DataGenerators.java | 26 +++----
.../org/apache/iceberg/flink/DataGenerators.java | 26 +++----
.../org/apache/iceberg/flink/DataGenerators.java | 26 +++----
.../spark/extensions/TestAddFilesProcedure.java | 4 +-
.../java/org/apache/iceberg/spark/SparkUtil.java | 43 ++++++++++--
.../org/apache/iceberg/spark/TestSparkUtil.java | 81 ++++++++++++++++++++++
.../spark/sql/TestTimestampWithoutZone.java | 3 +-
.../spark/extensions/TestAddFilesProcedure.java | 4 +-
.../java/org/apache/iceberg/spark/SparkUtil.java | 43 ++++++++++--
.../org/apache/iceberg/spark/TestSparkUtil.java | 81 ++++++++++++++++++++++
.../spark/sql/TestTimestampWithoutZone.java | 3 +-
.../spark/extensions/TestAddFilesProcedure.java | 4 +-
.../java/org/apache/iceberg/spark/SparkUtil.java | 43 ++++++++++--
.../org/apache/iceberg/spark/TestSparkUtil.java | 81 ++++++++++++++++++++++
.../spark/sql/TestTimestampWithoutZone.java | 3 +-
.../spark/extensions/TestAddFilesProcedure.java | 4 +-
.../java/org/apache/iceberg/spark/SparkUtil.java | 43 ++++++++++--
.../org/apache/iceberg/spark/TestSparkUtil.java | 81 ++++++++++++++++++++++
.../spark/sql/TestTimestampWithoutZone.java | 3 +-
20 files changed, 524 insertions(+), 104 deletions(-)
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
index 72d53bc038..f7fd5da1cc 100644
--- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
@@ -50,9 +50,6 @@ import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Days;
/**
* Util class to generate test data with extensive coverage different field types: from primitives
@@ -61,12 +58,6 @@ import org.joda.time.Days;
public class DataGenerators {
public static class Primitives implements DataGenerator {
- private static final DateTime JODA_DATETIME_EPOC =
- new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final DateTime JODA_DATETIME_20220110 =
- new DateTime(2022, 1, 10, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final int DAYS_BTW_EPOC_AND_20220110 =
- Days.daysBetween(JODA_DATETIME_EPOC, JODA_DATETIME_20220110).getDays();
private static final int HOUR_8_IN_MILLI = (int) TimeUnit.HOURS.toMillis(8);
private static final int MICROS_OF_MILLI_20220110 = 456;
@@ -74,6 +65,10 @@ public class DataGenerators {
private static final LocalTime JAVA_LOCAL_TIME_HOUR8 = LocalTime.of(8, 0);
private static final OffsetDateTime JAVA_OFFSET_DATE_TIME_20220110 =
OffsetDateTime.of(2022, 1, 10, 0, 0, 0, 0, ZoneOffset.UTC);
+ private static final long MILLIS_20220110 =
+ JAVA_OFFSET_DATE_TIME_20220110.toInstant().toEpochMilli();
+ private static final int DAYS_BTW_EPOC_AND_20220110 =
+ (int) JAVA_OFFSET_DATE_TIME_20220110.toLocalDate().toEpochDay();
private static final LocalDateTime JAVA_LOCAL_DATE_TIME_20220110 =
LocalDateTime.of(2022, 1, 10, 0, 0, 0);
private static final OffsetDateTime JAVA_OFFSET_DATE_TIME_MAX_NANO =
@@ -227,10 +222,8 @@ public class DataGenerators {
HOUR_8_IN_MILLI,
// Although Avro logical type for timestamp fields are in micro seconds,
// AvroToRowDataConverters only looks for long value in milliseconds.
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
uuidBytes,
binaryBytes,
DecimalData.fromBigDecimal(BIG_DECIMAL_NEGATIVE, 9, 2),
@@ -256,12 +249,9 @@ public class DataGenerators {
genericRecord.put("time_field", HOUR_8_IN_MILLI);
// Now that AvroToRowDataConverters correctly supports microseconds,
// we must inject correct microsecond scale values into the Avro data.
+ genericRecord.put("ts_with_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put(
- "ts_with_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
- genericRecord.put(
- "ts_without_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
+ "ts_without_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put("ts_ns_with_zone_field", ICEBERG_MAX_NANOS_EPOCH);
genericRecord.put("ts_ns_without_zone_field", ICEBERG_MAX_NANOS_EPOCH);
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
index 5393c3958b..6a7a4a4699 100644
--- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
@@ -53,9 +53,6 @@ import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Days;
/**
* Util class to generate test data with extensive coverage different field types: from primitives
@@ -64,12 +61,6 @@ import org.joda.time.Days;
public class DataGenerators {
public static class Primitives implements DataGenerator {
- private static final DateTime JODA_DATETIME_EPOC =
- new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final DateTime JODA_DATETIME_20220110 =
- new DateTime(2022, 1, 10, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final int DAYS_BTW_EPOC_AND_20220110 =
- Days.daysBetween(JODA_DATETIME_EPOC, JODA_DATETIME_20220110).getDays();
private static final int HOUR_8_IN_MILLI = (int) TimeUnit.HOURS.toMillis(8);
private static final int MICROS_OF_MILLI_20220110 = 456;
@@ -77,6 +68,10 @@ public class DataGenerators {
private static final LocalTime JAVA_LOCAL_TIME_HOUR8 = LocalTime.of(8, 0);
private static final OffsetDateTime JAVA_OFFSET_DATE_TIME_20220110 =
OffsetDateTime.of(2022, 1, 10, 0, 0, 0, 0, ZoneOffset.UTC);
+ private static final long MILLIS_20220110 =
+ JAVA_OFFSET_DATE_TIME_20220110.toInstant().toEpochMilli();
+ private static final int DAYS_BTW_EPOC_AND_20220110 =
+ (int) JAVA_OFFSET_DATE_TIME_20220110.toLocalDate().toEpochDay();
private static final Instant JAVA_INSTANT_DATE_TIME_20220110 =
Instant.ofEpochSecond(JAVA_OFFSET_DATE_TIME_20220110.toEpochSecond(), 0);
private static final LocalDateTime JAVA_LOCAL_DATE_TIME_20220110 =
@@ -232,10 +227,8 @@ public class DataGenerators {
HOUR_8_IN_MILLI,
// Although Avro logical type for timestamp fields are in micro seconds,
// AvroToRowDataConverters only looks for long value in milliseconds.
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
uuidBytes,
binaryBytes,
DecimalData.fromBigDecimal(BIG_DECIMAL_NEGATIVE, 9, 2),
@@ -305,12 +298,9 @@ public class DataGenerators {
genericRecord.put("time_field", HOUR_8_IN_MILLI);
// Now that AvroToRowDataConverters correctly supports microseconds,
// we must inject correct microsecond scale values into the Avro data.
+ genericRecord.put("ts_with_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put(
- "ts_with_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
- genericRecord.put(
- "ts_without_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
+ "ts_without_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put("ts_ns_with_zone_field", ICEBERG_MAX_NANOS_EPOCH);
genericRecord.put("ts_ns_without_zone_field", ICEBERG_MAX_NANOS_EPOCH);
diff --git a/flink/v2.2/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java b/flink/v2.2/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
index 5393c3958b..6a7a4a4699 100644
--- a/flink/v2.2/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
+++ b/flink/v2.2/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
@@ -53,9 +53,6 @@ import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Days;
/**
* Util class to generate test data with extensive coverage different field types: from primitives
@@ -64,12 +61,6 @@ import org.joda.time.Days;
public class DataGenerators {
public static class Primitives implements DataGenerator {
- private static final DateTime JODA_DATETIME_EPOC =
- new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final DateTime JODA_DATETIME_20220110 =
- new DateTime(2022, 1, 10, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final int DAYS_BTW_EPOC_AND_20220110 =
- Days.daysBetween(JODA_DATETIME_EPOC, JODA_DATETIME_20220110).getDays();
private static final int HOUR_8_IN_MILLI = (int) TimeUnit.HOURS.toMillis(8);
private static final int MICROS_OF_MILLI_20220110 = 456;
@@ -77,6 +68,10 @@ public class DataGenerators {
private static final LocalTime JAVA_LOCAL_TIME_HOUR8 = LocalTime.of(8, 0);
private static final OffsetDateTime JAVA_OFFSET_DATE_TIME_20220110 =
OffsetDateTime.of(2022, 1, 10, 0, 0, 0, 0, ZoneOffset.UTC);
+ private static final long MILLIS_20220110 =
+ JAVA_OFFSET_DATE_TIME_20220110.toInstant().toEpochMilli();
+ private static final int DAYS_BTW_EPOC_AND_20220110 =
+ (int) JAVA_OFFSET_DATE_TIME_20220110.toLocalDate().toEpochDay();
private static final Instant JAVA_INSTANT_DATE_TIME_20220110 =
Instant.ofEpochSecond(JAVA_OFFSET_DATE_TIME_20220110.toEpochSecond(), 0);
private static final LocalDateTime JAVA_LOCAL_DATE_TIME_20220110 =
@@ -232,10 +227,8 @@ public class DataGenerators {
HOUR_8_IN_MILLI,
// Although Avro logical type for timestamp fields are in micro seconds,
// AvroToRowDataConverters only looks for long value in milliseconds.
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
uuidBytes,
binaryBytes,
DecimalData.fromBigDecimal(BIG_DECIMAL_NEGATIVE, 9, 2),
@@ -305,12 +298,9 @@ public class DataGenerators {
genericRecord.put("time_field", HOUR_8_IN_MILLI);
// Now that AvroToRowDataConverters correctly supports microseconds,
// we must inject correct microsecond scale values into the Avro data.
+ genericRecord.put("ts_with_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put(
- "ts_with_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
- genericRecord.put(
- "ts_without_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
+ "ts_without_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put("ts_ns_with_zone_field", ICEBERG_MAX_NANOS_EPOCH);
genericRecord.put("ts_ns_without_zone_field", ICEBERG_MAX_NANOS_EPOCH);
diff --git a/flink/v2.3/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java b/flink/v2.3/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
index 5393c3958b..6a7a4a4699 100644
--- a/flink/v2.3/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
+++ b/flink/v2.3/flink/src/test/java/org/apache/iceberg/flink/DataGenerators.java
@@ -53,9 +53,6 @@ import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Types;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Days;
/**
* Util class to generate test data with extensive coverage different field types: from primitives
@@ -64,12 +61,6 @@ import org.joda.time.Days;
public class DataGenerators {
public static class Primitives implements DataGenerator {
- private static final DateTime JODA_DATETIME_EPOC =
- new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final DateTime JODA_DATETIME_20220110 =
- new DateTime(2022, 1, 10, 0, 0, 0, 0, DateTimeZone.UTC);
- private static final int DAYS_BTW_EPOC_AND_20220110 =
- Days.daysBetween(JODA_DATETIME_EPOC, JODA_DATETIME_20220110).getDays();
private static final int HOUR_8_IN_MILLI = (int) TimeUnit.HOURS.toMillis(8);
private static final int MICROS_OF_MILLI_20220110 = 456;
@@ -77,6 +68,10 @@ public class DataGenerators {
private static final LocalTime JAVA_LOCAL_TIME_HOUR8 = LocalTime.of(8, 0);
private static final OffsetDateTime JAVA_OFFSET_DATE_TIME_20220110 =
OffsetDateTime.of(2022, 1, 10, 0, 0, 0, 0, ZoneOffset.UTC);
+ private static final long MILLIS_20220110 =
+ JAVA_OFFSET_DATE_TIME_20220110.toInstant().toEpochMilli();
+ private static final int DAYS_BTW_EPOC_AND_20220110 =
+ (int) JAVA_OFFSET_DATE_TIME_20220110.toLocalDate().toEpochDay();
private static final Instant JAVA_INSTANT_DATE_TIME_20220110 =
Instant.ofEpochSecond(JAVA_OFFSET_DATE_TIME_20220110.toEpochSecond(), 0);
private static final LocalDateTime JAVA_LOCAL_DATE_TIME_20220110 =
@@ -232,10 +227,8 @@ public class DataGenerators {
HOUR_8_IN_MILLI,
// Although Avro logical type for timestamp fields are in micro seconds,
// AvroToRowDataConverters only looks for long value in milliseconds.
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
- TimestampData.fromEpochMillis(
- JODA_DATETIME_20220110.getMillis(), MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
+ TimestampData.fromEpochMillis(MILLIS_20220110, MICROS_OF_MILLI_20220110 * 1000),
uuidBytes,
binaryBytes,
DecimalData.fromBigDecimal(BIG_DECIMAL_NEGATIVE, 9, 2),
@@ -305,12 +298,9 @@ public class DataGenerators {
genericRecord.put("time_field", HOUR_8_IN_MILLI);
// Now that AvroToRowDataConverters correctly supports microseconds,
// we must inject correct microsecond scale values into the Avro data.
+ genericRecord.put("ts_with_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put(
- "ts_with_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
- genericRecord.put(
- "ts_without_zone_field",
- JODA_DATETIME_20220110.getMillis() * 1000L + MICROS_OF_MILLI_20220110);
+ "ts_without_zone_field", MILLIS_20220110 * 1000L + MICROS_OF_MILLI_20220110);
genericRecord.put("ts_ns_with_zone_field", ICEBERG_MAX_NANOS_EPOCH);
genericRecord.put("ts_ns_without_zone_field", ICEBERG_MAX_NANOS_EPOCH);
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
index 1cfdbdaf7c..2d32e3e823 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assumptions.assumeThat;
import java.io.File;
+import java.time.LocalDate;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,7 +60,6 @@ import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
@@ -1280,7 +1280,7 @@ public class TestAddFilesProcedure extends ExtensionsTestBase {
};
private static java.sql.Date toDate(String value) {
- return new java.sql.Date(DateTime.parse(value).getMillis());
+ return java.sql.Date.valueOf(LocalDate.parse(value));
}
private Dataset<Row> dateDF() {
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
index c88a907e0f..4a7d205466 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
@@ -22,6 +22,13 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.sql.Date;
import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
@@ -59,7 +66,6 @@ import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.BlockManagerId;
import org.apache.spark.storage.BlockManagerMaster;
import org.apache.spark.unsafe.types.UTF8String;
-import org.joda.time.DateTime;
import scala.collection.JavaConverters;
import scala.collection.Seq;
@@ -226,15 +232,14 @@ public class SparkUtil {
new EqualTo(
ref,
Literal.create(
- new Date(DateTime.parse(entry.getValue()).getMillis()),
- DataTypes.DateType)));
+ new Date(parseToEpochMillis(entry.getValue())), DataTypes.DateType)));
break;
case "timestamp":
filterExpressions.add(
new EqualTo(
ref,
Literal.create(
- new Timestamp(DateTime.parse(entry.getValue()).getMillis()),
+ new Timestamp(parseToEpochMillis(entry.getValue())),
DataTypes.TimestampType)));
break;
default:
@@ -249,6 +254,36 @@ public class SparkUtil {
return filterExpressions;
}
+ // Lenient ISO parser accepting either a date (e.g. "2021-01-01") or a date-time
+ // (e.g. "2021-01-01T12:34:56"), with an optional zone offset (e.g. "2021-01-01T12:34:56Z" or
+ // "2021-01-01T12:34:56+00:00"), defaulting any missing time fields to zero.
+ private static final DateTimeFormatter LENIENT_ISO_DATE_TIME =
+ new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .optionalStart()
+ .appendLiteral('T')
+ .append(DateTimeFormatter.ISO_LOCAL_TIME)
+ .optionalEnd()
+ .optionalStart()
+ .appendOffsetId()
+ .optionalEnd()
+ .parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
+ .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
+ .toFormatter();
+
+ private static long parseToEpochMillis(String value) {
+ TemporalAccessor parsed = LENIENT_ISO_DATE_TIME.parse(value);
+ LocalDateTime dateTime = LocalDateTime.from(parsed);
+ // Respect an explicit offset when present (as Joda's DateTime.parse did), otherwise
+ // interpret the value in the system default zone.
+ ZoneId zone =
+ parsed.isSupported(ChronoField.OFFSET_SECONDS)
+ ? ZoneOffset.from(parsed)
+ : ZoneId.systemDefault();
+ return dateTime.atZone(zone).toInstant().toEpochMilli();
+ }
+
public static String toColumnName(NamedReference ref) {
return DOT.join(ref.fieldNames());
}
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
new file mode 100644
index 0000000000..b03438c71b
--- /dev/null
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.spark.sql.catalyst.expressions.EqualTo;
+import org.apache.spark.sql.catalyst.expressions.Expression;
+import org.apache.spark.sql.catalyst.expressions.Literal;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.Test;
+
+public class TestSparkUtil {
+
+ private static final StructType TIMESTAMP_SCHEMA =
+ new StructType(
+ new StructField[] {
+ new StructField("ts", DataTypes.TimestampType, true, Metadata.empty())
+ });
+
+ @Test
+ public void partitionMapToExpressionWithOffset() {
+ // an explicit, non-UTC offset must be respected regardless of the JVM default zone, matching
+ // the previous Joda-based behavior
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56+05:00").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56+05:00")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void partitionMapToExpressionWithUtcOffset() {
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56Z").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56Z")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void partitionMapToExpressionWithoutOffset() {
+ // without an offset the value is interpreted in the system default zone
+ long expectedMicros =
+ LocalDateTime.parse("2021-01-01T12:34:56")
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli()
+ * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56")).isEqualTo(expectedMicros);
+ }
+
+ private static long timestampLiteralMicros(String value) {
+ List<Expression> expressions =
+ SparkUtil.partitionMapToExpression(TIMESTAMP_SCHEMA, ImmutableMap.of("ts", value));
+ assertThat(expressions).hasSize(1);
+ Literal literal = (Literal) ((EqualTo) expressions.get(0)).right();
+ // Spark stores TIMESTAMP literals as microseconds since the epoch
+ return (Long) literal.value();
+ }
+}
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
index f721ff3dec..133bb8e229 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
@@ -38,7 +38,6 @@ import org.apache.iceberg.spark.SparkSessionCatalog;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestTemplate;
@@ -201,7 +200,7 @@ public class TestTimestampWithoutZone extends CatalogTestBase {
}
private Timestamp toTimestamp(String value) {
- return new Timestamp(DateTime.parse(value).getMillis());
+ return Timestamp.valueOf(LocalDateTime.parse(value));
}
private LocalDateTime toLocalDateTime(String value) {
diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
index 29993380b5..9a33ff8204 100644
--- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
+++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assumptions.assumeThat;
import java.io.File;
+import java.time.LocalDate;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,7 +60,6 @@ import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
@@ -1278,7 +1278,7 @@ public class TestAddFilesProcedure extends ExtensionsTestBase {
};
private static java.sql.Date toDate(String value) {
- return new java.sql.Date(DateTime.parse(value).getMillis());
+ return java.sql.Date.valueOf(LocalDate.parse(value));
}
private Dataset<Row> dateDF() {
diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
index ef9990c7bd..39cd3d77b6 100644
--- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
+++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
@@ -22,6 +22,13 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.sql.Date;
import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
@@ -59,7 +66,6 @@ import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.BlockManagerId;
import org.apache.spark.storage.BlockManagerMaster;
import org.apache.spark.unsafe.types.UTF8String;
-import org.joda.time.DateTime;
import scala.collection.JavaConverters;
import scala.collection.Seq;
@@ -227,15 +233,14 @@ public class SparkUtil {
new EqualTo(
ref,
Literal.create(
- new Date(DateTime.parse(entry.getValue()).getMillis()),
- DataTypes.DateType)));
+ new Date(parseToEpochMillis(entry.getValue())), DataTypes.DateType)));
break;
case "timestamp":
filterExpressions.add(
new EqualTo(
ref,
Literal.create(
- new Timestamp(DateTime.parse(entry.getValue()).getMillis()),
+ new Timestamp(parseToEpochMillis(entry.getValue())),
DataTypes.TimestampType)));
break;
default:
@@ -250,6 +255,36 @@ public class SparkUtil {
return filterExpressions;
}
+ // Lenient ISO parser accepting either a date (e.g. "2021-01-01") or a date-time
+ // (e.g. "2021-01-01T12:34:56"), with an optional zone offset (e.g. "2021-01-01T12:34:56Z" or
+ // "2021-01-01T12:34:56+00:00"), defaulting any missing time fields to zero.
+ private static final DateTimeFormatter LENIENT_ISO_DATE_TIME =
+ new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .optionalStart()
+ .appendLiteral('T')
+ .append(DateTimeFormatter.ISO_LOCAL_TIME)
+ .optionalEnd()
+ .optionalStart()
+ .appendOffsetId()
+ .optionalEnd()
+ .parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
+ .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
+ .toFormatter();
+
+ private static long parseToEpochMillis(String value) {
+ TemporalAccessor parsed = LENIENT_ISO_DATE_TIME.parse(value);
+ LocalDateTime dateTime = LocalDateTime.from(parsed);
+ // Respect an explicit offset when present (as Joda's DateTime.parse did), otherwise
+ // interpret the value in the system default zone.
+ ZoneId zone =
+ parsed.isSupported(ChronoField.OFFSET_SECONDS)
+ ? ZoneOffset.from(parsed)
+ : ZoneId.systemDefault();
+ return dateTime.atZone(zone).toInstant().toEpochMilli();
+ }
+
public static String toColumnName(NamedReference ref) {
return DOT.join(ref.fieldNames());
}
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
new file mode 100644
index 0000000000..1957068677
--- /dev/null
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.spark.sql.catalyst.expressions.EqualTo;
+import org.apache.spark.sql.catalyst.expressions.Expression;
+import org.apache.spark.sql.catalyst.expressions.Literal;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.Test;
+
+public class TestSparkUtil {
+
+ private static final StructType TIMESTAMP_SCHEMA =
+ new StructType(
+ new StructField[] {
+ new StructField("ts", DataTypes.TimestampType, true, Metadata.empty())
+ });
+
+ @Test
+ public void testPartitionMapToExpressionWithOffset() {
+ // an explicit, non-UTC offset must be respected regardless of the JVM default zone, matching
+ // the previous Joda-based behavior
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56+05:00").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56+05:00")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void testPartitionMapToExpressionWithUtcOffset() {
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56Z").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56Z")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void testPartitionMapToExpressionWithoutOffset() {
+ // without an offset the value is interpreted in the system default zone
+ long expectedMicros =
+ LocalDateTime.parse("2021-01-01T12:34:56")
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli()
+ * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56")).isEqualTo(expectedMicros);
+ }
+
+ private static long timestampLiteralMicros(String value) {
+ List<Expression> expressions =
+ SparkUtil.partitionMapToExpression(TIMESTAMP_SCHEMA, ImmutableMap.of("ts", value));
+ assertThat(expressions).hasSize(1);
+ Literal literal = (Literal) ((EqualTo) expressions.get(0)).right();
+ // Spark stores TIMESTAMP literals as microseconds since the epoch
+ return (Long) literal.value();
+ }
+}
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
index f721ff3dec..133bb8e229 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
@@ -38,7 +38,6 @@ import org.apache.iceberg.spark.SparkSessionCatalog;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestTemplate;
@@ -201,7 +200,7 @@ public class TestTimestampWithoutZone extends CatalogTestBase {
}
private Timestamp toTimestamp(String value) {
- return new Timestamp(DateTime.parse(value).getMillis());
+ return Timestamp.valueOf(LocalDateTime.parse(value));
}
private LocalDateTime toLocalDateTime(String value) {
diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
index 29993380b5..9a33ff8204 100644
--- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
+++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assumptions.assumeThat;
import java.io.File;
+import java.time.LocalDate;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,7 +60,6 @@ import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
@@ -1278,7 +1278,7 @@ public class TestAddFilesProcedure extends ExtensionsTestBase {
};
private static java.sql.Date toDate(String value) {
- return new java.sql.Date(DateTime.parse(value).getMillis());
+ return java.sql.Date.valueOf(LocalDate.parse(value));
}
private Dataset<Row> dateDF() {
diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
index ef9990c7bd..39cd3d77b6 100644
--- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
+++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
@@ -22,6 +22,13 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.sql.Date;
import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
@@ -59,7 +66,6 @@ import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.BlockManagerId;
import org.apache.spark.storage.BlockManagerMaster;
import org.apache.spark.unsafe.types.UTF8String;
-import org.joda.time.DateTime;
import scala.collection.JavaConverters;
import scala.collection.Seq;
@@ -227,15 +233,14 @@ public class SparkUtil {
new EqualTo(
ref,
Literal.create(
- new Date(DateTime.parse(entry.getValue()).getMillis()),
- DataTypes.DateType)));
+ new Date(parseToEpochMillis(entry.getValue())), DataTypes.DateType)));
break;
case "timestamp":
filterExpressions.add(
new EqualTo(
ref,
Literal.create(
- new Timestamp(DateTime.parse(entry.getValue()).getMillis()),
+ new Timestamp(parseToEpochMillis(entry.getValue())),
DataTypes.TimestampType)));
break;
default:
@@ -250,6 +255,36 @@ public class SparkUtil {
return filterExpressions;
}
+ // Lenient ISO parser accepting either a date (e.g. "2021-01-01") or a date-time
+ // (e.g. "2021-01-01T12:34:56"), with an optional zone offset (e.g. "2021-01-01T12:34:56Z" or
+ // "2021-01-01T12:34:56+00:00"), defaulting any missing time fields to zero.
+ private static final DateTimeFormatter LENIENT_ISO_DATE_TIME =
+ new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .optionalStart()
+ .appendLiteral('T')
+ .append(DateTimeFormatter.ISO_LOCAL_TIME)
+ .optionalEnd()
+ .optionalStart()
+ .appendOffsetId()
+ .optionalEnd()
+ .parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
+ .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
+ .toFormatter();
+
+ private static long parseToEpochMillis(String value) {
+ TemporalAccessor parsed = LENIENT_ISO_DATE_TIME.parse(value);
+ LocalDateTime dateTime = LocalDateTime.from(parsed);
+ // Respect an explicit offset when present (as Joda's DateTime.parse did), otherwise
+ // interpret the value in the system default zone.
+ ZoneId zone =
+ parsed.isSupported(ChronoField.OFFSET_SECONDS)
+ ? ZoneOffset.from(parsed)
+ : ZoneId.systemDefault();
+ return dateTime.atZone(zone).toInstant().toEpochMilli();
+ }
+
public static String toColumnName(NamedReference ref) {
return DOT.join(ref.fieldNames());
}
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
new file mode 100644
index 0000000000..1957068677
--- /dev/null
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.spark.sql.catalyst.expressions.EqualTo;
+import org.apache.spark.sql.catalyst.expressions.Expression;
+import org.apache.spark.sql.catalyst.expressions.Literal;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.Test;
+
+public class TestSparkUtil {
+
+ private static final StructType TIMESTAMP_SCHEMA =
+ new StructType(
+ new StructField[] {
+ new StructField("ts", DataTypes.TimestampType, true, Metadata.empty())
+ });
+
+ @Test
+ public void testPartitionMapToExpressionWithOffset() {
+ // an explicit, non-UTC offset must be respected regardless of the JVM default zone, matching
+ // the previous Joda-based behavior
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56+05:00").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56+05:00")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void testPartitionMapToExpressionWithUtcOffset() {
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56Z").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56Z")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void testPartitionMapToExpressionWithoutOffset() {
+ // without an offset the value is interpreted in the system default zone
+ long expectedMicros =
+ LocalDateTime.parse("2021-01-01T12:34:56")
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli()
+ * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56")).isEqualTo(expectedMicros);
+ }
+
+ private static long timestampLiteralMicros(String value) {
+ List<Expression> expressions =
+ SparkUtil.partitionMapToExpression(TIMESTAMP_SCHEMA, ImmutableMap.of("ts", value));
+ assertThat(expressions).hasSize(1);
+ Literal literal = (Literal) ((EqualTo) expressions.get(0)).right();
+ // Spark stores TIMESTAMP literals as microseconds since the epoch
+ return (Long) literal.value();
+ }
+}
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
index f721ff3dec..133bb8e229 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
@@ -38,7 +38,6 @@ import org.apache.iceberg.spark.SparkSessionCatalog;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestTemplate;
@@ -201,7 +200,7 @@ public class TestTimestampWithoutZone extends CatalogTestBase {
}
private Timestamp toTimestamp(String value) {
- return new Timestamp(DateTime.parse(value).getMillis());
+ return Timestamp.valueOf(LocalDateTime.parse(value));
}
private LocalDateTime toLocalDateTime(String value) {
diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
index 29993380b5..9a33ff8204 100644
--- a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
+++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java
@@ -23,6 +23,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assumptions.assumeThat;
import java.io.File;
+import java.time.LocalDate;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,7 +60,6 @@ import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
@@ -1278,7 +1278,7 @@ public class TestAddFilesProcedure extends ExtensionsTestBase {
};
private static java.sql.Date toDate(String value) {
- return new java.sql.Date(DateTime.parse(value).getMillis());
+ return java.sql.Date.valueOf(LocalDate.parse(value));
}
private Dataset<Row> dateDF() {
diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
index ef9990c7bd..39cd3d77b6 100644
--- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
+++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
@@ -22,6 +22,13 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.sql.Date;
import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
@@ -59,7 +66,6 @@ import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.BlockManagerId;
import org.apache.spark.storage.BlockManagerMaster;
import org.apache.spark.unsafe.types.UTF8String;
-import org.joda.time.DateTime;
import scala.collection.JavaConverters;
import scala.collection.Seq;
@@ -227,15 +233,14 @@ public class SparkUtil {
new EqualTo(
ref,
Literal.create(
- new Date(DateTime.parse(entry.getValue()).getMillis()),
- DataTypes.DateType)));
+ new Date(parseToEpochMillis(entry.getValue())), DataTypes.DateType)));
break;
case "timestamp":
filterExpressions.add(
new EqualTo(
ref,
Literal.create(
- new Timestamp(DateTime.parse(entry.getValue()).getMillis()),
+ new Timestamp(parseToEpochMillis(entry.getValue())),
DataTypes.TimestampType)));
break;
default:
@@ -250,6 +255,36 @@ public class SparkUtil {
return filterExpressions;
}
+ // Lenient ISO parser accepting either a date (e.g. "2021-01-01") or a date-time
+ // (e.g. "2021-01-01T12:34:56"), with an optional zone offset (e.g. "2021-01-01T12:34:56Z" or
+ // "2021-01-01T12:34:56+00:00"), defaulting any missing time fields to zero.
+ private static final DateTimeFormatter LENIENT_ISO_DATE_TIME =
+ new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .optionalStart()
+ .appendLiteral('T')
+ .append(DateTimeFormatter.ISO_LOCAL_TIME)
+ .optionalEnd()
+ .optionalStart()
+ .appendOffsetId()
+ .optionalEnd()
+ .parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
+ .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
+ .toFormatter();
+
+ private static long parseToEpochMillis(String value) {
+ TemporalAccessor parsed = LENIENT_ISO_DATE_TIME.parse(value);
+ LocalDateTime dateTime = LocalDateTime.from(parsed);
+ // Respect an explicit offset when present (as Joda's DateTime.parse did), otherwise
+ // interpret the value in the system default zone.
+ ZoneId zone =
+ parsed.isSupported(ChronoField.OFFSET_SECONDS)
+ ? ZoneOffset.from(parsed)
+ : ZoneId.systemDefault();
+ return dateTime.atZone(zone).toInstant().toEpochMilli();
+ }
+
public static String toColumnName(NamedReference ref) {
return DOT.join(ref.fieldNames());
}
diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
new file mode 100644
index 0000000000..1957068677
--- /dev/null
+++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkUtil.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.spark.sql.catalyst.expressions.EqualTo;
+import org.apache.spark.sql.catalyst.expressions.Expression;
+import org.apache.spark.sql.catalyst.expressions.Literal;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.Test;
+
+public class TestSparkUtil {
+
+ private static final StructType TIMESTAMP_SCHEMA =
+ new StructType(
+ new StructField[] {
+ new StructField("ts", DataTypes.TimestampType, true, Metadata.empty())
+ });
+
+ @Test
+ public void testPartitionMapToExpressionWithOffset() {
+ // an explicit, non-UTC offset must be respected regardless of the JVM default zone, matching
+ // the previous Joda-based behavior
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56+05:00").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56+05:00")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void testPartitionMapToExpressionWithUtcOffset() {
+ long expectedMicros =
+ OffsetDateTime.parse("2021-01-01T12:34:56Z").toInstant().toEpochMilli() * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56Z")).isEqualTo(expectedMicros);
+ }
+
+ @Test
+ public void testPartitionMapToExpressionWithoutOffset() {
+ // without an offset the value is interpreted in the system default zone
+ long expectedMicros =
+ LocalDateTime.parse("2021-01-01T12:34:56")
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli()
+ * 1000;
+ assertThat(timestampLiteralMicros("2021-01-01T12:34:56")).isEqualTo(expectedMicros);
+ }
+
+ private static long timestampLiteralMicros(String value) {
+ List<Expression> expressions =
+ SparkUtil.partitionMapToExpression(TIMESTAMP_SCHEMA, ImmutableMap.of("ts", value));
+ assertThat(expressions).hasSize(1);
+ Literal literal = (Literal) ((EqualTo) expressions.get(0)).right();
+ // Spark stores TIMESTAMP literals as microseconds since the epoch
+ return (Long) literal.value();
+ }
+}
diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
index e3f115abb0..34f880d536 100644
--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
+++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java
@@ -37,7 +37,6 @@ import org.apache.iceberg.spark.CatalogTestBase;
import org.apache.iceberg.spark.SparkSessionCatalog;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
-import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestTemplate;
@@ -176,7 +175,7 @@ public class TestTimestampWithoutZone extends CatalogTestBase {
}
private Timestamp toTimestamp(String value) {
- return new Timestamp(DateTime.parse(value).getMillis());
+ return Timestamp.valueOf(LocalDateTime.parse(value));
}
private LocalDateTime toLocalDateTime(String value) {