You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ah...@apache.org on 2024/10/10 14:19:23 UTC

(beam) branch master updated: [Managed Iceberg] Add support for TIMESTAMP, TIME, and DATE types (#32688)

This is an automated email from the ASF dual-hosted git repository.

ahmedabualsaud pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 9ceb14e23f4 [Managed Iceberg] Add support for TIMESTAMP, TIME, and DATE types (#32688)
9ceb14e23f4 is described below

commit 9ceb14e23f413ba4b48f0918e30f5a47da04931e
Author: Ahmed Abualsaud <65...@users.noreply.github.com>
AuthorDate: Thu Oct 10 17:19:16 2024 +0300

    [Managed Iceberg] Add support for TIMESTAMP, TIME, and DATE types (#32688)
    
    * support timestamp, time, date types
    
    * add to changes md
    
    * always write java time LocalDateTime for iceberg TIMESTAMP
    
    * update java doc
    
    * add timezone support with Strings
    
    * clean up; reading iceberg timestamptz will return sqltype.datetime
    
    * support string, long, sql.datetime, and datetime; timestamp returns sql.datetime and timestamptTZ returns datetime
---
 .../IO_Iceberg_Integration_Tests.json              |   2 +-
 CHANGES.md                                         |   1 +
 .../sdk/io/iceberg/hive/IcebergHiveCatalogIT.java  |  12 ++
 .../org/apache/beam/sdk/io/iceberg/IcebergIO.java  |  34 ++++-
 .../apache/beam/sdk/io/iceberg/IcebergUtils.java   | 166 ++++++++++++++++++---
 .../apache/beam/sdk/io/iceberg/IcebergIOIT.java    |  15 +-
 .../beam/sdk/io/iceberg/IcebergUtilsTest.java      | 166 ++++++++++++++++++---
 7 files changed, 355 insertions(+), 41 deletions(-)

diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
index 3f63c0c9975..bbdc3a3910e 100644
--- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json
+++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run",
-    "modification": 2
+    "modification": 3
 }
diff --git a/CHANGES.md b/CHANGES.md
index 774abefcb06..6a70a49b2ab 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -64,6 +64,7 @@
 
 * Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
 * [Managed Iceberg] Support creating tables if needed ([#32686](https://github.com/apache/beam/pull/32686))
+* [Managed Iceberg] Add support for TIMESTAMP, TIME, and DATE types ([#32688](https://github.com/apache/beam/pull/32688))
 
 ## New Features / Improvements
 
diff --git a/sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/IcebergHiveCatalogIT.java b/sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/IcebergHiveCatalogIT.java
index 54a4998d37f..ca4d862c2c7 100644
--- a/sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/IcebergHiveCatalogIT.java
+++ b/sdks/java/io/iceberg/hive/src/test/java/org/apache/beam/sdk/io/iceberg/hive/IcebergHiveCatalogIT.java
@@ -32,6 +32,7 @@ import org.apache.beam.sdk.io.iceberg.IcebergUtils;
 import org.apache.beam.sdk.io.iceberg.hive.testutils.HiveMetastoreExtension;
 import org.apache.beam.sdk.managed.Managed;
 import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
@@ -64,7 +65,10 @@ import org.apache.iceberg.io.DataWriter;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.util.DateTimeUtil;
 import org.apache.thrift.TException;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -100,6 +104,10 @@ public class IcebergHiveCatalogIT {
           .addArrayField("arr_long", Schema.FieldType.INT64)
           .addRowField("row", NESTED_ROW_SCHEMA)
           .addNullableRowField("nullable_row", NESTED_ROW_SCHEMA)
+          .addDateTimeField("datetime_tz")
+          .addLogicalTypeField("datetime", SqlTypes.DATETIME)
+          .addLogicalTypeField("date", SqlTypes.DATE)
+          .addLogicalTypeField("time", SqlTypes.TIME)
           .build();
 
   private static final SimpleFunction<Long, Row> ROW_FUNC =
@@ -127,6 +135,10 @@ public class IcebergHiveCatalogIT {
               .addValue(LongStream.range(1, num % 10).boxed().collect(Collectors.toList()))
               .addValue(nestedRow)
               .addValue(num % 2 == 0 ? null : nestedRow)
+              .addValue(new DateTime(num).withZone(DateTimeZone.forOffsetHoursMinutes(3, 25)))
+              .addValue(DateTimeUtil.timestampFromMicros(num))
+              .addValue(DateTimeUtil.dateFromDays(Integer.parseInt(strNum)))
+              .addValue(DateTimeUtil.timeFromMicros(num))
               .build();
         }
       };
diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
index d12f8914a33..fa4ff9714c7 100644
--- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
+++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java
@@ -150,7 +150,16 @@ import org.joda.time.Duration;
  *     <td> DOUBLE </td> <td> DOUBLE </td>
  *   </tr>
  *   <tr>
- *     <td> DATETIME </td> <td> STRING </td>
+ *     <td> SqlTypes.DATETIME </td> <td> TIMESTAMP </td>
+ *   </tr>
+ *   <tr>
+ *     <td> DATETIME </td> <td> TIMESTAMPTZ </td>
+ *   </tr>
+ *   <tr>
+ *     <td> SqlTypes.DATE </td> <td> DATE </td>
+ *   </tr>
+ *   <tr>
+ *     <td> SqlTypes.TIME </td> <td> TIME </td>
  *   </tr>
  *   <tr>
  *     <td> ITERABLE </td> <td> LIST </td>
@@ -166,6 +175,29 @@ import org.joda.time.Duration;
  *   </tr>
  * </table>
  *
+ * <p><b>Note:</b> {@code SqlTypes} are Beam logical types.
+ *
+ * <h3>Note on timestamps</h3>
+ *
+ * <p>For an existing table, the following Beam types are supported for both {@code timestamp} and
+ * {@code timestamptz}:
+ *
+ * <ul>
+ *   <li>{@code SqlTypes.DATETIME} --> Using a {@link java.time.LocalDateTime} object
+ *   <li>{@code DATETIME} --> Using a {@link org.joda.time.DateTime} object
+ *   <li>{@code INT64} --> Using a {@link Long} representing micros since EPOCH
+ *   <li>{@code STRING} --> Using a timestamp {@link String} representation (e.g. {@code
+ *       "2024-10-08T13:18:20.053+03:27"})
+ * </ul>
+ *
+ * <p><b>Note</b>: If you expect Beam to create the Iceberg table at runtime, please provide {@code
+ * SqlTypes.DATETIME} for a {@code timestamp} column and {@code DATETIME} for a {@code timestamptz}
+ * column. If the table does not exist, Beam will treat {@code STRING} and {@code INT64} at
+ * face-value and create equivalent column types.
+ *
+ * <p>For Iceberg reads, the connector will produce Beam {@code SqlTypes.DATETIME} types for
+ * Iceberg's {@code timestamp} and {@code DATETIME} types for {@code timestamptz}.
+ *
  * <h3>Dynamic Destinations</h3>
  *
  * <p>Managed Iceberg supports writing to dynamic destinations. To do so, please provide an
diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
index acd9b25a6a5..ef19a588136 100644
--- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
+++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java
@@ -20,12 +20,18 @@ package org.apache.beam.sdk.io.iceberg;
 import static org.apache.beam.sdk.util.Preconditions.checkArgumentNotNull;
 
 import java.nio.ByteBuffer;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
 import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes;
 import org.apache.beam.sdk.util.Preconditions;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
@@ -34,15 +40,13 @@ import org.apache.iceberg.data.GenericRecord;
 import org.apache.iceberg.data.Record;
 import org.apache.iceberg.types.Type;
 import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
+import org.joda.time.Instant;
 
-/** Utilities for converting between Beam and Iceberg types. */
+/** Utilities for converting between Beam and Iceberg types, made public for user's convenience. */
 public class IcebergUtils {
-  // This is made public for users convenience, as many may have more experience working with
-  // Iceberg types.
-
   private IcebergUtils() {}
 
   private static final Map<Schema.TypeName, Type> BEAM_TYPES_TO_ICEBERG_TYPES =
@@ -54,6 +58,14 @@ public class IcebergUtils {
           .put(Schema.TypeName.DOUBLE, Types.DoubleType.get())
           .put(Schema.TypeName.STRING, Types.StringType.get())
           .put(Schema.TypeName.BYTES, Types.BinaryType.get())
+          .put(Schema.TypeName.DATETIME, Types.TimestampType.withZone())
+          .build();
+
+  private static final Map<String, Type> BEAM_LOGICAL_TYPES_TO_ICEBERG_TYPES =
+      ImmutableMap.<String, Type>builder()
+          .put(SqlTypes.DATE.getIdentifier(), Types.DateType.get())
+          .put(SqlTypes.TIME.getIdentifier(), Types.TimeType.get())
+          .put(SqlTypes.DATETIME.getIdentifier(), Types.TimestampType.withoutZone())
           .build();
 
   private static Schema.FieldType icebergTypeToBeamFieldType(final Type type) {
@@ -69,9 +81,15 @@ public class IcebergUtils {
       case DOUBLE:
         return Schema.FieldType.DOUBLE;
       case DATE:
+        return Schema.FieldType.logicalType(SqlTypes.DATE);
       case TIME:
-      case TIMESTAMP: // TODO: Logical types?
-        return Schema.FieldType.DATETIME;
+        return Schema.FieldType.logicalType(SqlTypes.TIME);
+      case TIMESTAMP:
+        Types.TimestampType ts = (Types.TimestampType) type.asPrimitiveType();
+        if (ts.shouldAdjustToUTC()) {
+          return Schema.FieldType.DATETIME;
+        }
+        return Schema.FieldType.logicalType(SqlTypes.DATETIME);
       case STRING:
         return Schema.FieldType.STRING;
       case UUID:
@@ -151,6 +169,14 @@ public class IcebergUtils {
       // other types.
       return new TypeAndMaxId(
           --nestedFieldId, BEAM_TYPES_TO_ICEBERG_TYPES.get(beamType.getTypeName()));
+    } else if (beamType.getTypeName().isLogicalType()) {
+      String logicalTypeIdentifier =
+          checkArgumentNotNull(beamType.getLogicalType()).getIdentifier();
+      @Nullable Type type = BEAM_LOGICAL_TYPES_TO_ICEBERG_TYPES.get(logicalTypeIdentifier);
+      if (type == null) {
+        throw new RuntimeException("Unsupported Beam logical type " + logicalTypeIdentifier);
+      }
+      return new TypeAndMaxId(--nestedFieldId, type);
     } else if (beamType.getTypeName().isCollectionType()) { // ARRAY or ITERABLE
       Schema.FieldType beamCollectionType =
           Preconditions.checkArgumentNotNull(beamType.getCollectionElementType());
@@ -227,8 +253,6 @@ public class IcebergUtils {
    *
    * <p>The following unsupported Beam types will be defaulted to {@link Types.StringType}:
    * <li>{@link Schema.TypeName.DECIMAL}
-   * <li>{@link Schema.TypeName.DATETIME}
-   * <li>{@link Schema.TypeName.LOGICAL_TYPE}
    */
   public static org.apache.iceberg.Schema beamSchemaToIcebergSchema(final Schema schema) {
     List<Types.NestedField> fields = new ArrayList<>(schema.getFieldCount());
@@ -282,12 +306,20 @@ public class IcebergUtils {
         Optional.ofNullable(value.getDouble(name)).ifPresent(v -> rec.setField(name, v));
         break;
       case DATE:
-        throw new UnsupportedOperationException("Date fields not yet supported");
+        Optional.ofNullable(value.getLogicalTypeValue(name, LocalDate.class))
+            .ifPresent(v -> rec.setField(name, v));
+        break;
       case TIME:
-        throw new UnsupportedOperationException("Time fields not yet supported");
+        Optional.ofNullable(value.getLogicalTypeValue(name, LocalTime.class))
+            .ifPresent(v -> rec.setField(name, v));
+        break;
       case TIMESTAMP:
-        Optional.ofNullable(value.getDateTime(name))
-            .ifPresent(v -> rec.setField(name, v.getMillis()));
+        Object val = value.getValue(name);
+        if (val == null) {
+          break;
+        }
+        Types.TimestampType ts = (Types.TimestampType) field.type().asPrimitiveType();
+        rec.setField(name, getIcebergTimestampValue(val, ts.shouldAdjustToUTC()));
         break;
       case STRING:
         Optional.ofNullable(value.getString(name)).ifPresent(v -> rec.setField(name, v));
@@ -322,6 +354,55 @@ public class IcebergUtils {
     }
   }
 
+  /**
+   * Returns the appropriate value for an Iceberg timestamp field
+   *
+   * <p>If `timestamp`, we resolve incoming values to a {@link LocalDateTime}.
+   *
+   * <p>If `timestamptz`, we resolve to a UTC {@link OffsetDateTime}. Iceberg already resolves all
+   * incoming timestamps to UTC, so there is no harm in doing it from our side.
+   *
+   * <p>Valid types are:
+   *
+   * <ul>
+   *   <li>{@link SqlTypes.DATETIME} --> {@link LocalDateTime}
+   *   <li>{@link Schema.FieldType.DATETIME} --> {@link Instant}
+   *   <li>{@link Schema.FieldType.INT64} --> {@link Long}
+   *   <li>{@link Schema.FieldType.STRING} --> {@link String}
+   * </ul>
+   */
+  private static Object getIcebergTimestampValue(Object beamValue, boolean shouldAdjustToUtc) {
+    // timestamptz
+    if (shouldAdjustToUtc) {
+      if (beamValue instanceof LocalDateTime) { // SqlTypes.DATETIME
+        return OffsetDateTime.of((LocalDateTime) beamValue, ZoneOffset.UTC);
+      } else if (beamValue instanceof Instant) { // FieldType.DATETIME
+        return DateTimeUtil.timestamptzFromMicros(((Instant) beamValue).getMillis() * 1000L);
+      } else if (beamValue instanceof Long) { // FieldType.INT64
+        return DateTimeUtil.timestamptzFromMicros((Long) beamValue);
+      } else if (beamValue instanceof String) { // FieldType.STRING
+        return OffsetDateTime.parse((String) beamValue).withOffsetSameInstant(ZoneOffset.UTC);
+      } else {
+        throw new UnsupportedOperationException(
+            "Unsupported Beam type for Iceberg timestamp with timezone: " + beamValue.getClass());
+      }
+    }
+
+    // timestamp
+    if (beamValue instanceof LocalDateTime) { // SqlType.DATETIME
+      return beamValue;
+    } else if (beamValue instanceof Instant) { // FieldType.DATETIME
+      return DateTimeUtil.timestampFromMicros(((Instant) beamValue).getMillis() * 1000L);
+    } else if (beamValue instanceof Long) { // FieldType.INT64
+      return DateTimeUtil.timestampFromMicros((Long) beamValue);
+    } else if (beamValue instanceof String) { // FieldType.STRING
+      return LocalDateTime.parse((String) beamValue);
+    } else {
+      throw new UnsupportedOperationException(
+          "Unsupported Beam type for Iceberg timestamp with timezone: " + beamValue.getClass());
+    }
+  }
+
   /** Converts an Iceberg {@link Record} to a Beam {@link Row}. */
   public static Row icebergRecordToBeamRow(Schema schema, Record record) {
     Row.Builder rowBuilder = Row.withSchema(schema);
@@ -345,16 +426,17 @@ public class IcebergUtils {
         case FLOAT: // Iceberg and Beam both use float
         case DOUBLE: // Iceberg and Beam both use double
         case STRING: // Iceberg and Beam both use String
-        case BOOLEAN: // Iceberg and Beam both use String
+        case BOOLEAN: // Iceberg and Beam both use boolean
         case ARRAY:
         case ITERABLE:
         case MAP:
           rowBuilder.addValue(icebergValue);
           break;
         case DATETIME:
-          // Iceberg uses a long for millis; Beam uses joda time DateTime
-          long millis = (long) icebergValue;
-          rowBuilder.addValue(new DateTime(millis, DateTimeZone.UTC));
+          // Iceberg uses a long for micros.
+          // Beam DATETIME uses joda's DateTime, which only supports millis,
+          // so we do lose some precision here
+          rowBuilder.addValue(getBeamDateTimeValue(icebergValue));
           break;
         case BYTES:
           // Iceberg uses ByteBuffer; Beam uses byte[]
@@ -369,8 +451,8 @@ public class IcebergUtils {
           rowBuilder.addValue(icebergRecordToBeamRow(nestedSchema, nestedRecord));
           break;
         case LOGICAL_TYPE:
-          throw new UnsupportedOperationException(
-              "Cannot convert iceberg field to Beam logical type");
+          rowBuilder.addValue(getLogicalTypeValue(icebergValue, field.getType()));
+          break;
         default:
           throw new UnsupportedOperationException(
               "Unsupported Beam type: " + field.getType().getTypeName());
@@ -378,4 +460,50 @@ public class IcebergUtils {
     }
     return rowBuilder.build();
   }
+
+  private static DateTime getBeamDateTimeValue(Object icebergValue) {
+    long micros;
+    if (icebergValue instanceof OffsetDateTime) {
+      micros = DateTimeUtil.microsFromTimestamptz((OffsetDateTime) icebergValue);
+    } else if (icebergValue instanceof LocalDateTime) {
+      micros = DateTimeUtil.microsFromTimestamp((LocalDateTime) icebergValue);
+    } else if (icebergValue instanceof Long) {
+      micros = (long) icebergValue;
+    } else if (icebergValue instanceof String) {
+      return DateTime.parse((String) icebergValue);
+    } else {
+      throw new UnsupportedOperationException(
+          "Unsupported Iceberg type for Beam type DATETIME: " + icebergValue.getClass());
+    }
+    return new DateTime(micros / 1000L);
+  }
+
+  private static Object getLogicalTypeValue(Object icebergValue, Schema.FieldType type) {
+    if (icebergValue instanceof String) {
+      String strValue = (String) icebergValue;
+      if (type.isLogicalType(SqlTypes.DATE.getIdentifier())) {
+        return LocalDate.parse(strValue);
+      } else if (type.isLogicalType(SqlTypes.TIME.getIdentifier())) {
+        return LocalTime.parse(strValue);
+      } else if (type.isLogicalType(SqlTypes.DATETIME.getIdentifier())) {
+        return LocalDateTime.parse(strValue);
+      }
+    } else if (icebergValue instanceof Long) {
+      if (type.isLogicalType(SqlTypes.TIME.getIdentifier())) {
+        return DateTimeUtil.timeFromMicros((Long) icebergValue);
+      } else if (type.isLogicalType(SqlTypes.DATETIME.getIdentifier())) {
+        return DateTimeUtil.timestampFromMicros((Long) icebergValue);
+      }
+    } else if (icebergValue instanceof Integer
+        && type.isLogicalType(SqlTypes.DATE.getIdentifier())) {
+      return DateTimeUtil.dateFromDays((Integer) icebergValue);
+    } else if (icebergValue instanceof OffsetDateTime
+        && type.isLogicalType(SqlTypes.DATETIME.getIdentifier())) {
+      return ((OffsetDateTime) icebergValue)
+          .withOffsetSameInstant(ZoneOffset.UTC)
+          .toLocalDateTime();
+    }
+    // LocalDateTime, LocalDate, LocalTime
+    return icebergValue;
+  }
 }
diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOIT.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOIT.java
index a5c034ac901..5df8604699a 100644
--- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOIT.java
+++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergIOIT.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.io.iceberg;
 
+import static org.apache.beam.sdk.schemas.Schema.FieldType;
 import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -36,6 +37,7 @@ import java.util.stream.LongStream;
 import java.util.stream.Stream;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.managed.Managed;
+import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
@@ -72,7 +74,10 @@ import org.apache.iceberg.io.DataWriter;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.util.DateTimeUtil;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
@@ -107,9 +112,13 @@ public class IcebergIOIT implements Serializable {
           .addBooleanField("bool")
           .addInt32Field("int")
           .addRowField("row", NESTED_ROW_SCHEMA)
-          .addArrayField("arr_long", org.apache.beam.sdk.schemas.Schema.FieldType.INT64)
+          .addArrayField("arr_long", FieldType.INT64)
           .addNullableRowField("nullable_row", NESTED_ROW_SCHEMA)
           .addNullableInt64Field("nullable_long")
+          .addDateTimeField("datetime_tz")
+          .addLogicalTypeField("datetime", SqlTypes.DATETIME)
+          .addLogicalTypeField("date", SqlTypes.DATE)
+          .addLogicalTypeField("time", SqlTypes.TIME)
           .build();
 
   private static final SimpleFunction<Long, Row> ROW_FUNC =
@@ -139,6 +148,10 @@ public class IcebergIOIT implements Serializable {
               .addValue(LongStream.range(0, num % 10).boxed().collect(Collectors.toList()))
               .addValue(num % 2 == 0 ? null : nestedRow)
               .addValue(num)
+              .addValue(new DateTime(num).withZone(DateTimeZone.forOffsetHoursMinutes(3, 25)))
+              .addValue(DateTimeUtil.timestampFromMicros(num))
+              .addValue(DateTimeUtil.dateFromDays(Integer.parseInt(strNum)))
+              .addValue(DateTimeUtil.timeFromMicros(num))
               .build();
         }
       };
diff --git a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
index a20d5b7c8f5..134f05c34bf 100644
--- a/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
+++ b/sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergUtilsTest.java
@@ -28,16 +28,21 @@ import static org.junit.Assert.assertTrue;
 
 import java.math.BigDecimal;
 import java.nio.ByteBuffer;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.data.GenericRecord;
 import org.apache.iceberg.data.Record;
 import org.apache.iceberg.types.Type;
 import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Test;
@@ -45,6 +50,7 @@ import org.junit.experimental.runners.Enclosed;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
+/** Test class for {@link IcebergUtils}. */
 @RunWith(Enclosed.class)
 public class IcebergUtilsTest {
 
@@ -102,21 +108,87 @@ public class IcebergUtilsTest {
     }
 
     @Test
-    public void testDate() {}
+    public void testDate() {
+      checkRowValueToRecordValue(
+          Schema.FieldType.logicalType(SqlTypes.DATE),
+          Types.DateType.get(),
+          DateTimeUtil.dateFromDays(12345));
+    }
 
     @Test
-    public void testTime() {}
+    public void testTime() {
+      checkRowValueToRecordValue(
+          Schema.FieldType.logicalType(SqlTypes.TIME),
+          Types.TimeType.get(),
+          DateTimeUtil.timeFromMicros(12345678L));
+    }
 
     @Test
     public void testTimestamp() {
+      // SqlTypes.DATETIME
+      checkRowValueToRecordValue(
+          Schema.FieldType.logicalType(SqlTypes.DATETIME),
+          Types.TimestampType.withoutZone(),
+          DateTimeUtil.timestampFromMicros(123456789L));
+
+      // Schema.FieldType.DATETIME
       DateTime dateTime =
           new DateTime().withDate(1979, 03, 14).withTime(1, 2, 3, 4).withZone(DateTimeZone.UTC);
-
       checkRowValueToRecordValue(
           Schema.FieldType.DATETIME,
-          dateTime.toInstant(),
+          dateTime,
+          Types.TimestampType.withoutZone(),
+          DateTimeUtil.timestampFromMicros(dateTime.getMillis() * 1000L));
+
+      // Schema.FieldType.INT64
+      long micros = 1234567890L;
+      checkRowValueToRecordValue(
+          Schema.FieldType.INT64,
+          micros,
           Types.TimestampType.withoutZone(),
-          dateTime.getMillis());
+          DateTimeUtil.timestampFromMicros(micros));
+
+      // Schema.FieldType.STRING
+      String val = "2024-10-08T13:18:20.053";
+      LocalDateTime localDateTime = LocalDateTime.of(2024, 10, 8, 13, 18, 20, 53_000_000);
+      checkRowValueToRecordValue(
+          Schema.FieldType.STRING, val, Types.TimestampType.withoutZone(), localDateTime);
+    }
+
+    @Test
+    public void testTimestampWithZone() {
+      String val = "2024-10-08T13:18:20.053+03:27";
+      DateTime dateTime = DateTime.parse(val);
+      OffsetDateTime offsetDateTime = OffsetDateTime.parse(val);
+      LocalDateTime localDateTime =
+          offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC).toLocalDateTime();
+      // SqlTypes.DATETIME
+      checkRowValueToRecordValue(
+          Schema.FieldType.logicalType(SqlTypes.DATETIME),
+          localDateTime,
+          Types.TimestampType.withZone(),
+          offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC));
+
+      // Schema.FieldType.DATETIME
+      checkRowValueToRecordValue(
+          Schema.FieldType.DATETIME,
+          dateTime,
+          Types.TimestampType.withZone(),
+          offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC));
+
+      // Schema.FieldType.INT64
+      checkRowValueToRecordValue(
+          Schema.FieldType.INT64,
+          DateTimeUtil.microsFromTimestamptz(offsetDateTime),
+          Types.TimestampType.withZone(),
+          offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC));
+
+      // Schema.FieldType.STRING
+      checkRowValueToRecordValue(
+          Schema.FieldType.STRING,
+          val,
+          Types.TimestampType.withZone(),
+          offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC));
     }
 
     @Test
@@ -190,7 +262,7 @@ public class IcebergUtilsTest {
 
       Row row = IcebergUtils.icebergRecordToBeamRow(beamSchema, record);
 
-      assertThat(row.getBaseValue("v"), equalTo(destValue));
+      assertThat(row.getValue("v"), equalTo(destValue));
     }
 
     @Test
@@ -224,21 +296,75 @@ public class IcebergUtilsTest {
     }
 
     @Test
-    public void testDate() {}
+    public void testDate() {
+      checkRecordValueToRowValue(
+          Types.DateType.get(),
+          Schema.FieldType.logicalType(SqlTypes.DATE),
+          DateTimeUtil.dateFromDays(12345));
+    }
 
     @Test
-    public void testTime() {}
+    public void testTime() {
+      checkRecordValueToRowValue(
+          Types.TimeType.get(),
+          Schema.FieldType.logicalType(SqlTypes.TIME),
+          DateTimeUtil.timeFromMicros(1234567L));
+    }
 
     @Test
     public void testTimestamp() {
+      // SqlTypes.DATETIME
+      checkRecordValueToRowValue(
+          Types.TimestampType.withoutZone(),
+          Schema.FieldType.logicalType(SqlTypes.DATETIME),
+          DateTimeUtil.timestampFromMicros(123456789L));
+
+      // Schema.FieldType.DATETIME
       DateTime dateTime =
           new DateTime().withDate(1979, 03, 14).withTime(1, 2, 3, 4).withZone(DateTimeZone.UTC);
-
       checkRecordValueToRowValue(
           Types.TimestampType.withoutZone(),
-          dateTime.getMillis(),
+          dateTime.getMillis() * 1000L,
+          Schema.FieldType.DATETIME,
+          dateTime);
+    }
+
+    @Test
+    public void testTimestampWithZone() {
+      String timestamp = "2024-10-08T13:18:20.053+03:27";
+      OffsetDateTime offsetDateTime = OffsetDateTime.parse(timestamp);
+      LocalDateTime localDateTime =
+          offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC).toLocalDateTime();
+      // SqlTypes.DATETIME
+      checkRecordValueToRowValue(
+          Types.TimestampType.withZone(),
+          offsetDateTime,
+          Schema.FieldType.logicalType(SqlTypes.DATETIME),
+          localDateTime);
+      checkRecordValueToRowValue(
+          Types.TimestampType.withZone(),
+          localDateTime,
+          Schema.FieldType.logicalType(SqlTypes.DATETIME),
+          localDateTime);
+      checkRecordValueToRowValue(
+          Types.TimestampType.withZone(),
+          DateTimeUtil.microsFromTimestamptz(offsetDateTime),
+          Schema.FieldType.logicalType(SqlTypes.DATETIME),
+          localDateTime);
+
+      // Schema.FieldType.DATETIME
+      DateTime dateTime = DateTime.parse(timestamp).withZone(DateTimeZone.UTC);
+      checkRecordValueToRowValue(
+          Types.TimestampType.withZone(), offsetDateTime, Schema.FieldType.DATETIME, dateTime);
+      checkRecordValueToRowValue(
+          Types.TimestampType.withZone(), localDateTime, Schema.FieldType.DATETIME, dateTime);
+      checkRecordValueToRowValue(
+          Types.TimestampType.withZone(),
+          DateTimeUtil.microsFromTimestamptz(offsetDateTime),
           Schema.FieldType.DATETIME,
-          dateTime.toInstant());
+          dateTime);
+      checkRecordValueToRowValue(
+          Types.TimestampType.withZone(), timestamp, Schema.FieldType.DATETIME, dateTime);
     }
 
     @Test
@@ -425,7 +551,7 @@ public class IcebergUtilsTest {
               new BeamFieldTypeTestCase(
                   1,
                   Schema.FieldType.row(BEAM_SCHEMA_PRIMITIVE),
-                  7,
+                  11,
                   Types.StructType.of(ICEBERG_SCHEMA_PRIMITIVE.columns())),
               new BeamFieldTypeTestCase(
                   15,
@@ -537,6 +663,10 @@ public class IcebergUtilsTest {
             .addNullableStringField("str")
             .addNullableBooleanField("bool")
             .addByteArrayField("bytes")
+            .addDateTimeField("datetime_tz")
+            .addLogicalTypeField("datetime", SqlTypes.DATETIME)
+            .addLogicalTypeField("time", SqlTypes.TIME)
+            .addLogicalTypeField("date", SqlTypes.DATE)
             .build();
 
     static final org.apache.iceberg.Schema ICEBERG_SCHEMA_PRIMITIVE =
@@ -547,16 +677,17 @@ public class IcebergUtilsTest {
             required(4, "long", Types.LongType.get()),
             optional(5, "str", Types.StringType.get()),
             optional(6, "bool", Types.BooleanType.get()),
-            required(7, "bytes", Types.BinaryType.get()));
+            required(7, "bytes", Types.BinaryType.get()),
+            required(8, "datetime_tz", Types.TimestampType.withZone()),
+            required(9, "datetime", Types.TimestampType.withoutZone()),
+            required(10, "time", Types.TimeType.get()),
+            required(11, "date", Types.DateType.get()));
 
     @Test
     public void testPrimitiveBeamSchemaToIcebergSchema() {
       org.apache.iceberg.Schema convertedIcebergSchema =
           IcebergUtils.beamSchemaToIcebergSchema(BEAM_SCHEMA_PRIMITIVE);
 
-      System.out.println(convertedIcebergSchema);
-      System.out.println(ICEBERG_SCHEMA_PRIMITIVE);
-
       assertTrue(convertedIcebergSchema.sameSchema(ICEBERG_SCHEMA_PRIMITIVE));
     }
 
@@ -591,9 +722,6 @@ public class IcebergUtilsTest {
     public void testArrayIcebergSchemaToBeamSchema() {
       Schema convertedBeamSchema = IcebergUtils.icebergSchemaToBeamSchema(ICEBERG_SCHEMA_LIST);
 
-      System.out.println(convertedBeamSchema);
-      System.out.println(BEAM_SCHEMA_LIST);
-
       assertEquals(BEAM_SCHEMA_LIST, convertedBeamSchema);
     }