You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/09/09 10:41:47 UTC

(datafusion-comet) branch main updated: fix: match iceberg-java's exception for unclustered input to a clustered Iceberg write (#5779)

This is an automated email from the ASF dual-hosted git repository.

andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/main by this push:
     new 79d1be1eb3 fix: match iceberg-java's exception for unclustered input to a clustered Iceberg write (#5779)
79d1be1eb3 is described below

commit 79d1be1eb3cdb147d163231f54682970e02cbb6b
Author: Andy Grove <ag...@apache.org>
AuthorDate: Wed Sep 9 04:41:42 2026 -0600

    fix: match iceberg-java's exception for unclustered input to a clustered Iceberg write (#5779)
---
 .../execution/operators/iceberg_partition_path.rs  |  55 ++--
 .../core/src/execution/operators/iceberg_write.rs  | 296 ++++++++++++++++++++-
 native/jni-bridge/src/errors.rs                    |  87 ++++--
 .../org/apache/comet/CometIcebergTestBase.scala    |  10 +-
 .../comet/CometIcebergWriteActionSuite.scala       | 133 +++++++++
 5 files changed, 522 insertions(+), 59 deletions(-)

diff --git a/native/core/src/execution/operators/iceberg_partition_path.rs b/native/core/src/execution/operators/iceberg_partition_path.rs
index a2cc5a95a3..363a9f2f19 100644
--- a/native/core/src/execution/operators/iceberg_partition_path.rs
+++ b/native/core/src/execution/operators/iceberg_partition_path.rs
@@ -73,33 +73,42 @@ impl CometLocationGenerator {
         })
     }
 
-    /// Mirrors iceberg-java's `PartitionSpec#partitionToPath`: `name=value` pairs joined by `/`,
-    /// with both halves form-urlencoded. `form_urlencoded` leaves exactly the byte set Java's
-    /// `URLEncoder.encode(s, UTF_8)` leaves (`A-Za-z0-9`, `*`, `-`, `.`, `_`), maps space to `+`,
-    /// and percent-encodes the rest with uppercase hex, so the two agree byte for byte.
     fn partition_to_path(&self, key: &PartitionKey) -> String {
-        let fields = self.partition_type.fields();
-        key.spec()
-            .fields()
-            .iter()
-            .enumerate()
-            .map(|(index, field)| {
-                // Indexed rather than zipped so a spec/partition-type/value length disagreement
-                // renders as "null" instead of panicking; the three are built from the same spec,
-                // so they always line up in practice.
-                let value = key.data().fields().get(index).and_then(Option::as_ref);
-                let human = match fields.get(index) {
-                    Some(nested) => human_string(&field.transform, &nested.field_type, value),
-                    None => NULL.to_string(),
-                };
-                form_urlencoded::Serializer::new(String::new())
-                    .append_pair(&field.name, &human)
-                    .finish()
-            })
-            .join("/")
+        partition_to_path(&self.partition_type, key)
     }
 }
 
+/// Mirrors iceberg-java's `PartitionSpec#partitionToPath`: `name=value` pairs joined by `/`, with
+/// both halves form-urlencoded. `form_urlencoded` leaves exactly the byte set Java's
+/// `URLEncoder.encode(s, UTF_8)` leaves (`A-Za-z0-9`, `*`, `-`, `.`, `_`), maps space to `+`, and
+/// percent-encodes the rest with uppercase hex, so the two agree byte for byte.
+///
+/// `partition_type` is `key`'s spec resolved against its schema. It is passed in rather than
+/// resolved here because resolution can fail and every caller already holds one: the location
+/// generator resolves it at task start, and `ClusteredBatchSplitter` needs it to read partition
+/// values.
+pub(crate) fn partition_to_path(partition_type: &StructType, key: &PartitionKey) -> String {
+    let fields = partition_type.fields();
+    key.spec()
+        .fields()
+        .iter()
+        .enumerate()
+        .map(|(index, field)| {
+            // Indexed rather than zipped so a spec/partition-type/value length disagreement
+            // renders as "null" instead of panicking; the three are built from the same spec,
+            // so they always line up in practice.
+            let value = key.data().fields().get(index).and_then(Option::as_ref);
+            let human = match fields.get(index) {
+                Some(nested) => human_string(&field.transform, &nested.field_type, value),
+                None => NULL.to_string(),
+            };
+            form_urlencoded::Serializer::new(String::new())
+                .append_pair(&field.name, &human)
+                .finish()
+        })
+        .join("/")
+}
+
 impl LocationGenerator for CometLocationGenerator {
     fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String {
         // `is_effectively_none` is iceberg-rust's own predicate: no key, or a key whose spec is
diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs
index 1fefcf7dd4..297d9a0c74 100644
--- a/native/core/src/execution/operators/iceberg_write.rs
+++ b/native/core/src/execution/operators/iceberg_write.rs
@@ -58,6 +58,7 @@ use iceberg::writer::file_writer::ParquetWriterBuilder;
 use iceberg::writer::partitioning::clustered_writer::ClusteredWriter;
 use iceberg::writer::partitioning::fanout_writer::FanoutWriter;
 use iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter;
+use iceberg::ErrorKind;
 #[cfg(test)]
 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
 use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel};
@@ -69,8 +70,11 @@ use datafusion_comet_proto::spark_operator::{
 };
 
 use crate::cloud::s3::credential_bridge::AccessMode;
+use crate::errors::CometError;
 use crate::execution::operators::iceberg_common::load_file_io;
-use crate::execution::operators::iceberg_partition_path::CometLocationGenerator;
+use crate::execution::operators::iceberg_partition_path::{
+    partition_to_path, CometLocationGenerator,
+};
 
 /// Builder chain instantiated once per task and handed to the partitioning wrapper.
 type IcebergDataFileWriterBuilder =
@@ -417,11 +421,16 @@ impl InnerWriter {
                 Ok(())
             }
             InnerWriter::Clustered(w) => {
-                let parts = clustered_splitter
-                    .expect("clustered splitter must be Some for clustered writes")
-                    .split(&batch)?;
-                for (key, part) in parts {
-                    w.write(key, part).await.map_err(iceberg_err)?;
+                let splitter = clustered_splitter
+                    .expect("clustered splitter must be Some for clustered writes");
+                for (key, part) in splitter.split(&batch)? {
+                    // `write` consumes the key, but the unclustered-input error needs it for the
+                    // partition path. One extra spec clone per run on top of the one
+                    // `ClusteredBatchSplitter::partition_key` already pays.
+                    let key_for_error = key.clone();
+                    w.write(key, part)
+                        .await
+                        .map_err(|e| clustered_write_err(e, &key_for_error, splitter))?;
                 }
                 Ok(())
             }
@@ -458,6 +467,76 @@ fn iceberg_err(e: iceberg::Error) -> DataFusionError {
     DataFusionError::External(Box::new(e))
 }
 
+/// iceberg-java's `ClusteredWriter` preamble, copied verbatim from
+/// `core/src/main/java/org/apache/iceberg/io/ClusteredWriter.java`. Applications catch that
+/// writer's `IllegalStateException` and match on this text -- Iceberg's own
+/// `TestRequiredDistributionAndOrdering` does both -- so the native writer reproduces it instead
+/// of surfacing iceberg-rust's differently worded error.
+const NOT_CLUSTERED_ROWS_ERROR_MSG_TEMPLATE: &str = "Incoming records violate the writer \
+     assumption that records are clustered by spec and by partition within each spec. Either \
+     cluster the incoming records or switch to fanout writers.\n\
+     Encountered records that belong to already closed files:\n";
+
+/// The message iceberg-rust's `ClusteredWriter` raises for the same condition.
+const UNSORTED_INPUT_MESSAGE_PREFIX: &str = "The input is not sorted!";
+
+/// Restates iceberg-rust's unclustered-input failure as iceberg-java's, passing every other
+/// failure through unchanged.
+///
+/// Leaving `ClusteredWriter` as the sole judge of whether the input is clustered means coupling to
+/// its message text, which is the cheaper of the two couplings available: re-deriving the
+/// condition here would need a copy of the writer's closed-partition bookkeeping, and because that
+/// copy would fire first, no test could catch it drifting from the original. A wording change
+/// upstream instead makes `clustered_write_rejects_unclustered_input_like_iceberg_java` fail, since
+/// that test drives the real writer.
+fn clustered_write_err(
+    e: iceberg::Error,
+    key: &PartitionKey,
+    splitter: &ClusteredBatchSplitter,
+) -> DataFusionError {
+    if e.kind() == ErrorKind::Unexpected && e.message().starts_with(UNSORTED_INPUT_MESSAGE_PREFIX) {
+        not_clustered_error(key, &splitter.partition_type)
+    } else {
+        iceberg_err(e)
+    }
+}
+
+/// The error iceberg-java's `ClusteredWriter.write` raises when a closed partition is revisited,
+/// down to the `partition '<path>' in spec <spec>` context. Only the partition branch of that
+/// check is reachable here: a task writes through a single output spec, so the spec can never
+/// change mid-stream.
+///
+/// The path comes from the same Java-compatible renderer that names the data directories, not from
+/// `PartitionKey::to_path`: iceberg-rust hex-encodes binary where iceberg-java base64-encodes it,
+/// and panics outright on a pre-epoch `timestamptz` with a sub-second part.
+fn not_clustered_error(key: &PartitionKey, partition_type: &StructType) -> DataFusionError {
+    DataFusionError::from(CometError::IllegalState(format!(
+        "{NOT_CLUSTERED_ROWS_ERROR_MSG_TEMPLATE}partition '{}' in spec {}",
+        partition_to_path(partition_type, key),
+        format_partition_spec(key.spec())
+    )))
+}
+
+/// Renders a partition spec the way iceberg-java's `PartitionSpec.toString` and
+/// `PartitionField.toString` do, so the error context matches character for character.
+fn format_partition_spec(spec: &PartitionSpec) -> String {
+    let fields: String = spec
+        .fields()
+        .iter()
+        .map(|field| {
+            format!(
+                "\n  {}: {}: {}({})",
+                field.field_id, field.name, field.transform, field.source_id
+            )
+        })
+        .collect();
+    if fields.is_empty() {
+        "[]".to_string()
+    } else {
+        format!("[{fields}\n]")
+    }
+}
+
 fn build_output_schema() -> SchemaRef {
     Arc::new(ArrowSchema::new(vec![Field::new(
         "iceberg_manifest",
@@ -880,7 +959,8 @@ mod tests {
 
     mod integration {
         use super::super::*;
-        use arrow::array::{Int32Array, StringArray};
+        use arrow::array::{BinaryArray, Int32Array, StringArray, TimestampMicrosecondArray};
+        use arrow::datatypes::TimeUnit;
         use datafusion::common::Result as DFResult;
         use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
         use datafusion_comet_proto::spark_operator::{
@@ -925,8 +1005,13 @@ mod tests {
             .unwrap()
         }
 
+        /// Streams `batches` under their own schema, so a test can use a column layout other than
+        /// [`user_schema`].
         fn input_stream(batches: Vec<RecordBatch>) -> SendableRecordBatchStream {
-            let schema = user_schema();
+            let schema = batches
+                .first()
+                .map(RecordBatch::schema)
+                .unwrap_or_else(user_schema);
             Box::pin(RecordBatchStreamAdapter::new(
                 schema,
                 futures::stream::iter(batches.into_iter().map(Ok::<_, DataFusionError>)),
@@ -1125,6 +1210,201 @@ mod tests {
             assert_eq!(total, 5);
         }
 
+        /// Runs a clustered write over input that revisits a closed partition, and returns the
+        /// `CometError::IllegalState` message it has to fail with.
+        ///
+        /// Driving the real iceberg-rust writer is what makes these tests the tripwire for
+        /// [`UNSORTED_INPUT_MESSAGE_PREFIX`] going stale on an iceberg-rust bump: the translation
+        /// stops matching and the error arrives as a plain `iceberg::Error` instead.
+        async fn unclustered_write_message(
+            schema: Schema,
+            spec: PartitionSpec,
+            batches: Vec<RecordBatch>,
+        ) -> String {
+            let temp_dir = TempDir::new().unwrap();
+            let common = common(
+                format!("file://{}", temp_dir.path().display()),
+                serde_json::to_string(&spec).unwrap(),
+                serde_json::to_string(&schema).unwrap(),
+                ProtoIcebergWriterMode::IcebergWriterClustered,
+            );
+            let err = run(
+                common,
+                schema,
+                spec,
+                ProtoIcebergWriterMode::IcebergWriterClustered,
+                batches,
+            )
+            .await
+            .unwrap_err();
+            let comet_error = match &err {
+                DataFusionError::External(external) => external.downcast_ref::<CometError>(),
+                _ => None,
+            };
+            match comet_error {
+                Some(CometError::IllegalState(message)) => message.clone(),
+                _ => panic!("expected CometError::IllegalState, got {err:?}"),
+            }
+        }
+
+        /// Builds an identity-partitioned single-column spec over `schema`.
+        fn identity_spec(schema: &Schema, column: &str) -> PartitionSpec {
+            PartitionSpec::builder(Arc::new(schema.clone()))
+                .with_spec_id(1)
+                .add_partition_field(column, column, Transform::Identity)
+                .unwrap()
+                .build()
+                .unwrap()
+        }
+
+        /// Unclustered input must be rejected with iceberg-java's `ClusteredWriter` error rather
+        /// than iceberg-rust's "The input is not sorted!", and it has to reach the JVM as an
+        /// `IllegalStateException`. Iceberg's own `TestRequiredDistributionAndOrdering` asserts
+        /// both. See https://github.com/apache/datafusion-comet/issues/5698.
+        #[tokio::test]
+        async fn clustered_write_rejects_unclustered_input_like_iceberg_java() {
+            let schema = iceberg_user_schema();
+            let spec = identity_spec(&schema, "region");
+            // "eu" is revisited after the writer has moved on to "us" and closed it.
+            let message = unclustered_write_message(
+                schema,
+                spec,
+                vec![batch(&[1, 2], &["eu", "us"]), batch(&[3], &["eu"])],
+            )
+            .await;
+
+            // The wording itself is pinned against the real JVM writer by
+            // CometIcebergWriteActionSuite; here it only has to carry the right context.
+            assert_eq!(
+                message,
+                format!(
+                    "{NOT_CLUSTERED_ROWS_ERROR_MSG_TEMPLATE}\
+                     partition 'region=eu' in spec [\n  1000: region: identity(2)\n]"
+                )
+            );
+        }
+
+        /// iceberg-java base64-encodes a binary partition value where iceberg-rust's
+        /// `PartitionKey::to_path` writes uppercase hex, so the error has to go through the same
+        /// Java-compatible renderer that names the data directories.
+        #[tokio::test]
+        async fn unclustered_binary_partition_renders_like_iceberg_java() {
+            let schema = Schema::builder()
+                .with_schema_id(1)
+                .with_fields(vec![
+                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
+                    NestedField::required(2, "part", Type::Primitive(PrimitiveType::Binary)).into(),
+                ])
+                .build()
+                .unwrap();
+            let spec = identity_spec(&schema, "part");
+            let arrow_schema = Arc::new(ArrowSchema::new(vec![
+                Field::new("id", DataType::Int32, false),
+                Field::new("part", DataType::Binary, false),
+            ]));
+            let batch = |ids: &[i32], parts: Vec<&[u8]>| {
+                RecordBatch::try_new(
+                    Arc::clone(&arrow_schema),
+                    vec![
+                        Arc::new(Int32Array::from(ids.to_vec())),
+                        Arc::new(BinaryArray::from(parts)),
+                    ],
+                )
+                .unwrap()
+            };
+
+            let message = unclustered_write_message(
+                schema,
+                spec,
+                vec![
+                    batch(&[1, 2], vec![&[0x00, 0x01, 0xFF], &[0x00]]),
+                    batch(&[3], vec![&[0x00, 0x01, 0xFF]]),
+                ],
+            )
+            .await;
+
+            // base64(00 01 FF) = "AAH/", form-urlencoded to "AAH%2F". iceberg-rust would have
+            // rendered the same bytes as "0001FF".
+            assert!(
+                message
+                    .ends_with("partition 'part=AAH%2F' in spec [\n  1000: part: identity(2)\n]"),
+                "{message}"
+            );
+        }
+
+        /// iceberg-rust's `Transform::to_human_string` panics on a pre-epoch `timestamptz` with a
+        /// sub-second part, so routing the error through `PartitionKey::to_path` would crash the
+        /// task instead of raising `IllegalStateException`.
+        #[tokio::test]
+        async fn unclustered_negative_timestamptz_partition_renders_like_iceberg_java() {
+            let schema = Schema::builder()
+                .with_schema_id(1)
+                .with_fields(vec![
+                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
+                    NestedField::required(2, "ts", Type::Primitive(PrimitiveType::Timestamptz))
+                        .into(),
+                ])
+                .build()
+                .unwrap();
+            let spec = identity_spec(&schema, "ts");
+            let arrow_schema = Arc::new(ArrowSchema::new(vec![
+                Field::new("id", DataType::Int32, false),
+                Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
+                    false,
+                ),
+            ]));
+            let batch = |ids: &[i32], micros: &[i64]| {
+                RecordBatch::try_new(
+                    Arc::clone(&arrow_schema),
+                    vec![
+                        Arc::new(Int32Array::from(ids.to_vec())),
+                        Arc::new(
+                            TimestampMicrosecondArray::from(micros.to_vec())
+                                .with_timezone("+00:00"),
+                        ),
+                    ],
+                )
+                .unwrap()
+            };
+
+            // -1_500_000 micros is 1969-12-31T23:59:58.5Z -- negative with a sub-second part.
+            let message = unclustered_write_message(
+                schema,
+                spec,
+                vec![batch(&[1, 2], &[-1_500_000, 0]), batch(&[3], &[-1_500_000])],
+            )
+            .await;
+
+            assert!(
+                message.ends_with(
+                    "partition 'ts=1969-12-31T23%3A59%3A58.5%2B00%3A00' in spec \
+                     [\n  1000: ts: identity(2)\n]"
+                ),
+                "{message}"
+            );
+        }
+
+        /// The transform rendering in [`format_partition_spec`] has to match iceberg-java's
+        /// `PartitionField.toString` for the parameterised transforms too.
+        #[test]
+        fn partition_spec_renders_like_iceberg_java() {
+            let schema = iceberg_user_schema();
+            let spec = PartitionSpec::builder(Arc::new(schema))
+                .with_spec_id(7)
+                .add_partition_field("id", "id_bucket", Transform::Bucket(8))
+                .unwrap()
+                .add_partition_field("region", "region_trunc", Transform::Truncate(4))
+                .unwrap()
+                .build()
+                .unwrap();
+            assert_eq!(
+                format_partition_spec(&spec),
+                "[\n  1000: id_bucket: bucket[8](1)\n  1001: region_trunc: truncate[4](2)\n]"
+            );
+        }
+
         #[tokio::test]
         async fn encoded_manifest_round_trips_through_iceberg_parser() {
             let temp_dir = TempDir::new().unwrap();
diff --git a/native/jni-bridge/src/errors.rs b/native/jni-bridge/src/errors.rs
index a072be95ed..3e72f6e804 100644
--- a/native/jni-bridge/src/errors.rs
+++ b/native/jni-bridge/src/errors.rs
@@ -95,6 +95,12 @@ pub enum CometError {
     #[error("{0}")]
     ShuffleSizeLimit(String),
 
+    /// Must reach the JVM as `java.lang.IllegalStateException` carrying this exact message: for
+    /// runtime checks where Comet reimplements a JVM library's guard and callers match on the
+    /// exception that library raises. Internal invariants belong in [`CometError::Internal`].
+    #[error("{0}")]
+    IllegalState(String),
+
     #[error(transparent)]
     Arrow {
         #[from]
@@ -219,9 +225,10 @@ impl From<CometError> for DataFusionError {
             // own codegen inside the JVM UDF kernel) as an `External` error so it survives the trip
             // back through DataFusion and can be re-thrown with its exact type at the JNI boundary.
             // Flattening it to a string here would surface it as a generic CometNativeException.
-            value @ (CometError::JavaException { .. } | CometError::ShuffleSizeLimit(_)) => {
-                DataFusionError::External(Box::new(value))
-            }
+            // The same applies to the classifications that must keep their JVM exception class.
+            value @ (CometError::JavaException { .. }
+            | CometError::ShuffleSizeLimit(_)
+            | CometError::IllegalState(_)) => DataFusionError::External(Box::new(value)),
             _ => DataFusionError::Execution(value.to_string()),
         }
     }
@@ -349,6 +356,10 @@ impl jni::errors::ToException for CometError {
                 class: "org/apache/comet/CometShuffleSizeLimitException".to_string(),
                 msg: message.clone(),
             },
+            CometError::IllegalState(message) => Exception {
+                class: "java/lang/IllegalStateException".to_string(),
+                msg: message.clone(),
+            },
             _other => Exception {
                 class: "org/apache/comet/CometNativeException".to_string(),
                 msg: self.to_string(),
@@ -485,11 +496,12 @@ fn throw_exception(env: &mut Env, error: &CometError, backtrace: Option<String>)
     // If there isn't already an exception?
     if !env.exception_check() {
         // DataFusion operators can wrap the original failure in Context, Shared, or External
-        // errors. Keep capacity failures typed across those wrappers and the JNI boundary.
-        if let Some(message) = shuffle_size_limit_message(error) {
+        // errors. Keep the classifications that own their JVM exception class typed across those
+        // wrappers and the JNI boundary.
+        if let Some(exception) = typed_jvm_exception(error) {
             let _ = env.throw_new(
-                jni::jni_str!("org/apache/comet/CometShuffleSizeLimitException"),
-                JNIString::new(message),
+                JNIString::new(exception.class),
+                JNIString::new(exception.msg),
             );
             return;
         }
@@ -574,11 +586,17 @@ fn throw_exception(env: &mut Env, error: &CometError, backtrace: Option<String>)
     }
 }
 
-fn shuffle_size_limit_message<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a str> {
+/// Walks the cause chain for a `CometError` variant that owns its JVM exception class, so class and
+/// message survive DataFusion's `Context` / `Shared` / `External` wrappers. The class comes from
+/// `to_exception`, keeping one variant-to-class table. Classification is by variant only -- an
+/// error that merely mentions the same words keeps its normal classification.
+fn typed_jvm_exception(error: &(dyn std::error::Error + 'static)) -> Option<Exception> {
     let mut cause = Some(error);
     while let Some(error) = cause {
-        if let Some(CometError::ShuffleSizeLimit(message)) = error.downcast_ref::<CometError>() {
-            return Some(message);
+        if let Some(typed @ (CometError::ShuffleSizeLimit(_) | CometError::IllegalState(_))) =
+            error.downcast_ref::<CometError>()
+        {
+            return Some(typed.to_exception());
         }
         cause = error.source();
     }
@@ -943,34 +961,53 @@ mod tests {
         }
     }
 
-    #[test]
-    #[cfg_attr(miri, ignore)] // miri cannot create a JVM.
-    fn shuffle_size_limit_survives_datafusion_wrappers_and_jni() {
-        let message = "Remote shuffle exceeds spark.comet.shuffle.rss.maxInFlightBytes";
-        let error = DataFusionError::from(CometError::ShuffleSizeLimit(message.to_string()));
-        let error = DataFusionError::Shared(Arc::new(DataFusionError::Context(
-            "executing shuffle writer".to_string(),
-            Box::new(error),
+    /// Wraps `error` the way a DataFusion operator would, throws it, and asserts the pending Java
+    /// exception is `expected_class` with the message intact -- i.e. that the variant kept its own
+    /// JVM exception class instead of collapsing into `CometNativeException`.
+    fn assert_survives_datafusion_wrappers(error: CometError, message: &str, expected_class: &str) {
+        let wrapped = DataFusionError::Shared(Arc::new(DataFusionError::Context(
+            "executing operator".to_string(),
+            Box::new(DataFusionError::from(error)),
         )));
         jvm()
             .attach_current_thread(|env| -> jni::errors::Result<()> {
-                unwrap_or_throw_default::<()>(env, Err(CometError::from(error)));
-                assert_pending_java_exception_detailed(
-                    env,
-                    Some("org/apache/comet/CometShuffleSizeLimitException"),
-                    Some(message),
-                );
+                unwrap_or_throw_default::<()>(env, Err(CometError::from(wrapped)));
+                assert_pending_java_exception_detailed(env, Some(expected_class), Some(message));
                 Ok(())
             })
             .unwrap();
     }
 
+    #[test]
+    #[cfg_attr(miri, ignore)] // miri cannot create a JVM.
+    fn shuffle_size_limit_survives_datafusion_wrappers_and_jni() {
+        let message = "Remote shuffle exceeds spark.comet.shuffle.rss.maxInFlightBytes";
+        assert_survives_datafusion_wrappers(
+            CometError::ShuffleSizeLimit(message.to_string()),
+            message,
+            "org/apache/comet/CometShuffleSizeLimitException",
+        );
+    }
+
     #[test]
     fn shuffle_size_limit_is_not_inferred_from_error_text() {
         let error = CometError::from(DataFusionError::Execution(
             "Remote shuffle exceeds spark.comet.shuffle.rss.maxInFlightBytes".to_string(),
         ));
-        assert!(shuffle_size_limit_message(&error).is_none());
+        assert!(typed_jvm_exception(&error).is_none());
+    }
+
+    /// The native Iceberg writer rejects unclustered input with iceberg-java's own
+    /// `IllegalStateException`, so it has to survive the same wrappers.
+    #[test]
+    #[cfg_attr(miri, ignore)] // miri cannot create a JVM.
+    fn illegal_state_survives_datafusion_wrappers_and_jni() {
+        let message = "Incoming records violate the writer assumption that records are clustered";
+        assert_survives_datafusion_wrappers(
+            CometError::IllegalState(message.to_string()),
+            message,
+            "java/lang/IllegalStateException",
+        );
     }
 
     #[test]
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala b/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala
index f6e21f3901..03f8fe28f6 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala
@@ -137,15 +137,19 @@ trait CometIcebergTestBase {
     file.delete()
   }
 
-  /** The executed plan of every query that completes successfully while `action` runs. */
-  protected def capturePlans(spark: SparkSession)(action: => Unit): Seq[SparkPlan] = {
+  /**
+   * The executed plan of every query that ran while `action` ran. Queries that failed are
+   * included only when `includeFailures` is set, which is what an action expected to abort needs.
+   */
+  protected def capturePlans(spark: SparkSession, includeFailures: Boolean = false)(
+      action: => Unit): Seq[SparkPlan] = {
     val captured = mutable.Buffer.empty[SparkPlan]
     val listener = new QueryExecutionListener {
       override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
         captured += qe.executedPlan
       }
       override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit =
-        ()
+        if (includeFailures) captured += qe.executedPlan
     }
     spark.listenerManager.register(listener)
     try {
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala
index 9b0d8f412c..662d7cb359 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala
@@ -20,6 +20,7 @@
 package org.apache.comet
 
 import java.io.File
+import java.sql.Timestamp
 import java.util.concurrent.{CountDownLatch, TimeUnit}
 
 import scala.collection.mutable
@@ -29,6 +30,7 @@ import scala.concurrent.duration.DurationInt
 
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.CometTestBase
+import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.comet.{CometIcebergWriteExec, IcebergCommitExec, IcebergWriteExec}
 import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog
@@ -1222,6 +1224,110 @@ class CometIcebergWriteActionSuite
     }
   }
 
+  // https://github.com/apache/datafusion-comet/issues/5698. iceberg-java's `ClusteredWriter`
+  // rejects unclustered input with a documented `IllegalStateException`, and callers match on both
+  // the type and the message -- Iceberg's own `TestRequiredDistributionAndOrdering` does exactly
+  // that. Pin the native writer against the JVM writer on the same runtime rather than against a
+  // literal, so an iceberg-java wording change surfaces here as a parity failure.
+  test("native acceleration: clustered writer rejects unclustered input like the JVM writer") {
+    assumeNativeAcceleration()
+    withIcebergCatalog { warehouseDir =>
+      val (nativeTable, jvmTable) = ("unclustered_native", "unclustered_jvm")
+      val tables = Seq(nativeTable, jvmTable)
+      tables.foreach(createTable(warehouseDir, _, partitionSpec = "PARTITIONED BY (region)"))
+      val session = spark
+      import session.implicits._
+      // "us-east" is revisited after the writer has moved on to "eu" and closed it.
+      val rows =
+        Seq((1, "us-east", 1.0), (2, "eu", 2.0), (3, "us-east", 3.0)).toDF(
+          "id",
+          "region",
+          "amount")
+
+      var nativeFailure: Throwable = null
+      val nativePlans = withNativeEnabled {
+        capturePlans(spark, includeFailures = true) {
+          nativeFailure = intercept[Exception](appendUnclustered(nativeTable, rows))
+        }
+      }
+      val jvmFailure = intercept[Exception](appendUnclustered(jvmTable, rows))
+      assert(
+        nativePlans.exists(p =>
+          collectWithSubqueries(p) { case e: CometIcebergWriteExec => e }.nonEmpty),
+        "expected the aborted write to have gone through CometIcebergWriteExec. Plans:\n" +
+          nativePlans.mkString("\n--\n"))
+
+      val nativeError = clusteredWriterError(nativeFailure)
+      val jvmError = clusteredWriterError(jvmFailure)
+      // Equal depth is what makes `assertThatThrownBy(...).cause()` find it in the same place.
+      assert(nativeError == jvmError, s"native $nativeError != jvm $jvmError")
+      val (_, message) = jvmError
+      assert(
+        message.startsWith(
+          "Incoming records violate the writer assumption that records are clustered by spec"),
+        s"unexpected IllegalStateException from the JVM writer: $message")
+      // Neither write may have committed.
+      tables.foreach(assertRows(_, Seq.empty))
+    }
+  }
+
+  // The error names the offending partition, so it runs into the same rendering divergence the
+  // partition-path test below pins: iceberg-java base64-encodes a binary value where iceberg-rust
+  // writes uppercase hex, and iceberg-rust panics outright on a pre-epoch `timestamptz` with a
+  // sub-second part -- which here would crash the task before the exception was even built. Comet
+  // renders the partition with the same Java-compatible formatter that names the data directories.
+  //
+  // Gated on Iceberg 1.8+ for the reason spelled out on that test: 1.5.x renders a `timestamptz`
+  // partition value as `1969-12-31T23:59:58.500Z`, so the two writers' messages cannot agree.
+  test("native acceleration: unclustered-input error names the partition like iceberg-java") {
+    assumeNativeAcceleration()
+    assume(icebergVersionAtLeast(1, 8), "timestamptz partition path spelling changed in 1.8")
+    withIcebergCatalog { _ =>
+      // A UTC session zone makes the stored micros of each literal exact, so the expected
+      // rendering below is not a function of the machine's zone.
+      withSQLConf("spark.sql.session.timeZone" -> "UTC") {
+        val (nativeTable, jvmTable) = ("unclustered_path_native", "unclustered_path_jvm")
+        Seq(nativeTable, jvmTable).foreach { table =>
+          spark.sql(s"""
+            CREATE TABLE $catalog.$ns.$table (id INT, ts TIMESTAMP, bin BINARY)
+            USING iceberg PARTITIONED BY (ts, bin)
+          """)
+        }
+        val session = spark
+        import session.implicits._
+        // -1500 epoch millis is 1969-12-31T23:59:58.5Z, the value iceberg-rust panics on. Its
+        // partition is revisited after the writer has moved on to the epoch and closed it.
+        val rows = Seq(
+          (1, new Timestamp(-1500L), Array[Byte](0, 1, -1)),
+          (2, new Timestamp(0L), Array[Byte](0)),
+          (3, new Timestamp(-1500L), Array[Byte](0, 1, -1))).toDF("id", "ts", "bin")
+
+        var nativeFailure: Throwable = null
+        val nativePlans = withNativeEnabled {
+          capturePlans(spark, includeFailures = true) {
+            nativeFailure = intercept[Exception](appendUnclustered(nativeTable, rows))
+          }
+        }
+        val jvmFailure = intercept[Exception](appendUnclustered(jvmTable, rows))
+        assert(
+          nativePlans.exists(p =>
+            collectWithSubqueries(p) { case e: CometIcebergWriteExec => e }.nonEmpty),
+          "expected the aborted write to have gone through CometIcebergWriteExec. Plans:\n" +
+            nativePlans.mkString("\n--\n"))
+
+        val (_, nativeMessage) = clusteredWriterError(nativeFailure)
+        assert(
+          clusteredWriterError(nativeFailure) == clusteredWriterError(jvmFailure),
+          s"native message differs from the JVM writer's:\n$nativeMessage")
+        // Pinned explicitly too: base64 rather than hex for the binary value, and the ISO
+        // spelling of the pre-epoch timestamp rather than a panic.
+        assert(
+          nativeMessage.contains("partition 'ts=1969-12-31T23%3A59%3A58.5%2B00%3A00/bin=AAH%2F'"),
+          nativeMessage)
+      }
+    }
+  }
+
   test("native acceleration: target-file-size rolls one task across multiple files") {
     assumeNativeAcceleration()
     withIcebergCatalog { warehouseDir =>
@@ -1898,6 +2004,33 @@ class CometIcebergWriteActionSuite
       .append()
   }
 
+  /**
+   * Appends `rows` as a single task with the table's distribution and ordering disabled and
+   * fanout off. That is the supported way to hand a clustered writer input it cannot accept, and
+   * what Iceberg's own `TestRequiredDistributionAndOrdering` does: without the first option
+   * Iceberg requests a partition-local sort that would cluster the rows, and without the second
+   * the fanout writer would accept them.
+   */
+  private def appendUnclustered(tableName: String, rows: DataFrame): Unit =
+    rows
+      .coalesce(1)
+      .writeTo(s"$catalog.$ns.$tableName")
+      .option("use-table-distribution-and-ordering", "false")
+      .option("fanout-enabled", "false")
+      .append()
+
+  /**
+   * Where the clustered writer's `IllegalStateException` sits in `t`'s cause chain, and what it
+   * says. Stage ids and driver stack traces make the enclosing `SparkException` messages differ
+   * between runs, so only the writer's own exception can be compared across writers.
+   */
+  private def clusteredWriterError(t: Throwable): (Int, String) = {
+    val chain = causeChain(t)
+    val depth = chain.indexWhere(_.isInstanceOf[IllegalStateException])
+    assert(depth >= 0, s"no IllegalStateException in the cause chain of $t")
+    (depth, chain(depth).getMessage)
+  }
+
   private def captureWrite(tableName: String)(action: => Unit): WriteSnapshot = {
     val before = countSnapshots(tableName)
     val plans = capturePlans(spark)(action)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org