You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/09/09 10:41:47 UTC
(datafusion-comet) branch main updated: fix: match iceberg-java's exception for unclustered input to a clustered Iceberg write (#5779)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 79d1be1eb3 fix: match iceberg-java's exception for unclustered input to a clustered Iceberg write (#5779)
79d1be1eb3 is described below
commit 79d1be1eb3cdb147d163231f54682970e02cbb6b
Author: Andy Grove <ag...@apache.org>
AuthorDate: Wed Sep 9 04:41:42 2026 -0600
fix: match iceberg-java's exception for unclustered input to a clustered Iceberg write (#5779)
---
.../execution/operators/iceberg_partition_path.rs | 55 ++--
.../core/src/execution/operators/iceberg_write.rs | 296 ++++++++++++++++++++-
native/jni-bridge/src/errors.rs | 87 ++++--
.../org/apache/comet/CometIcebergTestBase.scala | 10 +-
.../comet/CometIcebergWriteActionSuite.scala | 133 +++++++++
5 files changed, 522 insertions(+), 59 deletions(-)
diff --git a/native/core/src/execution/operators/iceberg_partition_path.rs b/native/core/src/execution/operators/iceberg_partition_path.rs
index a2cc5a95a3..363a9f2f19 100644
--- a/native/core/src/execution/operators/iceberg_partition_path.rs
+++ b/native/core/src/execution/operators/iceberg_partition_path.rs
@@ -73,33 +73,42 @@ impl CometLocationGenerator {
})
}
- /// Mirrors iceberg-java's `PartitionSpec#partitionToPath`: `name=value` pairs joined by `/`,
- /// with both halves form-urlencoded. `form_urlencoded` leaves exactly the byte set Java's
- /// `URLEncoder.encode(s, UTF_8)` leaves (`A-Za-z0-9`, `*`, `-`, `.`, `_`), maps space to `+`,
- /// and percent-encodes the rest with uppercase hex, so the two agree byte for byte.
fn partition_to_path(&self, key: &PartitionKey) -> String {
- let fields = self.partition_type.fields();
- key.spec()
- .fields()
- .iter()
- .enumerate()
- .map(|(index, field)| {
- // Indexed rather than zipped so a spec/partition-type/value length disagreement
- // renders as "null" instead of panicking; the three are built from the same spec,
- // so they always line up in practice.
- let value = key.data().fields().get(index).and_then(Option::as_ref);
- let human = match fields.get(index) {
- Some(nested) => human_string(&field.transform, &nested.field_type, value),
- None => NULL.to_string(),
- };
- form_urlencoded::Serializer::new(String::new())
- .append_pair(&field.name, &human)
- .finish()
- })
- .join("/")
+ partition_to_path(&self.partition_type, key)
}
}
+/// Mirrors iceberg-java's `PartitionSpec#partitionToPath`: `name=value` pairs joined by `/`, with
+/// both halves form-urlencoded. `form_urlencoded` leaves exactly the byte set Java's
+/// `URLEncoder.encode(s, UTF_8)` leaves (`A-Za-z0-9`, `*`, `-`, `.`, `_`), maps space to `+`, and
+/// percent-encodes the rest with uppercase hex, so the two agree byte for byte.
+///
+/// `partition_type` is `key`'s spec resolved against its schema. It is passed in rather than
+/// resolved here because resolution can fail and every caller already holds one: the location
+/// generator resolves it at task start, and `ClusteredBatchSplitter` needs it to read partition
+/// values.
+pub(crate) fn partition_to_path(partition_type: &StructType, key: &PartitionKey) -> String {
+ let fields = partition_type.fields();
+ key.spec()
+ .fields()
+ .iter()
+ .enumerate()
+ .map(|(index, field)| {
+ // Indexed rather than zipped so a spec/partition-type/value length disagreement
+ // renders as "null" instead of panicking; the three are built from the same spec,
+ // so they always line up in practice.
+ let value = key.data().fields().get(index).and_then(Option::as_ref);
+ let human = match fields.get(index) {
+ Some(nested) => human_string(&field.transform, &nested.field_type, value),
+ None => NULL.to_string(),
+ };
+ form_urlencoded::Serializer::new(String::new())
+ .append_pair(&field.name, &human)
+ .finish()
+ })
+ .join("/")
+}
+
impl LocationGenerator for CometLocationGenerator {
fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String {
// `is_effectively_none` is iceberg-rust's own predicate: no key, or a key whose spec is
diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs
index 1fefcf7dd4..297d9a0c74 100644
--- a/native/core/src/execution/operators/iceberg_write.rs
+++ b/native/core/src/execution/operators/iceberg_write.rs
@@ -58,6 +58,7 @@ use iceberg::writer::file_writer::ParquetWriterBuilder;
use iceberg::writer::partitioning::clustered_writer::ClusteredWriter;
use iceberg::writer::partitioning::fanout_writer::FanoutWriter;
use iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter;
+use iceberg::ErrorKind;
#[cfg(test)]
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel};
@@ -69,8 +70,11 @@ use datafusion_comet_proto::spark_operator::{
};
use crate::cloud::s3::credential_bridge::AccessMode;
+use crate::errors::CometError;
use crate::execution::operators::iceberg_common::load_file_io;
-use crate::execution::operators::iceberg_partition_path::CometLocationGenerator;
+use crate::execution::operators::iceberg_partition_path::{
+ partition_to_path, CometLocationGenerator,
+};
/// Builder chain instantiated once per task and handed to the partitioning wrapper.
type IcebergDataFileWriterBuilder =
@@ -417,11 +421,16 @@ impl InnerWriter {
Ok(())
}
InnerWriter::Clustered(w) => {
- let parts = clustered_splitter
- .expect("clustered splitter must be Some for clustered writes")
- .split(&batch)?;
- for (key, part) in parts {
- w.write(key, part).await.map_err(iceberg_err)?;
+ let splitter = clustered_splitter
+ .expect("clustered splitter must be Some for clustered writes");
+ for (key, part) in splitter.split(&batch)? {
+ // `write` consumes the key, but the unclustered-input error needs it for the
+ // partition path. One extra spec clone per run on top of the one
+ // `ClusteredBatchSplitter::partition_key` already pays.
+ let key_for_error = key.clone();
+ w.write(key, part)
+ .await
+ .map_err(|e| clustered_write_err(e, &key_for_error, splitter))?;
}
Ok(())
}
@@ -458,6 +467,76 @@ fn iceberg_err(e: iceberg::Error) -> DataFusionError {
DataFusionError::External(Box::new(e))
}
+/// iceberg-java's `ClusteredWriter` preamble, copied verbatim from
+/// `core/src/main/java/org/apache/iceberg/io/ClusteredWriter.java`. Applications catch that
+/// writer's `IllegalStateException` and match on this text -- Iceberg's own
+/// `TestRequiredDistributionAndOrdering` does both -- so the native writer reproduces it instead
+/// of surfacing iceberg-rust's differently worded error.
+const NOT_CLUSTERED_ROWS_ERROR_MSG_TEMPLATE: &str = "Incoming records violate the writer \
+ assumption that records are clustered by spec and by partition within each spec. Either \
+ cluster the incoming records or switch to fanout writers.\n\
+ Encountered records that belong to already closed files:\n";
+
+/// The message iceberg-rust's `ClusteredWriter` raises for the same condition.
+const UNSORTED_INPUT_MESSAGE_PREFIX: &str = "The input is not sorted!";
+
+/// Restates iceberg-rust's unclustered-input failure as iceberg-java's, passing every other
+/// failure through unchanged.
+///
+/// Leaving `ClusteredWriter` as the sole judge of whether the input is clustered means coupling to
+/// its message text, which is the cheaper of the two couplings available: re-deriving the
+/// condition here would need a copy of the writer's closed-partition bookkeeping, and because that
+/// copy would fire first, no test could catch it drifting from the original. A wording change
+/// upstream instead makes `clustered_write_rejects_unclustered_input_like_iceberg_java` fail, since
+/// that test drives the real writer.
+fn clustered_write_err(
+ e: iceberg::Error,
+ key: &PartitionKey,
+ splitter: &ClusteredBatchSplitter,
+) -> DataFusionError {
+ if e.kind() == ErrorKind::Unexpected && e.message().starts_with(UNSORTED_INPUT_MESSAGE_PREFIX) {
+ not_clustered_error(key, &splitter.partition_type)
+ } else {
+ iceberg_err(e)
+ }
+}
+
+/// The error iceberg-java's `ClusteredWriter.write` raises when a closed partition is revisited,
+/// down to the `partition '<path>' in spec <spec>` context. Only the partition branch of that
+/// check is reachable here: a task writes through a single output spec, so the spec can never
+/// change mid-stream.
+///
+/// The path comes from the same Java-compatible renderer that names the data directories, not from
+/// `PartitionKey::to_path`: iceberg-rust hex-encodes binary where iceberg-java base64-encodes it,
+/// and panics outright on a pre-epoch `timestamptz` with a sub-second part.
+fn not_clustered_error(key: &PartitionKey, partition_type: &StructType) -> DataFusionError {
+ DataFusionError::from(CometError::IllegalState(format!(
+ "{NOT_CLUSTERED_ROWS_ERROR_MSG_TEMPLATE}partition '{}' in spec {}",
+ partition_to_path(partition_type, key),
+ format_partition_spec(key.spec())
+ )))
+}
+
+/// Renders a partition spec the way iceberg-java's `PartitionSpec.toString` and
+/// `PartitionField.toString` do, so the error context matches character for character.
+fn format_partition_spec(spec: &PartitionSpec) -> String {
+ let fields: String = spec
+ .fields()
+ .iter()
+ .map(|field| {
+ format!(
+ "\n {}: {}: {}({})",
+ field.field_id, field.name, field.transform, field.source_id
+ )
+ })
+ .collect();
+ if fields.is_empty() {
+ "[]".to_string()
+ } else {
+ format!("[{fields}\n]")
+ }
+}
+
fn build_output_schema() -> SchemaRef {
Arc::new(ArrowSchema::new(vec![Field::new(
"iceberg_manifest",
@@ -880,7 +959,8 @@ mod tests {
mod integration {
use super::super::*;
- use arrow::array::{Int32Array, StringArray};
+ use arrow::array::{BinaryArray, Int32Array, StringArray, TimestampMicrosecondArray};
+ use arrow::datatypes::TimeUnit;
use datafusion::common::Result as DFResult;
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion_comet_proto::spark_operator::{
@@ -925,8 +1005,13 @@ mod tests {
.unwrap()
}
+ /// Streams `batches` under their own schema, so a test can use a column layout other than
+ /// [`user_schema`].
fn input_stream(batches: Vec<RecordBatch>) -> SendableRecordBatchStream {
- let schema = user_schema();
+ let schema = batches
+ .first()
+ .map(RecordBatch::schema)
+ .unwrap_or_else(user_schema);
Box::pin(RecordBatchStreamAdapter::new(
schema,
futures::stream::iter(batches.into_iter().map(Ok::<_, DataFusionError>)),
@@ -1125,6 +1210,201 @@ mod tests {
assert_eq!(total, 5);
}
+ /// Runs a clustered write over input that revisits a closed partition, and returns the
+ /// `CometError::IllegalState` message it has to fail with.
+ ///
+ /// Driving the real iceberg-rust writer is what makes these tests the tripwire for
+ /// [`UNSORTED_INPUT_MESSAGE_PREFIX`] going stale on an iceberg-rust bump: the translation
+ /// stops matching and the error arrives as a plain `iceberg::Error` instead.
+ async fn unclustered_write_message(
+ schema: Schema,
+ spec: PartitionSpec,
+ batches: Vec<RecordBatch>,
+ ) -> String {
+ let temp_dir = TempDir::new().unwrap();
+ let common = common(
+ format!("file://{}", temp_dir.path().display()),
+ serde_json::to_string(&spec).unwrap(),
+ serde_json::to_string(&schema).unwrap(),
+ ProtoIcebergWriterMode::IcebergWriterClustered,
+ );
+ let err = run(
+ common,
+ schema,
+ spec,
+ ProtoIcebergWriterMode::IcebergWriterClustered,
+ batches,
+ )
+ .await
+ .unwrap_err();
+ let comet_error = match &err {
+ DataFusionError::External(external) => external.downcast_ref::<CometError>(),
+ _ => None,
+ };
+ match comet_error {
+ Some(CometError::IllegalState(message)) => message.clone(),
+ _ => panic!("expected CometError::IllegalState, got {err:?}"),
+ }
+ }
+
+ /// Builds an identity-partitioned single-column spec over `schema`.
+ fn identity_spec(schema: &Schema, column: &str) -> PartitionSpec {
+ PartitionSpec::builder(Arc::new(schema.clone()))
+ .with_spec_id(1)
+ .add_partition_field(column, column, Transform::Identity)
+ .unwrap()
+ .build()
+ .unwrap()
+ }
+
+ /// Unclustered input must be rejected with iceberg-java's `ClusteredWriter` error rather
+ /// than iceberg-rust's "The input is not sorted!", and it has to reach the JVM as an
+ /// `IllegalStateException`. Iceberg's own `TestRequiredDistributionAndOrdering` asserts
+ /// both. See https://github.com/apache/datafusion-comet/issues/5698.
+ #[tokio::test]
+ async fn clustered_write_rejects_unclustered_input_like_iceberg_java() {
+ let schema = iceberg_user_schema();
+ let spec = identity_spec(&schema, "region");
+ // "eu" is revisited after the writer has moved on to "us" and closed it.
+ let message = unclustered_write_message(
+ schema,
+ spec,
+ vec![batch(&[1, 2], &["eu", "us"]), batch(&[3], &["eu"])],
+ )
+ .await;
+
+ // The wording itself is pinned against the real JVM writer by
+ // CometIcebergWriteActionSuite; here it only has to carry the right context.
+ assert_eq!(
+ message,
+ format!(
+ "{NOT_CLUSTERED_ROWS_ERROR_MSG_TEMPLATE}\
+ partition 'region=eu' in spec [\n 1000: region: identity(2)\n]"
+ )
+ );
+ }
+
+ /// iceberg-java base64-encodes a binary partition value where iceberg-rust's
+ /// `PartitionKey::to_path` writes uppercase hex, so the error has to go through the same
+ /// Java-compatible renderer that names the data directories.
+ #[tokio::test]
+ async fn unclustered_binary_partition_renders_like_iceberg_java() {
+ let schema = Schema::builder()
+ .with_schema_id(1)
+ .with_fields(vec![
+ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::required(2, "part", Type::Primitive(PrimitiveType::Binary)).into(),
+ ])
+ .build()
+ .unwrap();
+ let spec = identity_spec(&schema, "part");
+ let arrow_schema = Arc::new(ArrowSchema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("part", DataType::Binary, false),
+ ]));
+ let batch = |ids: &[i32], parts: Vec<&[u8]>| {
+ RecordBatch::try_new(
+ Arc::clone(&arrow_schema),
+ vec![
+ Arc::new(Int32Array::from(ids.to_vec())),
+ Arc::new(BinaryArray::from(parts)),
+ ],
+ )
+ .unwrap()
+ };
+
+ let message = unclustered_write_message(
+ schema,
+ spec,
+ vec![
+ batch(&[1, 2], vec![&[0x00, 0x01, 0xFF], &[0x00]]),
+ batch(&[3], vec![&[0x00, 0x01, 0xFF]]),
+ ],
+ )
+ .await;
+
+ // base64(00 01 FF) = "AAH/", form-urlencoded to "AAH%2F". iceberg-rust would have
+ // rendered the same bytes as "0001FF".
+ assert!(
+ message
+ .ends_with("partition 'part=AAH%2F' in spec [\n 1000: part: identity(2)\n]"),
+ "{message}"
+ );
+ }
+
+ /// iceberg-rust's `Transform::to_human_string` panics on a pre-epoch `timestamptz` with a
+ /// sub-second part, so routing the error through `PartitionKey::to_path` would crash the
+ /// task instead of raising `IllegalStateException`.
+ #[tokio::test]
+ async fn unclustered_negative_timestamptz_partition_renders_like_iceberg_java() {
+ let schema = Schema::builder()
+ .with_schema_id(1)
+ .with_fields(vec![
+ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
+ NestedField::required(2, "ts", Type::Primitive(PrimitiveType::Timestamptz))
+ .into(),
+ ])
+ .build()
+ .unwrap();
+ let spec = identity_spec(&schema, "ts");
+ let arrow_schema = Arc::new(ArrowSchema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new(
+ "ts",
+ DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
+ false,
+ ),
+ ]));
+ let batch = |ids: &[i32], micros: &[i64]| {
+ RecordBatch::try_new(
+ Arc::clone(&arrow_schema),
+ vec![
+ Arc::new(Int32Array::from(ids.to_vec())),
+ Arc::new(
+ TimestampMicrosecondArray::from(micros.to_vec())
+ .with_timezone("+00:00"),
+ ),
+ ],
+ )
+ .unwrap()
+ };
+
+ // -1_500_000 micros is 1969-12-31T23:59:58.5Z -- negative with a sub-second part.
+ let message = unclustered_write_message(
+ schema,
+ spec,
+ vec![batch(&[1, 2], &[-1_500_000, 0]), batch(&[3], &[-1_500_000])],
+ )
+ .await;
+
+ assert!(
+ message.ends_with(
+ "partition 'ts=1969-12-31T23%3A59%3A58.5%2B00%3A00' in spec \
+ [\n 1000: ts: identity(2)\n]"
+ ),
+ "{message}"
+ );
+ }
+
+ /// The transform rendering in [`format_partition_spec`] has to match iceberg-java's
+ /// `PartitionField.toString` for the parameterised transforms too.
+ #[test]
+ fn partition_spec_renders_like_iceberg_java() {
+ let schema = iceberg_user_schema();
+ let spec = PartitionSpec::builder(Arc::new(schema))
+ .with_spec_id(7)
+ .add_partition_field("id", "id_bucket", Transform::Bucket(8))
+ .unwrap()
+ .add_partition_field("region", "region_trunc", Transform::Truncate(4))
+ .unwrap()
+ .build()
+ .unwrap();
+ assert_eq!(
+ format_partition_spec(&spec),
+ "[\n 1000: id_bucket: bucket[8](1)\n 1001: region_trunc: truncate[4](2)\n]"
+ );
+ }
+
#[tokio::test]
async fn encoded_manifest_round_trips_through_iceberg_parser() {
let temp_dir = TempDir::new().unwrap();
diff --git a/native/jni-bridge/src/errors.rs b/native/jni-bridge/src/errors.rs
index a072be95ed..3e72f6e804 100644
--- a/native/jni-bridge/src/errors.rs
+++ b/native/jni-bridge/src/errors.rs
@@ -95,6 +95,12 @@ pub enum CometError {
#[error("{0}")]
ShuffleSizeLimit(String),
+ /// Must reach the JVM as `java.lang.IllegalStateException` carrying this exact message: for
+ /// runtime checks where Comet reimplements a JVM library's guard and callers match on the
+ /// exception that library raises. Internal invariants belong in [`CometError::Internal`].
+ #[error("{0}")]
+ IllegalState(String),
+
#[error(transparent)]
Arrow {
#[from]
@@ -219,9 +225,10 @@ impl From<CometError> for DataFusionError {
// own codegen inside the JVM UDF kernel) as an `External` error so it survives the trip
// back through DataFusion and can be re-thrown with its exact type at the JNI boundary.
// Flattening it to a string here would surface it as a generic CometNativeException.
- value @ (CometError::JavaException { .. } | CometError::ShuffleSizeLimit(_)) => {
- DataFusionError::External(Box::new(value))
- }
+ // The same applies to the classifications that must keep their JVM exception class.
+ value @ (CometError::JavaException { .. }
+ | CometError::ShuffleSizeLimit(_)
+ | CometError::IllegalState(_)) => DataFusionError::External(Box::new(value)),
_ => DataFusionError::Execution(value.to_string()),
}
}
@@ -349,6 +356,10 @@ impl jni::errors::ToException for CometError {
class: "org/apache/comet/CometShuffleSizeLimitException".to_string(),
msg: message.clone(),
},
+ CometError::IllegalState(message) => Exception {
+ class: "java/lang/IllegalStateException".to_string(),
+ msg: message.clone(),
+ },
_other => Exception {
class: "org/apache/comet/CometNativeException".to_string(),
msg: self.to_string(),
@@ -485,11 +496,12 @@ fn throw_exception(env: &mut Env, error: &CometError, backtrace: Option<String>)
// If there isn't already an exception?
if !env.exception_check() {
// DataFusion operators can wrap the original failure in Context, Shared, or External
- // errors. Keep capacity failures typed across those wrappers and the JNI boundary.
- if let Some(message) = shuffle_size_limit_message(error) {
+ // errors. Keep the classifications that own their JVM exception class typed across those
+ // wrappers and the JNI boundary.
+ if let Some(exception) = typed_jvm_exception(error) {
let _ = env.throw_new(
- jni::jni_str!("org/apache/comet/CometShuffleSizeLimitException"),
- JNIString::new(message),
+ JNIString::new(exception.class),
+ JNIString::new(exception.msg),
);
return;
}
@@ -574,11 +586,17 @@ fn throw_exception(env: &mut Env, error: &CometError, backtrace: Option<String>)
}
}
-fn shuffle_size_limit_message<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a str> {
+/// Walks the cause chain for a `CometError` variant that owns its JVM exception class, so class and
+/// message survive DataFusion's `Context` / `Shared` / `External` wrappers. The class comes from
+/// `to_exception`, keeping one variant-to-class table. Classification is by variant only -- an
+/// error that merely mentions the same words keeps its normal classification.
+fn typed_jvm_exception(error: &(dyn std::error::Error + 'static)) -> Option<Exception> {
let mut cause = Some(error);
while let Some(error) = cause {
- if let Some(CometError::ShuffleSizeLimit(message)) = error.downcast_ref::<CometError>() {
- return Some(message);
+ if let Some(typed @ (CometError::ShuffleSizeLimit(_) | CometError::IllegalState(_))) =
+ error.downcast_ref::<CometError>()
+ {
+ return Some(typed.to_exception());
}
cause = error.source();
}
@@ -943,34 +961,53 @@ mod tests {
}
}
- #[test]
- #[cfg_attr(miri, ignore)] // miri cannot create a JVM.
- fn shuffle_size_limit_survives_datafusion_wrappers_and_jni() {
- let message = "Remote shuffle exceeds spark.comet.shuffle.rss.maxInFlightBytes";
- let error = DataFusionError::from(CometError::ShuffleSizeLimit(message.to_string()));
- let error = DataFusionError::Shared(Arc::new(DataFusionError::Context(
- "executing shuffle writer".to_string(),
- Box::new(error),
+ /// Wraps `error` the way a DataFusion operator would, throws it, and asserts the pending Java
+ /// exception is `expected_class` with the message intact -- i.e. that the variant kept its own
+ /// JVM exception class instead of collapsing into `CometNativeException`.
+ fn assert_survives_datafusion_wrappers(error: CometError, message: &str, expected_class: &str) {
+ let wrapped = DataFusionError::Shared(Arc::new(DataFusionError::Context(
+ "executing operator".to_string(),
+ Box::new(DataFusionError::from(error)),
)));
jvm()
.attach_current_thread(|env| -> jni::errors::Result<()> {
- unwrap_or_throw_default::<()>(env, Err(CometError::from(error)));
- assert_pending_java_exception_detailed(
- env,
- Some("org/apache/comet/CometShuffleSizeLimitException"),
- Some(message),
- );
+ unwrap_or_throw_default::<()>(env, Err(CometError::from(wrapped)));
+ assert_pending_java_exception_detailed(env, Some(expected_class), Some(message));
Ok(())
})
.unwrap();
}
+ #[test]
+ #[cfg_attr(miri, ignore)] // miri cannot create a JVM.
+ fn shuffle_size_limit_survives_datafusion_wrappers_and_jni() {
+ let message = "Remote shuffle exceeds spark.comet.shuffle.rss.maxInFlightBytes";
+ assert_survives_datafusion_wrappers(
+ CometError::ShuffleSizeLimit(message.to_string()),
+ message,
+ "org/apache/comet/CometShuffleSizeLimitException",
+ );
+ }
+
#[test]
fn shuffle_size_limit_is_not_inferred_from_error_text() {
let error = CometError::from(DataFusionError::Execution(
"Remote shuffle exceeds spark.comet.shuffle.rss.maxInFlightBytes".to_string(),
));
- assert!(shuffle_size_limit_message(&error).is_none());
+ assert!(typed_jvm_exception(&error).is_none());
+ }
+
+ /// The native Iceberg writer rejects unclustered input with iceberg-java's own
+ /// `IllegalStateException`, so it has to survive the same wrappers.
+ #[test]
+ #[cfg_attr(miri, ignore)] // miri cannot create a JVM.
+ fn illegal_state_survives_datafusion_wrappers_and_jni() {
+ let message = "Incoming records violate the writer assumption that records are clustered";
+ assert_survives_datafusion_wrappers(
+ CometError::IllegalState(message.to_string()),
+ message,
+ "java/lang/IllegalStateException",
+ );
}
#[test]
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala b/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala
index f6e21f3901..03f8fe28f6 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala
@@ -137,15 +137,19 @@ trait CometIcebergTestBase {
file.delete()
}
- /** The executed plan of every query that completes successfully while `action` runs. */
- protected def capturePlans(spark: SparkSession)(action: => Unit): Seq[SparkPlan] = {
+ /**
+ * The executed plan of every query that ran while `action` ran. Queries that failed are
+ * included only when `includeFailures` is set, which is what an action expected to abort needs.
+ */
+ protected def capturePlans(spark: SparkSession, includeFailures: Boolean = false)(
+ action: => Unit): Seq[SparkPlan] = {
val captured = mutable.Buffer.empty[SparkPlan]
val listener = new QueryExecutionListener {
override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
captured += qe.executedPlan
}
override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit =
- ()
+ if (includeFailures) captured += qe.executedPlan
}
spark.listenerManager.register(listener)
try {
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala
index 9b0d8f412c..662d7cb359 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala
@@ -20,6 +20,7 @@
package org.apache.comet
import java.io.File
+import java.sql.Timestamp
import java.util.concurrent.{CountDownLatch, TimeUnit}
import scala.collection.mutable
@@ -29,6 +30,7 @@ import scala.concurrent.duration.DurationInt
import org.apache.spark.SparkConf
import org.apache.spark.sql.CometTestBase
+import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.Row
import org.apache.spark.sql.comet.{CometIcebergWriteExec, IcebergCommitExec, IcebergWriteExec}
import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog
@@ -1222,6 +1224,110 @@ class CometIcebergWriteActionSuite
}
}
+ // https://github.com/apache/datafusion-comet/issues/5698. iceberg-java's `ClusteredWriter`
+ // rejects unclustered input with a documented `IllegalStateException`, and callers match on both
+ // the type and the message -- Iceberg's own `TestRequiredDistributionAndOrdering` does exactly
+ // that. Pin the native writer against the JVM writer on the same runtime rather than against a
+ // literal, so an iceberg-java wording change surfaces here as a parity failure.
+ test("native acceleration: clustered writer rejects unclustered input like the JVM writer") {
+ assumeNativeAcceleration()
+ withIcebergCatalog { warehouseDir =>
+ val (nativeTable, jvmTable) = ("unclustered_native", "unclustered_jvm")
+ val tables = Seq(nativeTable, jvmTable)
+ tables.foreach(createTable(warehouseDir, _, partitionSpec = "PARTITIONED BY (region)"))
+ val session = spark
+ import session.implicits._
+ // "us-east" is revisited after the writer has moved on to "eu" and closed it.
+ val rows =
+ Seq((1, "us-east", 1.0), (2, "eu", 2.0), (3, "us-east", 3.0)).toDF(
+ "id",
+ "region",
+ "amount")
+
+ var nativeFailure: Throwable = null
+ val nativePlans = withNativeEnabled {
+ capturePlans(spark, includeFailures = true) {
+ nativeFailure = intercept[Exception](appendUnclustered(nativeTable, rows))
+ }
+ }
+ val jvmFailure = intercept[Exception](appendUnclustered(jvmTable, rows))
+ assert(
+ nativePlans.exists(p =>
+ collectWithSubqueries(p) { case e: CometIcebergWriteExec => e }.nonEmpty),
+ "expected the aborted write to have gone through CometIcebergWriteExec. Plans:\n" +
+ nativePlans.mkString("\n--\n"))
+
+ val nativeError = clusteredWriterError(nativeFailure)
+ val jvmError = clusteredWriterError(jvmFailure)
+ // Equal depth is what makes `assertThatThrownBy(...).cause()` find it in the same place.
+ assert(nativeError == jvmError, s"native $nativeError != jvm $jvmError")
+ val (_, message) = jvmError
+ assert(
+ message.startsWith(
+ "Incoming records violate the writer assumption that records are clustered by spec"),
+ s"unexpected IllegalStateException from the JVM writer: $message")
+ // Neither write may have committed.
+ tables.foreach(assertRows(_, Seq.empty))
+ }
+ }
+
+ // The error names the offending partition, so it runs into the same rendering divergence the
+ // partition-path test below pins: iceberg-java base64-encodes a binary value where iceberg-rust
+ // writes uppercase hex, and iceberg-rust panics outright on a pre-epoch `timestamptz` with a
+ // sub-second part -- which here would crash the task before the exception was even built. Comet
+ // renders the partition with the same Java-compatible formatter that names the data directories.
+ //
+ // Gated on Iceberg 1.8+ for the reason spelled out on that test: 1.5.x renders a `timestamptz`
+ // partition value as `1969-12-31T23:59:58.500Z`, so the two writers' messages cannot agree.
+ test("native acceleration: unclustered-input error names the partition like iceberg-java") {
+ assumeNativeAcceleration()
+ assume(icebergVersionAtLeast(1, 8), "timestamptz partition path spelling changed in 1.8")
+ withIcebergCatalog { _ =>
+ // A UTC session zone makes the stored micros of each literal exact, so the expected
+ // rendering below is not a function of the machine's zone.
+ withSQLConf("spark.sql.session.timeZone" -> "UTC") {
+ val (nativeTable, jvmTable) = ("unclustered_path_native", "unclustered_path_jvm")
+ Seq(nativeTable, jvmTable).foreach { table =>
+ spark.sql(s"""
+ CREATE TABLE $catalog.$ns.$table (id INT, ts TIMESTAMP, bin BINARY)
+ USING iceberg PARTITIONED BY (ts, bin)
+ """)
+ }
+ val session = spark
+ import session.implicits._
+ // -1500 epoch millis is 1969-12-31T23:59:58.5Z, the value iceberg-rust panics on. Its
+ // partition is revisited after the writer has moved on to the epoch and closed it.
+ val rows = Seq(
+ (1, new Timestamp(-1500L), Array[Byte](0, 1, -1)),
+ (2, new Timestamp(0L), Array[Byte](0)),
+ (3, new Timestamp(-1500L), Array[Byte](0, 1, -1))).toDF("id", "ts", "bin")
+
+ var nativeFailure: Throwable = null
+ val nativePlans = withNativeEnabled {
+ capturePlans(spark, includeFailures = true) {
+ nativeFailure = intercept[Exception](appendUnclustered(nativeTable, rows))
+ }
+ }
+ val jvmFailure = intercept[Exception](appendUnclustered(jvmTable, rows))
+ assert(
+ nativePlans.exists(p =>
+ collectWithSubqueries(p) { case e: CometIcebergWriteExec => e }.nonEmpty),
+ "expected the aborted write to have gone through CometIcebergWriteExec. Plans:\n" +
+ nativePlans.mkString("\n--\n"))
+
+ val (_, nativeMessage) = clusteredWriterError(nativeFailure)
+ assert(
+ clusteredWriterError(nativeFailure) == clusteredWriterError(jvmFailure),
+ s"native message differs from the JVM writer's:\n$nativeMessage")
+ // Pinned explicitly too: base64 rather than hex for the binary value, and the ISO
+ // spelling of the pre-epoch timestamp rather than a panic.
+ assert(
+ nativeMessage.contains("partition 'ts=1969-12-31T23%3A59%3A58.5%2B00%3A00/bin=AAH%2F'"),
+ nativeMessage)
+ }
+ }
+ }
+
test("native acceleration: target-file-size rolls one task across multiple files") {
assumeNativeAcceleration()
withIcebergCatalog { warehouseDir =>
@@ -1898,6 +2004,33 @@ class CometIcebergWriteActionSuite
.append()
}
+ /**
+ * Appends `rows` as a single task with the table's distribution and ordering disabled and
+ * fanout off. That is the supported way to hand a clustered writer input it cannot accept, and
+ * what Iceberg's own `TestRequiredDistributionAndOrdering` does: without the first option
+ * Iceberg requests a partition-local sort that would cluster the rows, and without the second
+ * the fanout writer would accept them.
+ */
+ private def appendUnclustered(tableName: String, rows: DataFrame): Unit =
+ rows
+ .coalesce(1)
+ .writeTo(s"$catalog.$ns.$tableName")
+ .option("use-table-distribution-and-ordering", "false")
+ .option("fanout-enabled", "false")
+ .append()
+
+ /**
+ * Where the clustered writer's `IllegalStateException` sits in `t`'s cause chain, and what it
+ * says. Stage ids and driver stack traces make the enclosing `SparkException` messages differ
+ * between runs, so only the writer's own exception can be compared across writers.
+ */
+ private def clusteredWriterError(t: Throwable): (Int, String) = {
+ val chain = causeChain(t)
+ val depth = chain.indexWhere(_.isInstanceOf[IllegalStateException])
+ assert(depth >= 0, s"no IllegalStateException in the cause chain of $t")
+ (depth, chain(depth).getMessage)
+ }
+
private def captureWrite(tableName: String)(action: => Unit): WriteSnapshot = {
val before = countSnapshots(tableName)
val plans = capturePlans(spark)(action)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org