You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by oz...@apache.org on 2023/11/21 16:36:59 UTC
(arrow-datafusion) branch main updated: Interval Arithmetic Updates (#8276)
This is an automated email from the ASF dual-hosted git repository.
ozankabak pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new e9b9645ca0 Interval Arithmetic Updates (#8276)
e9b9645ca0 is described below
commit e9b9645ca0da5c6ce3d1e7d8210e442cf565cea0
Author: Berkay Şahin <12...@users.noreply.github.com>
AuthorDate: Tue Nov 21 19:36:51 2023 +0300
Interval Arithmetic Updates (#8276)
* Interval lib can be accessible from logical plan
* committed to merge precision PR
* minor fix after merge, adding ts-interval handling
* bound openness removed
* Remove all interval bound related code
* test fix
* fix docstrings
* Fix after merge
* Minor changes
* Simplifications
* Resolve linter errors
* Addressing reviews
* Fix win tests
* Update interval_arithmetic.rs
* Code simplifications
* Review Part 1
* Review Part 2
* Addressing Ozan's feedback
* Resolving conflicts
* Review Part 3
* Better cardinality calculation
* fix clippy
* Review Part 4
* type check, test polish, bug fix
* Constructs filter graph with datatypes
* Update Cargo.lock
* Update Cargo.toml
* Review
* Other expectations of AND
* OR operator implementation
* Certainly false asserting comparison operators
* Update interval_arithmetic.rs
* Tests added, is_superset renamed
* Final review
* Resolving conflicts
* Address review feedback
---------
Co-authored-by: Mustafa Akur <mu...@synnada.ai>
Co-authored-by: Mehmet Ozan Kabak <oz...@gmail.com>
---
datafusion-cli/Cargo.lock | 55 +-
datafusion/common/Cargo.toml | 12 +-
datafusion/common/src/lib.rs | 1 +
.../src/intervals => common/src}/rounding.rs | 8 +-
datafusion/common/src/scalar.rs | 42 +
datafusion/expr/Cargo.toml | 5 +-
datafusion/expr/src/interval_arithmetic.rs | 3307 ++++++++++++++++++++
datafusion/expr/src/lib.rs | 24 +-
datafusion/expr/src/type_coercion/binary.rs | 1 -
.../src/simplify_expressions/expr_simplifier.rs | 44 +-
.../src/simplify_expressions/guarantees.rs | 69 +-
datafusion/physical-expr/Cargo.toml | 8 +-
datafusion/physical-expr/src/analysis.rs | 158 +-
datafusion/physical-expr/src/expressions/binary.rs | 117 +-
datafusion/physical-expr/src/expressions/cast.rs | 18 +-
.../physical-expr/src/expressions/negative.rs | 52 +-
.../physical-expr/src/intervals/cp_solver.rs | 1038 +++---
.../src/intervals/interval_aritmetic.rs | 1886 -----------
datafusion/physical-expr/src/intervals/mod.rs | 5 -
datafusion/physical-expr/src/intervals/utils.rs | 104 +-
datafusion/physical-expr/src/physical_expr.rs | 30 +-
datafusion/physical-expr/src/sort_properties.rs | 2 +-
datafusion/physical-expr/src/utils.rs | 14 +-
datafusion/physical-plan/src/filter.rs | 23 +-
.../physical-plan/src/joins/stream_join_utils.rs | 33 +-
.../physical-plan/src/joins/symmetric_hash_join.rs | 11 +-
datafusion/physical-plan/src/joins/test_utils.rs | 14 +
datafusion/physical-plan/src/joins/utils.rs | 98 +-
28 files changed, 4406 insertions(+), 2773 deletions(-)
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index 06bc14c5b6..fa2832ab3f 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -360,9 +360,9 @@ dependencies = [
[[package]]
name = "async-compression"
-version = "0.4.5"
+version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5"
+checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2"
dependencies = [
"bzip2",
"flate2",
@@ -851,11 +851,10 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.0.83"
+version = "1.0.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+checksum = "0f8e7c90afad890484a21653d08b6e209ae34770fb5ee298f9c699fcc1e5c856"
dependencies = [
- "jobserver",
"libc",
]
@@ -1180,6 +1179,7 @@ dependencies = [
"arrow-schema",
"chrono",
"half",
+ "libc",
"num_cpus",
"object_store",
"parquet",
@@ -1213,6 +1213,7 @@ dependencies = [
"arrow",
"arrow-array",
"datafusion-common",
+ "paste",
"sqlparser",
"strum",
"strum_macros",
@@ -1255,7 +1256,6 @@ dependencies = [
"hex",
"indexmap 2.1.0",
"itertools 0.12.0",
- "libc",
"log",
"md-5",
"paste",
@@ -1423,9 +1423,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
-version = "0.3.7"
+version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8"
+checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e"
dependencies = [
"libc",
"windows-sys",
@@ -1647,9 +1647,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "h2"
-version = "0.3.22"
+version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"
+checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833"
dependencies = [
"bytes",
"fnv",
@@ -1657,7 +1657,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http",
- "indexmap 2.1.0",
+ "indexmap 1.9.3",
"slab",
"tokio",
"tokio-util",
@@ -1738,9 +1738,9 @@ dependencies = [
[[package]]
name = "http"
-version = "0.2.11"
+version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
+checksum = "f95b9abcae896730d42b78e09c155ed4ddf82c07b4de772c64aee5b2d8b7c150"
dependencies = [
"bytes",
"fnv",
@@ -1927,15 +1927,6 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
-[[package]]
-name = "jobserver"
-version = "0.1.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "js-sys"
version = "0.3.65"
@@ -2830,9 +2821,9 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.38.24"
+version = "0.38.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ad981d6c340a49cdc40a1028d9c6084ec7e9fa33fcb839cab656a267071e234"
+checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3"
dependencies = [
"bitflags 2.4.1",
"errno",
@@ -3260,9 +3251,9 @@ dependencies = [
[[package]]
name = "termcolor"
-version = "1.4.0"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"
+checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64"
dependencies = [
"winapi-util",
]
@@ -3901,18 +3892,18 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.7.26"
+version = "0.7.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0"
+checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.7.26"
+version = "0.7.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f"
+checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b"
dependencies = [
"proc-macro2",
"quote",
@@ -3921,9 +3912,9 @@ dependencies = [
[[package]]
name = "zeroize"
-version = "1.6.1"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12a3946ecfc929b583800f4629b6c25b88ac6e92a40ea5670f77112a85d40a8b"
+checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
[[package]]
name = "zstd"
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index b3a8101539..b69e1f7f3d 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -38,14 +38,22 @@ backtrace = []
pyarrow = ["pyo3", "arrow/pyarrow", "parquet"]
[dependencies]
-ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-apache-avro = { version = "0.16", default-features = false, features = ["bzip", "snappy", "xz", "zstandard"], optional = true }
+ahash = { version = "0.8", default-features = false, features = [
+ "runtime-rng",
+] }
+apache-avro = { version = "0.16", default-features = false, features = [
+ "bzip",
+ "snappy",
+ "xz",
+ "zstandard",
+], optional = true }
arrow = { workspace = true }
arrow-array = { workspace = true }
arrow-buffer = { workspace = true }
arrow-schema = { workspace = true }
chrono = { workspace = true }
half = { version = "2.1", default-features = false }
+libc = "0.2.140"
num_cpus = { workspace = true }
object_store = { workspace = true, optional = true }
parquet = { workspace = true, optional = true, default-features = true }
diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs
index 53c3cfddff..90fb4a8814 100644
--- a/datafusion/common/src/lib.rs
+++ b/datafusion/common/src/lib.rs
@@ -34,6 +34,7 @@ pub mod file_options;
pub mod format;
pub mod hash_utils;
pub mod parsers;
+pub mod rounding;
pub mod scalar;
pub mod stats;
pub mod test_util;
diff --git a/datafusion/physical-expr/src/intervals/rounding.rs b/datafusion/common/src/rounding.rs
similarity index 98%
rename from datafusion/physical-expr/src/intervals/rounding.rs
rename to datafusion/common/src/rounding.rs
index c1172fba91..413067ecd6 100644
--- a/datafusion/physical-expr/src/intervals/rounding.rs
+++ b/datafusion/common/src/rounding.rs
@@ -22,8 +22,8 @@
use std::ops::{Add, BitAnd, Sub};
-use datafusion_common::Result;
-use datafusion_common::ScalarValue;
+use crate::Result;
+use crate::ScalarValue;
// Define constants for ARM
#[cfg(all(target_arch = "aarch64", not(target_os = "windows")))]
@@ -162,7 +162,7 @@ impl FloatBits for f64 {
/// # Examples
///
/// ```
-/// use datafusion_physical_expr::intervals::rounding::next_up;
+/// use datafusion_common::rounding::next_up;
///
/// let f: f32 = 1.0;
/// let next_f = next_up(f);
@@ -195,7 +195,7 @@ pub fn next_up<F: FloatBits + Copy>(float: F) -> F {
/// # Examples
///
/// ```
-/// use datafusion_physical_expr::intervals::rounding::next_down;
+/// use datafusion_common::rounding::next_down;
///
/// let f: f32 = 1.0;
/// let next_f = next_down(f);
diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
index e8dac2a7f4..fd1ceb5fad 100644
--- a/datafusion/common/src/scalar.rs
+++ b/datafusion/common/src/scalar.rs
@@ -988,6 +988,48 @@ impl ScalarValue {
Self::try_from_array(r.as_ref(), 0)
}
+ /// Wrapping multiplication of `ScalarValue`
+ ///
+ /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
+ /// should operate on Arrays directly, using vectorized array kernels.
+ pub fn mul<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
+ let r = mul_wrapping(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
+ Self::try_from_array(r.as_ref(), 0)
+ }
+
+ /// Checked multiplication of `ScalarValue`
+ ///
+ /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
+ /// should operate on Arrays directly, using vectorized array kernels.
+ pub fn mul_checked<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
+ let r = mul(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
+ Self::try_from_array(r.as_ref(), 0)
+ }
+
+ /// Performs `lhs / rhs`
+ ///
+ /// Overflow or division by zero will result in an error, with exception to
+ /// floating point numbers, which instead follow the IEEE 754 rules.
+ ///
+ /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
+ /// should operate on Arrays directly, using vectorized array kernels.
+ pub fn div<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
+ let r = div(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
+ Self::try_from_array(r.as_ref(), 0)
+ }
+
+ /// Performs `lhs % rhs`
+ ///
+ /// Overflow or division by zero will result in an error, with exception to
+ /// floating point numbers, which instead follow the IEEE 754 rules.
+ ///
+ /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
+ /// should operate on Arrays directly, using vectorized array kernels.
+ pub fn rem<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
+ let r = rem(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
+ Self::try_from_array(r.as_ref(), 0)
+ }
+
pub fn is_unsigned(&self) -> bool {
matches!(
self,
diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
index 5b1b421538..3e05dae619 100644
--- a/datafusion/expr/Cargo.toml
+++ b/datafusion/expr/Cargo.toml
@@ -35,10 +35,13 @@ path = "src/lib.rs"
[features]
[dependencies]
-ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
+ahash = { version = "0.8", default-features = false, features = [
+ "runtime-rng",
+] }
arrow = { workspace = true }
arrow-array = { workspace = true }
datafusion-common = { workspace = true }
+paste = "^1.0"
sqlparser = { workspace = true }
strum = { version = "0.25.0", features = ["derive"] }
strum_macros = "0.25.0"
diff --git a/datafusion/expr/src/interval_arithmetic.rs b/datafusion/expr/src/interval_arithmetic.rs
new file mode 100644
index 0000000000..c85c6fc66b
--- /dev/null
+++ b/datafusion/expr/src/interval_arithmetic.rs
@@ -0,0 +1,3307 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Interval arithmetic library
+
+use std::borrow::Borrow;
+use std::fmt::{self, Display, Formatter};
+use std::ops::{AddAssign, SubAssign};
+
+use crate::type_coercion::binary::get_result_type;
+use crate::Operator;
+
+use arrow::compute::{cast_with_options, CastOptions};
+use arrow::datatypes::DataType;
+use arrow::datatypes::{IntervalUnit, TimeUnit};
+use datafusion_common::rounding::{alter_fp_rounding_mode, next_down, next_up};
+use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue};
+
+macro_rules! get_extreme_value {
+ ($extreme:ident, $value:expr) => {
+ match $value {
+ DataType::UInt8 => ScalarValue::UInt8(Some(u8::$extreme)),
+ DataType::UInt16 => ScalarValue::UInt16(Some(u16::$extreme)),
+ DataType::UInt32 => ScalarValue::UInt32(Some(u32::$extreme)),
+ DataType::UInt64 => ScalarValue::UInt64(Some(u64::$extreme)),
+ DataType::Int8 => ScalarValue::Int8(Some(i8::$extreme)),
+ DataType::Int16 => ScalarValue::Int16(Some(i16::$extreme)),
+ DataType::Int32 => ScalarValue::Int32(Some(i32::$extreme)),
+ DataType::Int64 => ScalarValue::Int64(Some(i64::$extreme)),
+ DataType::Float32 => ScalarValue::Float32(Some(f32::$extreme)),
+ DataType::Float64 => ScalarValue::Float64(Some(f64::$extreme)),
+ DataType::Duration(TimeUnit::Second) => {
+ ScalarValue::DurationSecond(Some(i64::$extreme))
+ }
+ DataType::Duration(TimeUnit::Millisecond) => {
+ ScalarValue::DurationMillisecond(Some(i64::$extreme))
+ }
+ DataType::Duration(TimeUnit::Microsecond) => {
+ ScalarValue::DurationMicrosecond(Some(i64::$extreme))
+ }
+ DataType::Duration(TimeUnit::Nanosecond) => {
+ ScalarValue::DurationNanosecond(Some(i64::$extreme))
+ }
+ DataType::Timestamp(TimeUnit::Second, _) => {
+ ScalarValue::TimestampSecond(Some(i64::$extreme), None)
+ }
+ DataType::Timestamp(TimeUnit::Millisecond, _) => {
+ ScalarValue::TimestampMillisecond(Some(i64::$extreme), None)
+ }
+ DataType::Timestamp(TimeUnit::Microsecond, _) => {
+ ScalarValue::TimestampMicrosecond(Some(i64::$extreme), None)
+ }
+ DataType::Timestamp(TimeUnit::Nanosecond, _) => {
+ ScalarValue::TimestampNanosecond(Some(i64::$extreme), None)
+ }
+ DataType::Interval(IntervalUnit::YearMonth) => {
+ ScalarValue::IntervalYearMonth(Some(i32::$extreme))
+ }
+ DataType::Interval(IntervalUnit::DayTime) => {
+ ScalarValue::IntervalDayTime(Some(i64::$extreme))
+ }
+ DataType::Interval(IntervalUnit::MonthDayNano) => {
+ ScalarValue::IntervalMonthDayNano(Some(i128::$extreme))
+ }
+ _ => unreachable!(),
+ }
+ };
+}
+
+macro_rules! value_transition {
+ ($bound:ident, $direction:expr, $value:expr) => {
+ match $value {
+ UInt8(Some(value)) if value == u8::$bound => UInt8(None),
+ UInt16(Some(value)) if value == u16::$bound => UInt16(None),
+ UInt32(Some(value)) if value == u32::$bound => UInt32(None),
+ UInt64(Some(value)) if value == u64::$bound => UInt64(None),
+ Int8(Some(value)) if value == i8::$bound => Int8(None),
+ Int16(Some(value)) if value == i16::$bound => Int16(None),
+ Int32(Some(value)) if value == i32::$bound => Int32(None),
+ Int64(Some(value)) if value == i64::$bound => Int64(None),
+ Float32(Some(value)) if value == f32::$bound => Float32(None),
+ Float64(Some(value)) if value == f64::$bound => Float64(None),
+ DurationSecond(Some(value)) if value == i64::$bound => DurationSecond(None),
+ DurationMillisecond(Some(value)) if value == i64::$bound => {
+ DurationMillisecond(None)
+ }
+ DurationMicrosecond(Some(value)) if value == i64::$bound => {
+ DurationMicrosecond(None)
+ }
+ DurationNanosecond(Some(value)) if value == i64::$bound => {
+ DurationNanosecond(None)
+ }
+ TimestampSecond(Some(value), tz) if value == i64::$bound => {
+ TimestampSecond(None, tz)
+ }
+ TimestampMillisecond(Some(value), tz) if value == i64::$bound => {
+ TimestampMillisecond(None, tz)
+ }
+ TimestampMicrosecond(Some(value), tz) if value == i64::$bound => {
+ TimestampMicrosecond(None, tz)
+ }
+ TimestampNanosecond(Some(value), tz) if value == i64::$bound => {
+ TimestampNanosecond(None, tz)
+ }
+ IntervalYearMonth(Some(value)) if value == i32::$bound => {
+ IntervalYearMonth(None)
+ }
+ IntervalDayTime(Some(value)) if value == i64::$bound => IntervalDayTime(None),
+ IntervalMonthDayNano(Some(value)) if value == i128::$bound => {
+ IntervalMonthDayNano(None)
+ }
+ _ => next_value_helper::<$direction>($value),
+ }
+ };
+}
+
+/// The `Interval` type represents a closed interval used for computing
+/// reliable bounds for mathematical expressions.
+///
+/// Conventions:
+///
+/// 1. **Closed bounds**: The interval always encompasses its endpoints. We
+/// accommodate operations resulting in open intervals by incrementing or
+/// decrementing the interval endpoint value to its successor/predecessor.
+///
+/// 2. **Unbounded endpoints**: If the `lower` or `upper` bounds are indeterminate,
+/// they are labeled as *unbounded*. This is represented using a `NULL`.
+///
+/// 3. **Overflow handling**: If the `lower` or `upper` endpoints exceed their
+/// limits after any operation, they either become unbounded or they are fixed
+/// to the maximum/minimum value of the datatype, depending on the direction
+/// of the overflowing endpoint, opting for the safer choice.
+///
+/// 4. **Floating-point special cases**:
+/// - `INF` values are converted to `NULL`s while constructing an interval to
+/// ensure consistency, with other data types.
+/// - `NaN` (Not a Number) results are conservatively result in unbounded
+/// endpoints.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Interval {
+ lower: ScalarValue,
+ upper: ScalarValue,
+}
+
+/// This macro handles the `NaN` and `INF` floating point values.
+///
+/// - `NaN` values are always converted to unbounded i.e. `NULL` values.
+/// - For lower bounds:
+/// - A `NEG_INF` value is converted to a `NULL`.
+/// - An `INF` value is conservatively converted to the maximum representable
+/// number for the floating-point type in question. In this case, converting
+/// to `NULL` doesn't make sense as it would be interpreted as a `NEG_INF`.
+/// - For upper bounds:
+/// - An `INF` value is converted to a `NULL`.
+/// - An `NEG_INF` value is conservatively converted to the minimum representable
+/// number for the floating-point type in question. In this case, converting
+/// to `NULL` doesn't make sense as it would be interpreted as an `INF`.
+macro_rules! handle_float_intervals {
+ ($scalar_type:ident, $primitive_type:ident, $lower:expr, $upper:expr) => {{
+ let lower = match $lower {
+ ScalarValue::$scalar_type(Some(l_val))
+ if l_val == $primitive_type::NEG_INFINITY || l_val.is_nan() =>
+ {
+ ScalarValue::$scalar_type(None)
+ }
+ ScalarValue::$scalar_type(Some(l_val))
+ if l_val == $primitive_type::INFINITY =>
+ {
+ ScalarValue::$scalar_type(Some($primitive_type::MAX))
+ }
+ value @ ScalarValue::$scalar_type(Some(_)) => value,
+ _ => ScalarValue::$scalar_type(None),
+ };
+
+ let upper = match $upper {
+ ScalarValue::$scalar_type(Some(r_val))
+ if r_val == $primitive_type::INFINITY || r_val.is_nan() =>
+ {
+ ScalarValue::$scalar_type(None)
+ }
+ ScalarValue::$scalar_type(Some(r_val))
+ if r_val == $primitive_type::NEG_INFINITY =>
+ {
+ ScalarValue::$scalar_type(Some($primitive_type::MIN))
+ }
+ value @ ScalarValue::$scalar_type(Some(_)) => value,
+ _ => ScalarValue::$scalar_type(None),
+ };
+
+ Interval { lower, upper }
+ }};
+}
+
+/// Ordering floating-point numbers according to their binary representations
+/// contradicts with their natural ordering. Floating-point number ordering
+/// after unsigned integer transmutation looks like:
+///
+/// ```text
+/// 0, 1, 2, 3, ..., MAX, -0, -1, -2, ..., -MAX
+/// ```
+///
+/// This macro applies a one-to-one map that fixes the ordering above.
+macro_rules! map_floating_point_order {
+ ($value:expr, $ty:ty) => {{
+ let num_bits = std::mem::size_of::<$ty>() * 8;
+ let sign_bit = 1 << (num_bits - 1);
+ if $value & sign_bit == sign_bit {
+ // Negative numbers:
+ !$value
+ } else {
+ // Positive numbers:
+ $value | sign_bit
+ }
+ }};
+}
+
+impl Interval {
+ /// Attempts to create a new `Interval` from the given lower and upper bounds.
+ ///
+ /// # Notes
+ ///
+ /// This constructor creates intervals in a "canonical" form where:
+ /// - **Boolean intervals**:
+ /// - Unboundedness (`NULL`) for boolean endpoints is converted to `false`
+ /// for lower and `true` for upper bounds.
+ /// - **Floating-point intervals**:
+ /// - Floating-point endpoints with `NaN`, `INF`, or `NEG_INF` are converted
+ /// to `NULL`s.
+ pub fn try_new(lower: ScalarValue, upper: ScalarValue) -> Result<Self> {
+ if lower.data_type() != upper.data_type() {
+ return internal_err!("Endpoints of an Interval should have the same type");
+ }
+
+ let interval = Self::new(lower, upper);
+
+ if interval.lower.is_null()
+ || interval.upper.is_null()
+ || interval.lower <= interval.upper
+ {
+ Ok(interval)
+ } else {
+ internal_err!(
+ "Interval's lower bound {} is greater than the upper bound {}",
+ interval.lower,
+ interval.upper
+ )
+ }
+ }
+
+ /// Only for internal usage. Responsible for standardizing booleans and
+ /// floating-point values, as well as fixing NaNs. It doesn't validate
+ /// the given bounds for ordering, or verify that they have the same data
+ /// type. For its user-facing counterpart and more details, see
+ /// [`Interval::try_new`].
+ fn new(lower: ScalarValue, upper: ScalarValue) -> Self {
+ if let ScalarValue::Boolean(lower_bool) = lower {
+ let ScalarValue::Boolean(upper_bool) = upper else {
+ // We are sure that upper and lower bounds have the same type.
+ unreachable!();
+ };
+ // Standardize boolean interval endpoints:
+ Self {
+ lower: ScalarValue::Boolean(Some(lower_bool.unwrap_or(false))),
+ upper: ScalarValue::Boolean(Some(upper_bool.unwrap_or(true))),
+ }
+ }
+ // Standardize floating-point endpoints:
+ else if lower.data_type() == DataType::Float32 {
+ handle_float_intervals!(Float32, f32, lower, upper)
+ } else if lower.data_type() == DataType::Float64 {
+ handle_float_intervals!(Float64, f64, lower, upper)
+ } else {
+ // Other data types do not require standardization:
+ Self { lower, upper }
+ }
+ }
+
+ /// Convenience function to create a new `Interval` from the given (optional)
+ /// bounds, for use in tests only. Absence of either endpoint indicates
+ /// unboundedness on that side. See [`Interval::try_new`] for more information.
+ pub fn make<T>(lower: Option<T>, upper: Option<T>) -> Result<Self>
+ where
+ ScalarValue: From<Option<T>>,
+ {
+ Self::try_new(ScalarValue::from(lower), ScalarValue::from(upper))
+ }
+
+ /// Creates an unbounded interval from both sides if the datatype supported.
+ pub fn make_unbounded(data_type: &DataType) -> Result<Self> {
+ let unbounded_endpoint = ScalarValue::try_from(data_type)?;
+ Ok(Self::new(unbounded_endpoint.clone(), unbounded_endpoint))
+ }
+
+ /// Returns a reference to the lower bound.
+ pub fn lower(&self) -> &ScalarValue {
+ &self.lower
+ }
+
+ /// Returns a reference to the upper bound.
+ pub fn upper(&self) -> &ScalarValue {
+ &self.upper
+ }
+
+ /// Converts this `Interval` into its boundary scalar values. It's useful
+ /// when you need to work with the individual bounds directly.
+ pub fn into_bounds(self) -> (ScalarValue, ScalarValue) {
+ (self.lower, self.upper)
+ }
+
+ /// This function returns the data type of this interval.
+ pub fn data_type(&self) -> DataType {
+ let lower_type = self.lower.data_type();
+ let upper_type = self.upper.data_type();
+
+ // There must be no way to create an interval whose endpoints have
+ // different types.
+ assert!(
+ lower_type == upper_type,
+ "Interval bounds have different types: {lower_type} != {upper_type}"
+ );
+ lower_type
+ }
+
+ /// Casts this interval to `data_type` using `cast_options`.
+ pub fn cast_to(
+ &self,
+ data_type: &DataType,
+ cast_options: &CastOptions,
+ ) -> Result<Self> {
+ Self::try_new(
+ cast_scalar_value(&self.lower, data_type, cast_options)?,
+ cast_scalar_value(&self.upper, data_type, cast_options)?,
+ )
+ }
+
+ pub const CERTAINLY_FALSE: Self = Self {
+ lower: ScalarValue::Boolean(Some(false)),
+ upper: ScalarValue::Boolean(Some(false)),
+ };
+
+ pub const UNCERTAIN: Self = Self {
+ lower: ScalarValue::Boolean(Some(false)),
+ upper: ScalarValue::Boolean(Some(true)),
+ };
+
+ pub const CERTAINLY_TRUE: Self = Self {
+ lower: ScalarValue::Boolean(Some(true)),
+ upper: ScalarValue::Boolean(Some(true)),
+ };
+
+ /// Decide if this interval is certainly greater than, possibly greater than,
+ /// or can't be greater than `other` by returning `[true, true]`,
+ /// `[false, true]` or `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub(crate) fn gt<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ if self.data_type().ne(&rhs.data_type()) {
+ internal_err!(
+ "Only intervals with the same data type are comparable, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ )
+ } else if !(self.upper.is_null() || rhs.lower.is_null())
+ && self.upper <= rhs.lower
+ {
+ // Values in this interval are certainly less than or equal to
+ // those in the given interval.
+ Ok(Self::CERTAINLY_FALSE)
+ } else if !(self.lower.is_null() || rhs.upper.is_null())
+ && (self.lower > rhs.upper)
+ {
+ // Values in this interval are certainly greater than those in the
+ // given interval.
+ Ok(Self::CERTAINLY_TRUE)
+ } else {
+ // All outcomes are possible.
+ Ok(Self::UNCERTAIN)
+ }
+ }
+
+ /// Decide if this interval is certainly greater than or equal to, possibly
+ /// greater than or equal to, or can't be greater than or equal to `other`
+ /// by returning `[true, true]`, `[false, true]` or `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub(crate) fn gt_eq<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ if self.data_type().ne(&rhs.data_type()) {
+ internal_err!(
+ "Only intervals with the same data type are comparable, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ )
+ } else if !(self.lower.is_null() || rhs.upper.is_null())
+ && self.lower >= rhs.upper
+ {
+ // Values in this interval are certainly greater than or equal to
+ // those in the given interval.
+ Ok(Self::CERTAINLY_TRUE)
+ } else if !(self.upper.is_null() || rhs.lower.is_null())
+ && (self.upper < rhs.lower)
+ {
+ // Values in this interval are certainly less than those in the
+ // given interval.
+ Ok(Self::CERTAINLY_FALSE)
+ } else {
+ // All outcomes are possible.
+ Ok(Self::UNCERTAIN)
+ }
+ }
+
+ /// Decide if this interval is certainly less than, possibly less than, or
+ /// can't be less than `other` by returning `[true, true]`, `[false, true]`
+ /// or `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub(crate) fn lt<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ other.borrow().gt(self)
+ }
+
+ /// Decide if this interval is certainly less than or equal to, possibly
+ /// less than or equal to, or can't be less than or equal to `other` by
+ /// returning `[true, true]`, `[false, true]` or `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub(crate) fn lt_eq<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ other.borrow().gt_eq(self)
+ }
+
+ /// Decide if this interval is certainly equal to, possibly equal to, or
+ /// can't be equal to `other` by returning `[true, true]`, `[false, true]`
+ /// or `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub(crate) fn equal<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ if get_result_type(&self.data_type(), &Operator::Eq, &rhs.data_type()).is_err() {
+ internal_err!(
+ "Interval data types must be compatible for equality checks, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ )
+ } else if !self.lower.is_null()
+ && (self.lower == self.upper)
+ && (rhs.lower == rhs.upper)
+ && (self.lower == rhs.lower)
+ {
+ Ok(Self::CERTAINLY_TRUE)
+ } else if self.intersect(rhs)?.is_none() {
+ Ok(Self::CERTAINLY_FALSE)
+ } else {
+ Ok(Self::UNCERTAIN)
+ }
+ }
+
+ /// Compute the logical conjunction of this (boolean) interval with the
+ /// given boolean interval.
+ pub(crate) fn and<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ match (&self.lower, &self.upper, &rhs.lower, &rhs.upper) {
+ (
+ &ScalarValue::Boolean(Some(self_lower)),
+ &ScalarValue::Boolean(Some(self_upper)),
+ &ScalarValue::Boolean(Some(other_lower)),
+ &ScalarValue::Boolean(Some(other_upper)),
+ ) => {
+ let lower = self_lower && other_lower;
+ let upper = self_upper && other_upper;
+
+ Ok(Self {
+ lower: ScalarValue::Boolean(Some(lower)),
+ upper: ScalarValue::Boolean(Some(upper)),
+ })
+ }
+ _ => internal_err!("Incompatible data types for logical conjunction"),
+ }
+ }
+
+ /// Compute the logical negation of this (boolean) interval.
+ pub(crate) fn not(&self) -> Result<Self> {
+ if self.data_type().ne(&DataType::Boolean) {
+ internal_err!("Cannot apply logical negation to a non-boolean interval")
+ } else if self == &Self::CERTAINLY_TRUE {
+ Ok(Self::CERTAINLY_FALSE)
+ } else if self == &Self::CERTAINLY_FALSE {
+ Ok(Self::CERTAINLY_TRUE)
+ } else {
+ Ok(Self::UNCERTAIN)
+ }
+ }
+
+ /// Compute the intersection of this interval with the given interval.
+ /// If the intersection is empty, return `None`.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub fn intersect<T: Borrow<Self>>(&self, other: T) -> Result<Option<Self>> {
+ let rhs = other.borrow();
+ if self.data_type().ne(&rhs.data_type()) {
+ return internal_err!(
+ "Only intervals with the same data type are intersectable, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ );
+ };
+
+ // If it is evident that the result is an empty interval, short-circuit
+ // and directly return `None`.
+ if (!(self.lower.is_null() || rhs.upper.is_null()) && self.lower > rhs.upper)
+ || (!(self.upper.is_null() || rhs.lower.is_null()) && self.upper < rhs.lower)
+ {
+ return Ok(None);
+ }
+
+ let lower = max_of_bounds(&self.lower, &rhs.lower);
+ let upper = min_of_bounds(&self.upper, &rhs.upper);
+
+ // New lower and upper bounds must always construct a valid interval.
+ assert!(
+ (lower.is_null() || upper.is_null() || (lower <= upper)),
+ "The intersection of two intervals can not be an invalid interval"
+ );
+
+ Ok(Some(Self { lower, upper }))
+ }
+
+ /// Decide if this interval certainly contains, possibly contains, or can't
+ /// contain a [`ScalarValue`] (`other`) by returning `[true, true]`,
+ /// `[false, true]` or `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub fn contains_value<T: Borrow<ScalarValue>>(&self, other: T) -> Result<bool> {
+ let rhs = other.borrow();
+ if self.data_type().ne(&rhs.data_type()) {
+ return internal_err!(
+ "Data types must be compatible for containment checks, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ );
+ }
+
+ // We only check the upper bound for a `None` value because `None`
+ // values are less than `Some` values according to Rust.
+ Ok(&self.lower <= rhs && (self.upper.is_null() || rhs <= &self.upper))
+ }
+
+ /// Decide if this interval is a superset of, overlaps with, or
+ /// disjoint with `other` by returning `[true, true]`, `[false, true]` or
+ /// `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub fn contains<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ if self.data_type().ne(&rhs.data_type()) {
+ return internal_err!(
+ "Interval data types must match for containment checks, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ );
+ };
+
+ match self.intersect(rhs)? {
+ Some(intersection) => {
+ if &intersection == rhs {
+ Ok(Self::CERTAINLY_TRUE)
+ } else {
+ Ok(Self::UNCERTAIN)
+ }
+ }
+ None => Ok(Self::CERTAINLY_FALSE),
+ }
+ }
+
+ /// Add the given interval (`other`) to this interval. Say we have intervals
+ /// `[a1, b1]` and `[a2, b2]`, then their sum is `[a1 + a2, b1 + b2]`. Note
+ /// that this represents all possible values the sum can take if one can
+ /// choose single values arbitrarily from each of the operands.
+ pub fn add<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ let dt = get_result_type(&self.data_type(), &Operator::Plus, &rhs.data_type())?;
+
+ Ok(Self::new(
+ add_bounds::<false>(&dt, &self.lower, &rhs.lower),
+ add_bounds::<true>(&dt, &self.upper, &rhs.upper),
+ ))
+ }
+
+ /// Subtract the given interval (`other`) from this interval. Say we have
+ /// intervals `[a1, b1]` and `[a2, b2]`, then their difference is
+ /// `[a1 - b2, b1 - a2]`. Note that this represents all possible values the
+ /// difference can take if one can choose single values arbitrarily from
+ /// each of the operands.
+ pub fn sub<T: Borrow<Interval>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ let dt = get_result_type(&self.data_type(), &Operator::Minus, &rhs.data_type())?;
+
+ Ok(Self::new(
+ sub_bounds::<false>(&dt, &self.lower, &rhs.upper),
+ sub_bounds::<true>(&dt, &self.upper, &rhs.lower),
+ ))
+ }
+
+ /// Multiply the given interval (`other`) with this interval. Say we have
+ /// intervals `[a1, b1]` and `[a2, b2]`, then their product is `[min(a1 * a2,
+ /// a1 * b2, b1 * a2, b1 * b2), max(a1 * a2, a1 * b2, b1 * a2, b1 * b2)]`.
+ /// Note that this represents all possible values the product can take if
+ /// one can choose single values arbitrarily from each of the operands.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub fn mul<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ let dt = if self.data_type().eq(&rhs.data_type()) {
+ self.data_type()
+ } else {
+ return internal_err!(
+ "Intervals must have the same data type for multiplication, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ );
+ };
+
+ let zero = ScalarValue::new_zero(&dt)?;
+
+ let result = match (
+ self.contains_value(&zero)?,
+ rhs.contains_value(&zero)?,
+ dt.is_unsigned_integer(),
+ ) {
+ (true, true, false) => mul_helper_multi_zero_inclusive(&dt, self, rhs),
+ (true, false, false) => {
+ mul_helper_single_zero_inclusive(&dt, self, rhs, zero)
+ }
+ (false, true, false) => {
+ mul_helper_single_zero_inclusive(&dt, rhs, self, zero)
+ }
+ _ => mul_helper_zero_exclusive(&dt, self, rhs, zero),
+ };
+ Ok(result)
+ }
+
+ /// Divide this interval by the given interval (`other`). Say we have intervals
+ /// `[a1, b1]` and `[a2, b2]`, then their division is `[a1, b1] * [1 / b2, 1 / a2]`
+ /// if `0 ∉ [a2, b2]` and `[NEG_INF, INF]` otherwise. Note that this represents
+ /// all possible values the quotient can take if one can choose single values
+ /// arbitrarily from each of the operands.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ ///
+ /// **TODO**: Once interval sets are supported, cases where the divisor contains
+ /// zero should result in an interval set, not the universal set.
+ pub fn div<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ let dt = if self.data_type().eq(&rhs.data_type()) {
+ self.data_type()
+ } else {
+ return internal_err!(
+ "Intervals must have the same data type for division, lhs:{}, rhs:{}",
+ self.data_type(),
+ rhs.data_type()
+ );
+ };
+
+ let zero = ScalarValue::new_zero(&dt)?;
+ // We want 0 to be approachable from both negative and positive sides.
+ let zero_point = match &dt {
+ DataType::Float32 | DataType::Float64 => Self::new(zero.clone(), zero),
+ _ => Self::new(prev_value(zero.clone()), next_value(zero.clone())),
+ };
+
+ // Exit early with an unbounded interval if zero is strictly inside the
+ // right hand side:
+ if rhs.contains(&zero_point)? == Self::CERTAINLY_TRUE && !dt.is_unsigned_integer()
+ {
+ Self::make_unbounded(&dt)
+ }
+ // At this point, we know that only one endpoint of the right hand side
+ // can be zero.
+ else if self.contains(&zero_point)? == Self::CERTAINLY_TRUE
+ && !dt.is_unsigned_integer()
+ {
+ Ok(div_helper_lhs_zero_inclusive(&dt, self, rhs, &zero_point))
+ } else {
+ Ok(div_helper_zero_exclusive(&dt, self, rhs, &zero_point))
+ }
+ }
+
+ /// Returns the cardinality of this interval, which is the number of all
+ /// distinct points inside it. This function returns `None` if:
+ /// - The interval is unbounded from either side, or
+ /// - Cardinality calculations for the datatype in question is not
+ /// implemented yet, or
+ /// - An overflow occurs during the calculation: This case can only arise
+ /// when the calculated cardinality does not fit in an `u64`.
+ pub fn cardinality(&self) -> Option<u64> {
+ let data_type = self.data_type();
+ if data_type.is_integer() {
+ self.upper.distance(&self.lower).map(|diff| diff as u64)
+ } else if data_type.is_floating() {
+ // Negative numbers are sorted in the reverse order. To
+ // always have a positive difference after the subtraction,
+ // we perform following transformation:
+ match (&self.lower, &self.upper) {
+ // Exploit IEEE 754 ordering properties to calculate the correct
+ // cardinality in all cases (including subnormals).
+ (
+ ScalarValue::Float32(Some(lower)),
+ ScalarValue::Float32(Some(upper)),
+ ) => {
+ let lower_bits = map_floating_point_order!(lower.to_bits(), u32);
+ let upper_bits = map_floating_point_order!(upper.to_bits(), u32);
+ Some((upper_bits - lower_bits) as u64)
+ }
+ (
+ ScalarValue::Float64(Some(lower)),
+ ScalarValue::Float64(Some(upper)),
+ ) => {
+ let lower_bits = map_floating_point_order!(lower.to_bits(), u64);
+ let upper_bits = map_floating_point_order!(upper.to_bits(), u64);
+ let count = upper_bits - lower_bits;
+ (count != u64::MAX).then_some(count)
+ }
+ _ => None,
+ }
+ } else {
+ // Cardinality calculations are not implemented for this data type yet:
+ None
+ }
+ .map(|result| result + 1)
+ }
+}
+
+impl Display for Interval {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ write!(f, "[{}, {}]", self.lower, self.upper)
+ }
+}
+
+/// Applies the given binary operator the `lhs` and `rhs` arguments.
+pub fn apply_operator(op: &Operator, lhs: &Interval, rhs: &Interval) -> Result<Interval> {
+ match *op {
+ Operator::Eq => lhs.equal(rhs),
+ Operator::NotEq => lhs.equal(rhs)?.not(),
+ Operator::Gt => lhs.gt(rhs),
+ Operator::GtEq => lhs.gt_eq(rhs),
+ Operator::Lt => lhs.lt(rhs),
+ Operator::LtEq => lhs.lt_eq(rhs),
+ Operator::And => lhs.and(rhs),
+ Operator::Plus => lhs.add(rhs),
+ Operator::Minus => lhs.sub(rhs),
+ Operator::Multiply => lhs.mul(rhs),
+ Operator::Divide => lhs.div(rhs),
+ _ => internal_err!("Interval arithmetic does not support the operator {op}"),
+ }
+}
+
+/// Helper function used for adding the end-point values of intervals.
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`, and may
+/// return non-standardized interval bounds. Therefore, it should be used
+/// with caution. Currently, it is used in contexts where the `DataType`
+/// (`dt`) is validated prior to calling this function, and the following
+/// interval creation is standardized with `Interval::new`.
+fn add_bounds<const UPPER: bool>(
+ dt: &DataType,
+ lhs: &ScalarValue,
+ rhs: &ScalarValue,
+) -> ScalarValue {
+ if lhs.is_null() || rhs.is_null() {
+ return ScalarValue::try_from(dt).unwrap();
+ }
+
+ match dt {
+ DataType::Float64 | DataType::Float32 => {
+ alter_fp_rounding_mode::<UPPER, _>(lhs, rhs, |lhs, rhs| lhs.add_checked(rhs))
+ }
+ _ => lhs.add_checked(rhs),
+ }
+ .unwrap_or_else(|_| handle_overflow::<UPPER>(dt, Operator::Plus, lhs, rhs))
+}
+
+/// Helper function used for subtracting the end-point values of intervals.
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`, and may
+/// return non-standardized interval bounds. Therefore, it should be used
+/// with caution. Currently, it is used in contexts where the `DataType`
+/// (`dt`) is validated prior to calling this function, and the following
+/// interval creation is standardized with `Interval::new`.
+fn sub_bounds<const UPPER: bool>(
+ dt: &DataType,
+ lhs: &ScalarValue,
+ rhs: &ScalarValue,
+) -> ScalarValue {
+ if lhs.is_null() || rhs.is_null() {
+ return ScalarValue::try_from(dt).unwrap();
+ }
+
+ match dt {
+ DataType::Float64 | DataType::Float32 => {
+ alter_fp_rounding_mode::<UPPER, _>(lhs, rhs, |lhs, rhs| lhs.sub_checked(rhs))
+ }
+ _ => lhs.sub_checked(rhs),
+ }
+ .unwrap_or_else(|_| handle_overflow::<UPPER>(dt, Operator::Minus, lhs, rhs))
+}
+
+/// Helper function used for multiplying the end-point values of intervals.
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`, and may
+/// return non-standardized interval bounds. Therefore, it should be used
+/// with caution. Currently, it is used in contexts where the `DataType`
+/// (`dt`) is validated prior to calling this function, and the following
+/// interval creation is standardized with `Interval::new`.
+fn mul_bounds<const UPPER: bool>(
+ dt: &DataType,
+ lhs: &ScalarValue,
+ rhs: &ScalarValue,
+) -> ScalarValue {
+ if lhs.is_null() || rhs.is_null() {
+ return ScalarValue::try_from(dt).unwrap();
+ }
+
+ match dt {
+ DataType::Float64 | DataType::Float32 => {
+ alter_fp_rounding_mode::<UPPER, _>(lhs, rhs, |lhs, rhs| lhs.mul_checked(rhs))
+ }
+ _ => lhs.mul_checked(rhs),
+ }
+ .unwrap_or_else(|_| handle_overflow::<UPPER>(dt, Operator::Multiply, lhs, rhs))
+}
+
+/// Helper function used for dividing the end-point values of intervals.
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`, and may
+/// return non-standardized interval bounds. Therefore, it should be used
+/// with caution. Currently, it is used in contexts where the `DataType`
+/// (`dt`) is validated prior to calling this function, and the following
+/// interval creation is standardized with `Interval::new`.
+fn div_bounds<const UPPER: bool>(
+ dt: &DataType,
+ lhs: &ScalarValue,
+ rhs: &ScalarValue,
+) -> ScalarValue {
+ let zero = ScalarValue::new_zero(dt).unwrap();
+
+ if (lhs.is_null() || rhs.eq(&zero)) || (dt.is_unsigned_integer() && rhs.is_null()) {
+ return ScalarValue::try_from(dt).unwrap();
+ } else if rhs.is_null() {
+ return zero;
+ }
+
+ match dt {
+ DataType::Float64 | DataType::Float32 => {
+ alter_fp_rounding_mode::<UPPER, _>(lhs, rhs, |lhs, rhs| lhs.div(rhs))
+ }
+ _ => lhs.div(rhs),
+ }
+ .unwrap_or_else(|_| handle_overflow::<UPPER>(dt, Operator::Divide, lhs, rhs))
+}
+
+/// This function handles cases where an operation results in an overflow. Such
+/// results are converted to an *unbounded endpoint* if:
+/// - We are calculating an upper bound and we have a positive overflow.
+/// - We are calculating a lower bound and we have a negative overflow.
+/// Otherwise; the function sets the endpoint as:
+/// - The minimum representable number with the given datatype (`dt`) if
+/// we are calculating an upper bound and we have a negative overflow.
+/// - The maximum representable number with the given datatype (`dt`) if
+/// we are calculating a lower bound and we have a positive overflow.
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`, and may
+/// return non-standardized interval bounds. Therefore, it should be used
+/// with caution. Currently, it is used in contexts where the `DataType`
+/// (`dt`) is validated prior to calling this function, `op` is supported by
+/// interval library, and the following interval creation is standardized with
+/// `Interval::new`.
+fn handle_overflow<const UPPER: bool>(
+ dt: &DataType,
+ op: Operator,
+ lhs: &ScalarValue,
+ rhs: &ScalarValue,
+) -> ScalarValue {
+ let zero = ScalarValue::new_zero(dt).unwrap();
+ let positive_sign = match op {
+ Operator::Multiply | Operator::Divide => {
+ lhs.lt(&zero) && rhs.lt(&zero) || lhs.gt(&zero) && rhs.gt(&zero)
+ }
+ Operator::Plus => lhs.ge(&zero),
+ Operator::Minus => lhs.ge(rhs),
+ _ => {
+ unreachable!()
+ }
+ };
+ match (UPPER, positive_sign) {
+ (true, true) | (false, false) => ScalarValue::try_from(dt).unwrap(),
+ (true, false) => {
+ get_extreme_value!(MIN, dt)
+ }
+ (false, true) => {
+ get_extreme_value!(MAX, dt)
+ }
+ }
+}
+
+// This function should remain private since it may corrupt the an interval if
+// used without caution.
+fn next_value(value: ScalarValue) -> ScalarValue {
+ use ScalarValue::*;
+ value_transition!(MAX, true, value)
+}
+
+// This function should remain private since it may corrupt the an interval if
+// used without caution.
+fn prev_value(value: ScalarValue) -> ScalarValue {
+ use ScalarValue::*;
+ value_transition!(MIN, false, value)
+}
+
+trait OneTrait: Sized + std::ops::Add + std::ops::Sub {
+ fn one() -> Self;
+}
+macro_rules! impl_OneTrait{
+ ($($m:ty),*) => {$( impl OneTrait for $m { fn one() -> Self { 1 as $m } })*}
+}
+impl_OneTrait! {u8, u16, u32, u64, i8, i16, i32, i64, i128}
+
+/// This function either increments or decrements its argument, depending on
+/// the `INC` value (where a `true` value corresponds to the increment).
+fn increment_decrement<const INC: bool, T: OneTrait + SubAssign + AddAssign>(
+ mut value: T,
+) -> T {
+ if INC {
+ value.add_assign(T::one());
+ } else {
+ value.sub_assign(T::one());
+ }
+ value
+}
+
+/// This function returns the next/previous value depending on the `INC` value.
+/// If `true`, it returns the next value; otherwise it returns the previous value.
+fn next_value_helper<const INC: bool>(value: ScalarValue) -> ScalarValue {
+ use ScalarValue::*;
+ match value {
+ // f32/f64::NEG_INF/INF and f32/f64::NaN values should not emerge at this point.
+ Float32(Some(val)) => {
+ assert!(val.is_finite(), "Non-standardized floating point usage");
+ Float32(Some(if INC { next_up(val) } else { next_down(val) }))
+ }
+ Float64(Some(val)) => {
+ assert!(val.is_finite(), "Non-standardized floating point usage");
+ Float64(Some(if INC { next_up(val) } else { next_down(val) }))
+ }
+ Int8(Some(val)) => Int8(Some(increment_decrement::<INC, i8>(val))),
+ Int16(Some(val)) => Int16(Some(increment_decrement::<INC, i16>(val))),
+ Int32(Some(val)) => Int32(Some(increment_decrement::<INC, i32>(val))),
+ Int64(Some(val)) => Int64(Some(increment_decrement::<INC, i64>(val))),
+ UInt8(Some(val)) => UInt8(Some(increment_decrement::<INC, u8>(val))),
+ UInt16(Some(val)) => UInt16(Some(increment_decrement::<INC, u16>(val))),
+ UInt32(Some(val)) => UInt32(Some(increment_decrement::<INC, u32>(val))),
+ UInt64(Some(val)) => UInt64(Some(increment_decrement::<INC, u64>(val))),
+ DurationSecond(Some(val)) => {
+ DurationSecond(Some(increment_decrement::<INC, i64>(val)))
+ }
+ DurationMillisecond(Some(val)) => {
+ DurationMillisecond(Some(increment_decrement::<INC, i64>(val)))
+ }
+ DurationMicrosecond(Some(val)) => {
+ DurationMicrosecond(Some(increment_decrement::<INC, i64>(val)))
+ }
+ DurationNanosecond(Some(val)) => {
+ DurationNanosecond(Some(increment_decrement::<INC, i64>(val)))
+ }
+ TimestampSecond(Some(val), tz) => {
+ TimestampSecond(Some(increment_decrement::<INC, i64>(val)), tz)
+ }
+ TimestampMillisecond(Some(val), tz) => {
+ TimestampMillisecond(Some(increment_decrement::<INC, i64>(val)), tz)
+ }
+ TimestampMicrosecond(Some(val), tz) => {
+ TimestampMicrosecond(Some(increment_decrement::<INC, i64>(val)), tz)
+ }
+ TimestampNanosecond(Some(val), tz) => {
+ TimestampNanosecond(Some(increment_decrement::<INC, i64>(val)), tz)
+ }
+ IntervalYearMonth(Some(val)) => {
+ IntervalYearMonth(Some(increment_decrement::<INC, i32>(val)))
+ }
+ IntervalDayTime(Some(val)) => {
+ IntervalDayTime(Some(increment_decrement::<INC, i64>(val)))
+ }
+ IntervalMonthDayNano(Some(val)) => {
+ IntervalMonthDayNano(Some(increment_decrement::<INC, i128>(val)))
+ }
+ _ => value, // Unbounded values return without change.
+ }
+}
+
+/// Returns the greater of the given interval bounds. Assumes that a `NULL`
+/// value represents `NEG_INF`.
+fn max_of_bounds(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {
+ if !first.is_null() && (second.is_null() || first >= second) {
+ first.clone()
+ } else {
+ second.clone()
+ }
+}
+
+/// Returns the lesser of the given interval bounds. Assumes that a `NULL`
+/// value represents `INF`.
+fn min_of_bounds(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {
+ if !first.is_null() && (second.is_null() || first <= second) {
+ first.clone()
+ } else {
+ second.clone()
+ }
+}
+
+/// This function updates the given intervals by enforcing (i.e. propagating)
+/// the inequality `left > right` (or the `left >= right` inequality, if `strict`
+/// is `true`).
+///
+/// Returns a `Result` wrapping an `Option` containing the tuple of resulting
+/// intervals. If the comparison is infeasible, returns `None`.
+///
+/// Example usage:
+/// ```
+/// use datafusion_common::DataFusionError;
+/// use datafusion_expr::interval_arithmetic::{satisfy_greater, Interval};
+///
+/// let left = Interval::make(Some(-1000.0_f32), Some(1000.0_f32))?;
+/// let right = Interval::make(Some(500.0_f32), Some(2000.0_f32))?;
+/// let strict = false;
+/// assert_eq!(
+/// satisfy_greater(&left, &right, strict)?,
+/// Some((
+/// Interval::make(Some(500.0_f32), Some(1000.0_f32))?,
+/// Interval::make(Some(500.0_f32), Some(1000.0_f32))?
+/// ))
+/// );
+/// Ok::<(), DataFusionError>(())
+/// ```
+///
+/// NOTE: This function only works with intervals of the same data type.
+/// Attempting to compare intervals of different data types will lead
+/// to an error.
+pub fn satisfy_greater(
+ left: &Interval,
+ right: &Interval,
+ strict: bool,
+) -> Result<Option<(Interval, Interval)>> {
+ if left.data_type().ne(&right.data_type()) {
+ return internal_err!(
+ "Intervals must have the same data type, lhs:{}, rhs:{}",
+ left.data_type(),
+ right.data_type()
+ );
+ }
+
+ if !left.upper.is_null() && left.upper <= right.lower {
+ if !strict && left.upper == right.lower {
+ // Singleton intervals:
+ return Ok(Some((
+ Interval::new(left.upper.clone(), left.upper.clone()),
+ Interval::new(left.upper.clone(), left.upper.clone()),
+ )));
+ } else {
+ // Left-hand side: <--======----0------------>
+ // Right-hand side: <------------0--======---->
+ // No intersection, infeasible to propagate:
+ return Ok(None);
+ }
+ }
+
+ // Only the lower bound of left hand side and the upper bound of the right
+ // hand side can change after propagating the greater-than operation.
+ let new_left_lower = if left.lower.is_null() || left.lower <= right.lower {
+ if strict {
+ next_value(right.lower.clone())
+ } else {
+ right.lower.clone()
+ }
+ } else {
+ left.lower.clone()
+ };
+ // Below code is asymmetric relative to the above if statement, because
+ // `None` compares less than `Some` in Rust.
+ let new_right_upper = if right.upper.is_null()
+ || (!left.upper.is_null() && left.upper <= right.upper)
+ {
+ if strict {
+ prev_value(left.upper.clone())
+ } else {
+ left.upper.clone()
+ }
+ } else {
+ right.upper.clone()
+ };
+
+ Ok(Some((
+ Interval::new(new_left_lower, left.upper.clone()),
+ Interval::new(right.lower.clone(), new_right_upper),
+ )))
+}
+
+/// Multiplies two intervals that both contain zero.
+///
+/// This function takes in two intervals (`lhs` and `rhs`) as arguments and
+/// returns their product (whose data type is known to be `dt`). It is
+/// specifically designed to handle intervals that contain zero within their
+/// ranges. Returns an error if the multiplication of bounds fails.
+///
+/// ```text
+/// Left-hand side: <-------=====0=====------->
+/// Right-hand side: <-------=====0=====------->
+/// ```
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`. Therefore,
+/// it should be used with caution. Currently, it is used in contexts where the
+/// `DataType` (`dt`) is validated prior to calling this function.
+fn mul_helper_multi_zero_inclusive(
+ dt: &DataType,
+ lhs: &Interval,
+ rhs: &Interval,
+) -> Interval {
+ if lhs.lower.is_null()
+ || lhs.upper.is_null()
+ || rhs.lower.is_null()
+ || rhs.upper.is_null()
+ {
+ return Interval::make_unbounded(dt).unwrap();
+ }
+ // Since unbounded cases are handled above, we can safely
+ // use the utility functions here to eliminate code duplication.
+ let lower = min_of_bounds(
+ &mul_bounds::<false>(dt, &lhs.lower, &rhs.upper),
+ &mul_bounds::<false>(dt, &rhs.lower, &lhs.upper),
+ );
+ let upper = max_of_bounds(
+ &mul_bounds::<true>(dt, &lhs.upper, &rhs.upper),
+ &mul_bounds::<true>(dt, &lhs.lower, &rhs.lower),
+ );
+ // There is no possibility to create an invalid interval.
+ Interval::new(lower, upper)
+}
+
+/// Multiplies two intervals when only left-hand side interval contains zero.
+///
+/// This function takes in two intervals (`lhs` and `rhs`) as arguments and
+/// returns their product (whose data type is known to be `dt`). This function
+/// serves as a subroutine that handles the specific case when only `lhs` contains
+/// zero within its range. The interval not containing zero, i.e. rhs, can lie
+/// on either side of zero. Returns an error if the multiplication of bounds fails.
+///
+/// ``` text
+/// Left-hand side: <-------=====0=====------->
+/// Right-hand side: <--======----0------------>
+///
+/// or
+///
+/// Left-hand side: <-------=====0=====------->
+/// Right-hand side: <------------0--======---->
+/// ```
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`. Therefore,
+/// it should be used with caution. Currently, it is used in contexts where the
+/// `DataType` (`dt`) is validated prior to calling this function.
+fn mul_helper_single_zero_inclusive(
+ dt: &DataType,
+ lhs: &Interval,
+ rhs: &Interval,
+ zero: ScalarValue,
+) -> Interval {
+ // With the following interval bounds, there is no possibility to create an invalid interval.
+ if rhs.upper <= zero && !rhs.upper.is_null() {
+ // <-------=====0=====------->
+ // <--======----0------------>
+ let lower = mul_bounds::<false>(dt, &lhs.upper, &rhs.lower);
+ let upper = mul_bounds::<true>(dt, &lhs.lower, &rhs.lower);
+ Interval::new(lower, upper)
+ } else {
+ // <-------=====0=====------->
+ // <------------0--======---->
+ let lower = mul_bounds::<false>(dt, &lhs.lower, &rhs.upper);
+ let upper = mul_bounds::<true>(dt, &lhs.upper, &rhs.upper);
+ Interval::new(lower, upper)
+ }
+}
+
+/// Multiplies two intervals when neither of them contains zero.
+///
+/// This function takes in two intervals (`lhs` and `rhs`) as arguments and
+/// returns their product (whose data type is known to be `dt`). It is
+/// specifically designed to handle intervals that do not contain zero within
+/// their ranges. Returns an error if the multiplication of bounds fails.
+///
+/// ``` text
+/// Left-hand side: <--======----0------------>
+/// Right-hand side: <--======----0------------>
+///
+/// or
+///
+/// Left-hand side: <--======----0------------>
+/// Right-hand side: <------------0--======---->
+///
+/// or
+///
+/// Left-hand side: <------------0--======---->
+/// Right-hand side: <--======----0------------>
+///
+/// or
+///
+/// Left-hand side: <------------0--======---->
+/// Right-hand side: <------------0--======---->
+/// ```
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`. Therefore,
+/// it should be used with caution. Currently, it is used in contexts where the
+/// `DataType` (`dt`) is validated prior to calling this function.
+fn mul_helper_zero_exclusive(
+ dt: &DataType,
+ lhs: &Interval,
+ rhs: &Interval,
+ zero: ScalarValue,
+) -> Interval {
+ let (lower, upper) = match (
+ lhs.upper <= zero && !lhs.upper.is_null(),
+ rhs.upper <= zero && !rhs.upper.is_null(),
+ ) {
+ // With the following interval bounds, there is no possibility to create an invalid interval.
+ (true, true) => (
+ // <--======----0------------>
+ // <--======----0------------>
+ mul_bounds::<false>(dt, &lhs.upper, &rhs.upper),
+ mul_bounds::<true>(dt, &lhs.lower, &rhs.lower),
+ ),
+ (true, false) => (
+ // <--======----0------------>
+ // <------------0--======---->
+ mul_bounds::<false>(dt, &lhs.lower, &rhs.upper),
+ mul_bounds::<true>(dt, &lhs.upper, &rhs.lower),
+ ),
+ (false, true) => (
+ // <------------0--======---->
+ // <--======----0------------>
+ mul_bounds::<false>(dt, &rhs.lower, &lhs.upper),
+ mul_bounds::<true>(dt, &rhs.upper, &lhs.lower),
+ ),
+ (false, false) => (
+ // <------------0--======---->
+ // <------------0--======---->
+ mul_bounds::<false>(dt, &lhs.lower, &rhs.lower),
+ mul_bounds::<true>(dt, &lhs.upper, &rhs.upper),
+ ),
+ };
+ Interval::new(lower, upper)
+}
+
+/// Divides the left-hand side interval by the right-hand side interval when
+/// the former contains zero.
+///
+/// This function takes in two intervals (`lhs` and `rhs`) as arguments and
+/// returns their quotient (whose data type is known to be `dt`). This function
+/// serves as a subroutine that handles the specific case when only `lhs` contains
+/// zero within its range. Returns an error if the division of bounds fails.
+///
+/// ``` text
+/// Left-hand side: <-------=====0=====------->
+/// Right-hand side: <--======----0------------>
+///
+/// or
+///
+/// Left-hand side: <-------=====0=====------->
+/// Right-hand side: <------------0--======---->
+/// ```
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`. Therefore,
+/// it should be used with caution. Currently, it is used in contexts where the
+/// `DataType` (`dt`) is validated prior to calling this function.
+fn div_helper_lhs_zero_inclusive(
+ dt: &DataType,
+ lhs: &Interval,
+ rhs: &Interval,
+ zero_point: &Interval,
+) -> Interval {
+ // With the following interval bounds, there is no possibility to create an invalid interval.
+ if rhs.upper <= zero_point.lower && !rhs.upper.is_null() {
+ // <-------=====0=====------->
+ // <--======----0------------>
+ let lower = div_bounds::<false>(dt, &lhs.upper, &rhs.upper);
+ let upper = div_bounds::<true>(dt, &lhs.lower, &rhs.upper);
+ Interval::new(lower, upper)
+ } else {
+ // <-------=====0=====------->
+ // <------------0--======---->
+ let lower = div_bounds::<false>(dt, &lhs.lower, &rhs.lower);
+ let upper = div_bounds::<true>(dt, &lhs.upper, &rhs.lower);
+ Interval::new(lower, upper)
+ }
+}
+
+/// Divides the left-hand side interval by the right-hand side interval when
+/// neither interval contains zero.
+///
+/// This function takes in two intervals (`lhs` and `rhs`) as arguments and
+/// returns their quotient (whose data type is known to be `dt`). It is
+/// specifically designed to handle intervals that do not contain zero within
+/// their ranges. Returns an error if the division of bounds fails.
+///
+/// ``` text
+/// Left-hand side: <--======----0------------>
+/// Right-hand side: <--======----0------------>
+///
+/// or
+///
+/// Left-hand side: <--======----0------------>
+/// Right-hand side: <------------0--======---->
+///
+/// or
+///
+/// Left-hand side: <------------0--======---->
+/// Right-hand side: <--======----0------------>
+///
+/// or
+///
+/// Left-hand side: <------------0--======---->
+/// Right-hand side: <------------0--======---->
+/// ```
+///
+/// **Caution:** This function contains multiple calls to `unwrap()`. Therefore,
+/// it should be used with caution. Currently, it is used in contexts where the
+/// `DataType` (`dt`) is validated prior to calling this function.
+fn div_helper_zero_exclusive(
+ dt: &DataType,
+ lhs: &Interval,
+ rhs: &Interval,
+ zero_point: &Interval,
+) -> Interval {
+ let (lower, upper) = match (
+ lhs.upper <= zero_point.lower && !lhs.upper.is_null(),
+ rhs.upper <= zero_point.lower && !rhs.upper.is_null(),
+ ) {
+ // With the following interval bounds, there is no possibility to create an invalid interval.
+ (true, true) => (
+ // <--======----0------------>
+ // <--======----0------------>
+ div_bounds::<false>(dt, &lhs.upper, &rhs.lower),
+ div_bounds::<true>(dt, &lhs.lower, &rhs.upper),
+ ),
+ (true, false) => (
+ // <--======----0------------>
+ // <------------0--======---->
+ div_bounds::<false>(dt, &lhs.lower, &rhs.lower),
+ div_bounds::<true>(dt, &lhs.upper, &rhs.upper),
+ ),
+ (false, true) => (
+ // <------------0--======---->
+ // <--======----0------------>
+ div_bounds::<false>(dt, &lhs.upper, &rhs.upper),
+ div_bounds::<true>(dt, &lhs.lower, &rhs.lower),
+ ),
+ (false, false) => (
+ // <------------0--======---->
+ // <------------0--======---->
+ div_bounds::<false>(dt, &lhs.lower, &rhs.upper),
+ div_bounds::<true>(dt, &lhs.upper, &rhs.lower),
+ ),
+ };
+ Interval::new(lower, upper)
+}
+
+/// This function computes the selectivity of an operation by computing the
+/// cardinality ratio of the given input/output intervals. If this can not be
+/// calculated for some reason, it returns `1.0` meaning fully selective (no
+/// filtering).
+pub fn cardinality_ratio(initial_interval: &Interval, final_interval: &Interval) -> f64 {
+ match (final_interval.cardinality(), initial_interval.cardinality()) {
+ (Some(final_interval), Some(initial_interval)) => {
+ (final_interval as f64) / (initial_interval as f64)
+ }
+ _ => 1.0,
+ }
+}
+
+/// Cast scalar value to the given data type using an arrow kernel.
+fn cast_scalar_value(
+ value: &ScalarValue,
+ data_type: &DataType,
+ cast_options: &CastOptions,
+) -> Result<ScalarValue> {
+ let cast_array = cast_with_options(&value.to_array()?, data_type, cast_options)?;
+ ScalarValue::try_from_array(&cast_array, 0)
+}
+
+/// An [Interval] that also tracks null status using a boolean interval.
+///
+/// This represents values that may be in a particular range or be null.
+///
+/// # Examples
+///
+/// ```
+/// use arrow::datatypes::DataType;
+/// use datafusion_common::ScalarValue;
+/// use datafusion_expr::interval_arithmetic::Interval;
+/// use datafusion_expr::interval_arithmetic::NullableInterval;
+///
+/// // [1, 2) U {NULL}
+/// let maybe_null = NullableInterval::MaybeNull {
+/// values: Interval::try_new(
+/// ScalarValue::Int32(Some(1)),
+/// ScalarValue::Int32(Some(2)),
+/// ).unwrap(),
+/// };
+///
+/// // (0, ∞)
+/// let not_null = NullableInterval::NotNull {
+/// values: Interval::try_new(
+/// ScalarValue::Int32(Some(0)),
+/// ScalarValue::Int32(None),
+/// ).unwrap(),
+/// };
+///
+/// // {NULL}
+/// let null_interval = NullableInterval::Null { datatype: DataType::Int32 };
+///
+/// // {4}
+/// let single_value = NullableInterval::from(ScalarValue::Int32(Some(4)));
+/// ```
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum NullableInterval {
+ /// The value is always null. This is typed so it can be used in physical
+ /// expressions, which don't do type coercion.
+ Null { datatype: DataType },
+ /// The value may or may not be null. If it is non-null, its is within the
+ /// specified range.
+ MaybeNull { values: Interval },
+ /// The value is definitely not null, and is within the specified range.
+ NotNull { values: Interval },
+}
+
+impl Display for NullableInterval {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ match self {
+ Self::Null { .. } => write!(f, "NullableInterval: {{NULL}}"),
+ Self::MaybeNull { values } => {
+ write!(f, "NullableInterval: {} U {{NULL}}", values)
+ }
+ Self::NotNull { values } => write!(f, "NullableInterval: {}", values),
+ }
+ }
+}
+
+impl From<ScalarValue> for NullableInterval {
+ /// Create an interval that represents a single value.
+ fn from(value: ScalarValue) -> Self {
+ if value.is_null() {
+ Self::Null {
+ datatype: value.data_type(),
+ }
+ } else {
+ Self::NotNull {
+ values: Interval {
+ lower: value.clone(),
+ upper: value,
+ },
+ }
+ }
+ }
+}
+
+impl NullableInterval {
+ /// Get the values interval, or None if this interval is definitely null.
+ pub fn values(&self) -> Option<&Interval> {
+ match self {
+ Self::Null { .. } => None,
+ Self::MaybeNull { values } | Self::NotNull { values } => Some(values),
+ }
+ }
+
+ /// Get the data type
+ pub fn data_type(&self) -> DataType {
+ match self {
+ Self::Null { datatype } => datatype.clone(),
+ Self::MaybeNull { values } | Self::NotNull { values } => values.data_type(),
+ }
+ }
+
+ /// Return true if the value is definitely true (and not null).
+ pub fn is_certainly_true(&self) -> bool {
+ match self {
+ Self::Null { .. } | Self::MaybeNull { .. } => false,
+ Self::NotNull { values } => values == &Interval::CERTAINLY_TRUE,
+ }
+ }
+
+ /// Return true if the value is definitely false (and not null).
+ pub fn is_certainly_false(&self) -> bool {
+ match self {
+ Self::Null { .. } => false,
+ Self::MaybeNull { .. } => false,
+ Self::NotNull { values } => values == &Interval::CERTAINLY_FALSE,
+ }
+ }
+
+ /// Perform logical negation on a boolean nullable interval.
+ fn not(&self) -> Result<Self> {
+ match self {
+ Self::Null { datatype } => Ok(Self::Null {
+ datatype: datatype.clone(),
+ }),
+ Self::MaybeNull { values } => Ok(Self::MaybeNull {
+ values: values.not()?,
+ }),
+ Self::NotNull { values } => Ok(Self::NotNull {
+ values: values.not()?,
+ }),
+ }
+ }
+
+ /// Apply the given operator to this interval and the given interval.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use datafusion_common::ScalarValue;
+ /// use datafusion_expr::Operator;
+ /// use datafusion_expr::interval_arithmetic::Interval;
+ /// use datafusion_expr::interval_arithmetic::NullableInterval;
+ ///
+ /// // 4 > 3 -> true
+ /// let lhs = NullableInterval::from(ScalarValue::Int32(Some(4)));
+ /// let rhs = NullableInterval::from(ScalarValue::Int32(Some(3)));
+ /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
+ /// assert_eq!(result, NullableInterval::from(ScalarValue::Boolean(Some(true))));
+ ///
+ /// // [1, 3) > NULL -> NULL
+ /// let lhs = NullableInterval::NotNull {
+ /// values: Interval::try_new(
+ /// ScalarValue::Int32(Some(1)),
+ /// ScalarValue::Int32(Some(3)),
+ /// ).unwrap(),
+ /// };
+ /// let rhs = NullableInterval::from(ScalarValue::Int32(None));
+ /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
+ /// assert_eq!(result.single_value(), Some(ScalarValue::Boolean(None)));
+ ///
+ /// // [1, 3] > [2, 4] -> [false, true]
+ /// let lhs = NullableInterval::NotNull {
+ /// values: Interval::try_new(
+ /// ScalarValue::Int32(Some(1)),
+ /// ScalarValue::Int32(Some(3)),
+ /// ).unwrap(),
+ /// };
+ /// let rhs = NullableInterval::NotNull {
+ /// values: Interval::try_new(
+ /// ScalarValue::Int32(Some(2)),
+ /// ScalarValue::Int32(Some(4)),
+ /// ).unwrap(),
+ /// };
+ /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
+ /// // Both inputs are valid (non-null), so result must be non-null
+ /// assert_eq!(result, NullableInterval::NotNull {
+ /// // Uncertain whether inequality is true or false
+ /// values: Interval::UNCERTAIN,
+ /// });
+ /// ```
+ pub fn apply_operator(&self, op: &Operator, rhs: &Self) -> Result<Self> {
+ match op {
+ Operator::IsDistinctFrom => {
+ let values = match (self, rhs) {
+ // NULL is distinct from NULL -> False
+ (Self::Null { .. }, Self::Null { .. }) => Interval::CERTAINLY_FALSE,
+ // x is distinct from y -> x != y,
+ // if at least one of them is never null.
+ (Self::NotNull { .. }, _) | (_, Self::NotNull { .. }) => {
+ let lhs_values = self.values();
+ let rhs_values = rhs.values();
+ match (lhs_values, rhs_values) {
+ (Some(lhs_values), Some(rhs_values)) => {
+ lhs_values.equal(rhs_values)?.not()?
+ }
+ (Some(_), None) | (None, Some(_)) => Interval::CERTAINLY_TRUE,
+ (None, None) => unreachable!("Null case handled above"),
+ }
+ }
+ _ => Interval::UNCERTAIN,
+ };
+ // IsDistinctFrom never returns null.
+ Ok(Self::NotNull { values })
+ }
+ Operator::IsNotDistinctFrom => self
+ .apply_operator(&Operator::IsDistinctFrom, rhs)
+ .map(|i| i.not())?,
+ _ => {
+ if let (Some(left_values), Some(right_values)) =
+ (self.values(), rhs.values())
+ {
+ let values = apply_operator(op, left_values, right_values)?;
+ match (self, rhs) {
+ (Self::NotNull { .. }, Self::NotNull { .. }) => {
+ Ok(Self::NotNull { values })
+ }
+ _ => Ok(Self::MaybeNull { values }),
+ }
+ } else if op.is_comparison_operator() {
+ Ok(Self::Null {
+ datatype: DataType::Boolean,
+ })
+ } else {
+ Ok(Self::Null {
+ datatype: self.data_type(),
+ })
+ }
+ }
+ }
+ }
+
+ /// Decide if this interval is a superset of, overlaps with, or
+ /// disjoint with `other` by returning `[true, true]`, `[false, true]` or
+ /// `[false, false]` respectively.
+ ///
+ /// NOTE: This function only works with intervals of the same data type.
+ /// Attempting to compare intervals of different data types will lead
+ /// to an error.
+ pub fn contains<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
+ let rhs = other.borrow();
+ if let (Some(left_values), Some(right_values)) = (self.values(), rhs.values()) {
+ left_values
+ .contains(right_values)
+ .map(|values| match (self, rhs) {
+ (Self::NotNull { .. }, Self::NotNull { .. }) => {
+ Self::NotNull { values }
+ }
+ _ => Self::MaybeNull { values },
+ })
+ } else {
+ Ok(Self::Null {
+ datatype: DataType::Boolean,
+ })
+ }
+ }
+
+ /// If the interval has collapsed to a single value, return that value.
+ /// Otherwise, returns `None`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use datafusion_common::ScalarValue;
+ /// use datafusion_expr::interval_arithmetic::Interval;
+ /// use datafusion_expr::interval_arithmetic::NullableInterval;
+ ///
+ /// let interval = NullableInterval::from(ScalarValue::Int32(Some(4)));
+ /// assert_eq!(interval.single_value(), Some(ScalarValue::Int32(Some(4))));
+ ///
+ /// let interval = NullableInterval::from(ScalarValue::Int32(None));
+ /// assert_eq!(interval.single_value(), Some(ScalarValue::Int32(None)));
+ ///
+ /// let interval = NullableInterval::MaybeNull {
+ /// values: Interval::try_new(
+ /// ScalarValue::Int32(Some(1)),
+ /// ScalarValue::Int32(Some(4)),
+ /// ).unwrap(),
+ /// };
+ /// assert_eq!(interval.single_value(), None);
+ /// ```
+ pub fn single_value(&self) -> Option<ScalarValue> {
+ match self {
+ Self::Null { datatype } => {
+ Some(ScalarValue::try_from(datatype).unwrap_or(ScalarValue::Null))
+ }
+ Self::MaybeNull { values } | Self::NotNull { values }
+ if values.lower == values.upper && !values.lower.is_null() =>
+ {
+ Some(values.lower.clone())
+ }
+ _ => None,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::interval_arithmetic::{next_value, prev_value, satisfy_greater, Interval};
+
+ use arrow::datatypes::DataType;
+ use datafusion_common::{Result, ScalarValue};
+
+ #[test]
+ fn test_next_prev_value() -> Result<()> {
+ let zeros = vec![
+ ScalarValue::new_zero(&DataType::UInt8)?,
+ ScalarValue::new_zero(&DataType::UInt16)?,
+ ScalarValue::new_zero(&DataType::UInt32)?,
+ ScalarValue::new_zero(&DataType::UInt64)?,
+ ScalarValue::new_zero(&DataType::Int8)?,
+ ScalarValue::new_zero(&DataType::Int16)?,
+ ScalarValue::new_zero(&DataType::Int32)?,
+ ScalarValue::new_zero(&DataType::Int64)?,
+ ];
+ let ones = vec![
+ ScalarValue::new_one(&DataType::UInt8)?,
+ ScalarValue::new_one(&DataType::UInt16)?,
+ ScalarValue::new_one(&DataType::UInt32)?,
+ ScalarValue::new_one(&DataType::UInt64)?,
+ ScalarValue::new_one(&DataType::Int8)?,
+ ScalarValue::new_one(&DataType::Int16)?,
+ ScalarValue::new_one(&DataType::Int32)?,
+ ScalarValue::new_one(&DataType::Int64)?,
+ ];
+ zeros.into_iter().zip(ones).for_each(|(z, o)| {
+ assert_eq!(next_value(z.clone()), o);
+ assert_eq!(prev_value(o), z);
+ });
+
+ let values = vec![
+ ScalarValue::new_zero(&DataType::Float32)?,
+ ScalarValue::new_zero(&DataType::Float64)?,
+ ];
+ let eps = vec![
+ ScalarValue::Float32(Some(1e-6)),
+ ScalarValue::Float64(Some(1e-6)),
+ ];
+ values.into_iter().zip(eps).for_each(|(value, eps)| {
+ assert!(next_value(value.clone())
+ .sub(value.clone())
+ .unwrap()
+ .lt(&eps));
+ assert!(value
+ .clone()
+ .sub(prev_value(value.clone()))
+ .unwrap()
+ .lt(&eps));
+ assert_ne!(next_value(value.clone()), value);
+ assert_ne!(prev_value(value.clone()), value);
+ });
+
+ let min_max = vec![
+ (
+ ScalarValue::UInt64(Some(u64::MIN)),
+ ScalarValue::UInt64(Some(u64::MAX)),
+ ),
+ (
+ ScalarValue::Int8(Some(i8::MIN)),
+ ScalarValue::Int8(Some(i8::MAX)),
+ ),
+ (
+ ScalarValue::Float32(Some(f32::MIN)),
+ ScalarValue::Float32(Some(f32::MAX)),
+ ),
+ (
+ ScalarValue::Float64(Some(f64::MIN)),
+ ScalarValue::Float64(Some(f64::MAX)),
+ ),
+ ];
+ let inf = vec![
+ ScalarValue::UInt64(None),
+ ScalarValue::Int8(None),
+ ScalarValue::Float32(None),
+ ScalarValue::Float64(None),
+ ];
+ min_max.into_iter().zip(inf).for_each(|((min, max), inf)| {
+ assert_eq!(next_value(max.clone()), inf);
+ assert_ne!(prev_value(max.clone()), max);
+ assert_ne!(prev_value(max.clone()), inf);
+
+ assert_eq!(prev_value(min.clone()), inf);
+ assert_ne!(next_value(min.clone()), min);
+ assert_ne!(next_value(min.clone()), inf);
+
+ assert_eq!(next_value(inf.clone()), inf);
+ assert_eq!(prev_value(inf.clone()), inf);
+ });
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_new_interval() -> Result<()> {
+ use ScalarValue::*;
+
+ let cases = vec![
+ (
+ (Boolean(None), Boolean(Some(false))),
+ Boolean(Some(false)),
+ Boolean(Some(false)),
+ ),
+ (
+ (Boolean(Some(false)), Boolean(None)),
+ Boolean(Some(false)),
+ Boolean(Some(true)),
+ ),
+ (
+ (Boolean(Some(false)), Boolean(Some(true))),
+ Boolean(Some(false)),
+ Boolean(Some(true)),
+ ),
+ (
+ (UInt16(Some(u16::MAX)), UInt16(None)),
+ UInt16(Some(u16::MAX)),
+ UInt16(None),
+ ),
+ (
+ (Int16(None), Int16(Some(-1000))),
+ Int16(None),
+ Int16(Some(-1000)),
+ ),
+ (
+ (Float32(Some(f32::MAX)), Float32(Some(f32::MAX))),
+ Float32(Some(f32::MAX)),
+ Float32(Some(f32::MAX)),
+ ),
+ (
+ (Float32(Some(f32::NAN)), Float32(Some(f32::MIN))),
+ Float32(None),
+ Float32(Some(f32::MIN)),
+ ),
+ (
+ (
+ Float64(Some(f64::NEG_INFINITY)),
+ Float64(Some(f64::INFINITY)),
+ ),
+ Float64(None),
+ Float64(None),
+ ),
+ ];
+ for (inputs, lower, upper) in cases {
+ let result = Interval::try_new(inputs.0, inputs.1)?;
+ assert_eq!(result.clone().lower(), &lower);
+ assert_eq!(result.upper(), &upper);
+ }
+
+ let invalid_intervals = vec![
+ (Float32(Some(f32::INFINITY)), Float32(Some(100_f32))),
+ (Float64(Some(0_f64)), Float64(Some(f64::NEG_INFINITY))),
+ (Boolean(Some(true)), Boolean(Some(false))),
+ (Int32(Some(1000)), Int32(Some(-2000))),
+ (UInt64(Some(1)), UInt64(Some(0))),
+ ];
+ for (lower, upper) in invalid_intervals {
+ Interval::try_new(lower, upper).expect_err(
+ "Given parameters should have given an invalid interval error",
+ );
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_make_unbounded() -> Result<()> {
+ use ScalarValue::*;
+
+ let unbounded_cases = vec![
+ (DataType::Boolean, Boolean(Some(false)), Boolean(Some(true))),
+ (DataType::UInt8, UInt8(None), UInt8(None)),
+ (DataType::UInt16, UInt16(None), UInt16(None)),
+ (DataType::UInt32, UInt32(None), UInt32(None)),
+ (DataType::UInt64, UInt64(None), UInt64(None)),
+ (DataType::Int8, Int8(None), Int8(None)),
+ (DataType::Int16, Int16(None), Int16(None)),
+ (DataType::Int32, Int32(None), Int32(None)),
+ (DataType::Int64, Int64(None), Int64(None)),
+ (DataType::Float32, Float32(None), Float32(None)),
+ (DataType::Float64, Float64(None), Float64(None)),
+ ];
+ for (dt, lower, upper) in unbounded_cases {
+ let inf = Interval::make_unbounded(&dt)?;
+ assert_eq!(inf.clone().lower(), &lower);
+ assert_eq!(inf.upper(), &upper);
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn gt_lt_test() -> Result<()> {
+ let exactly_gt_cases = vec![
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(999_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ Interval::make(None, Some(999_i64))?,
+ ),
+ (
+ Interval::make(Some(501_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(500_i64))?,
+ ),
+ (
+ Interval::make(Some(-1000_i64), Some(1000_i64))?,
+ Interval::make(None, Some(-1500_i64))?,
+ ),
+ (
+ Interval::try_new(
+ next_value(ScalarValue::Float32(Some(0.0))),
+ next_value(ScalarValue::Float32(Some(0.0))),
+ )?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1.0_f32), Some(-1.0_f32))?,
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(-1.0))),
+ prev_value(ScalarValue::Float32(Some(-1.0))),
+ )?,
+ ),
+ ];
+ for (first, second) in exactly_gt_cases {
+ assert_eq!(first.gt(second.clone())?, Interval::CERTAINLY_TRUE);
+ assert_eq!(second.lt(first)?, Interval::CERTAINLY_TRUE);
+ }
+
+ let possibly_gt_cases = vec![
+ (
+ Interval::make(Some(1000_i64), Some(2000_i64))?,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(Some(1000_i64), None)?,
+ ),
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::try_new(
+ ScalarValue::Float32(Some(0.0_f32)),
+ next_value(ScalarValue::Float32(Some(0.0_f32))),
+ )?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1.0_f32), Some(-1.0_f32))?,
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(-1.0_f32))),
+ ScalarValue::Float32(Some(-1.0_f32)),
+ )?,
+ ),
+ ];
+ for (first, second) in possibly_gt_cases {
+ assert_eq!(first.gt(second.clone())?, Interval::UNCERTAIN);
+ assert_eq!(second.lt(first)?, Interval::UNCERTAIN);
+ }
+
+ let not_gt_cases = vec![
+ (
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(1000_i64))?,
+ Interval::make(Some(1000_i64), None)?,
+ ),
+ (
+ Interval::make(None, Some(1000_i64))?,
+ Interval::make(Some(1000_i64), Some(1500_i64))?,
+ ),
+ (
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(0.0_f32))),
+ ScalarValue::Float32(Some(0.0_f32)),
+ )?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1.0_f32), Some(-1.0_f32))?,
+ Interval::try_new(
+ ScalarValue::Float32(Some(-1.0_f32)),
+ next_value(ScalarValue::Float32(Some(-1.0_f32))),
+ )?,
+ ),
+ ];
+ for (first, second) in not_gt_cases {
+ assert_eq!(first.gt(second.clone())?, Interval::CERTAINLY_FALSE);
+ assert_eq!(second.lt(first)?, Interval::CERTAINLY_FALSE);
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn gteq_lteq_test() -> Result<()> {
+ let exactly_gteq_cases = vec![
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ Interval::make(None, Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(500_i64))?,
+ ),
+ (
+ Interval::make(Some(-1000_i64), Some(1000_i64))?,
+ Interval::make(None, Some(-1500_i64))?,
+ ),
+ (
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::try_new(
+ ScalarValue::Float32(Some(-1.0)),
+ next_value(ScalarValue::Float32(Some(-1.0))),
+ )?,
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(-1.0))),
+ ScalarValue::Float32(Some(-1.0)),
+ )?,
+ ),
+ ];
+ for (first, second) in exactly_gteq_cases {
+ assert_eq!(first.gt_eq(second.clone())?, Interval::CERTAINLY_TRUE);
+ assert_eq!(second.lt_eq(first)?, Interval::CERTAINLY_TRUE);
+ }
+
+ let possibly_gteq_cases = vec![
+ (
+ Interval::make(Some(999_i64), Some(2000_i64))?,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(1001_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), None)?,
+ Interval::make(Some(1000_i64), None)?,
+ ),
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(0.0))),
+ ScalarValue::Float32(Some(0.0)),
+ )?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1.0_f32), Some(-1.0_f32))?,
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(-1.0_f32))),
+ next_value(ScalarValue::Float32(Some(-1.0_f32))),
+ )?,
+ ),
+ ];
+ for (first, second) in possibly_gteq_cases {
+ assert_eq!(first.gt_eq(second.clone())?, Interval::UNCERTAIN);
+ assert_eq!(second.lt_eq(first)?, Interval::UNCERTAIN);
+ }
+
+ let not_gteq_cases = vec![
+ (
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ Interval::make(Some(2000_i64), Some(2000_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(999_i64))?,
+ Interval::make(Some(1000_i64), None)?,
+ ),
+ (
+ Interval::make(None, Some(1000_i64))?,
+ Interval::make(Some(1001_i64), Some(1500_i64))?,
+ ),
+ (
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(0.0_f32))),
+ prev_value(ScalarValue::Float32(Some(0.0_f32))),
+ )?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1.0_f32), Some(-1.0_f32))?,
+ Interval::try_new(
+ next_value(ScalarValue::Float32(Some(-1.0))),
+ next_value(ScalarValue::Float32(Some(-1.0))),
+ )?,
+ ),
+ ];
+ for (first, second) in not_gteq_cases {
+ assert_eq!(first.gt_eq(second.clone())?, Interval::CERTAINLY_FALSE);
+ assert_eq!(second.lt_eq(first)?, Interval::CERTAINLY_FALSE);
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn equal_test() -> Result<()> {
+ let exactly_eq_cases = vec![
+ (
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ ),
+ (
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ ),
+ (
+ Interval::make(Some(f64::MIN), Some(f64::MIN))?,
+ Interval::make(Some(f64::MIN), Some(f64::MIN))?,
+ ),
+ ];
+ for (first, second) in exactly_eq_cases {
+ assert_eq!(first.equal(second.clone())?, Interval::CERTAINLY_TRUE);
+ assert_eq!(second.equal(first)?, Interval::CERTAINLY_TRUE);
+ }
+
+ let possibly_eq_cases = vec![
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(0_i64))?,
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(0_i64))?,
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(100.0_f32), Some(200.0_f32))?,
+ Interval::make(Some(0.0_f32), Some(1000.0_f32))?,
+ ),
+ (
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(0.0))),
+ ScalarValue::Float32(Some(0.0)),
+ )?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1.0_f32), Some(-1.0_f32))?,
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(-1.0))),
+ next_value(ScalarValue::Float32(Some(-1.0))),
+ )?,
+ ),
+ ];
+ for (first, second) in possibly_eq_cases {
+ assert_eq!(first.equal(second.clone())?, Interval::UNCERTAIN);
+ assert_eq!(second.equal(first)?, Interval::UNCERTAIN);
+ }
+
+ let not_eq_cases = vec![
+ (
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ Interval::make(Some(2000_i64), Some(2000_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(999_i64))?,
+ Interval::make(Some(1000_i64), None)?,
+ ),
+ (
+ Interval::make(None, Some(1000_i64))?,
+ Interval::make(Some(1001_i64), Some(1500_i64))?,
+ ),
+ (
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(0.0))),
+ prev_value(ScalarValue::Float32(Some(0.0))),
+ )?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1.0_f32), Some(-1.0_f32))?,
+ Interval::try_new(
+ next_value(ScalarValue::Float32(Some(-1.0))),
+ next_value(ScalarValue::Float32(Some(-1.0))),
+ )?,
+ ),
+ ];
+ for (first, second) in not_eq_cases {
+ assert_eq!(first.equal(second.clone())?, Interval::CERTAINLY_FALSE);
+ assert_eq!(second.equal(first)?, Interval::CERTAINLY_FALSE);
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn and_test() -> Result<()> {
+ let cases = vec![
+ (false, true, false, false, false, false),
+ (false, false, false, true, false, false),
+ (false, true, false, true, false, true),
+ (false, true, true, true, false, true),
+ (false, false, false, false, false, false),
+ (true, true, true, true, true, true),
+ ];
+
+ for case in cases {
+ assert_eq!(
+ Interval::make(Some(case.0), Some(case.1))?
+ .and(Interval::make(Some(case.2), Some(case.3))?)?,
+ Interval::make(Some(case.4), Some(case.5))?
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn not_test() -> Result<()> {
+ let cases = vec![
+ (false, true, false, true),
+ (false, false, true, true),
+ (true, true, false, false),
+ ];
+
+ for case in cases {
+ assert_eq!(
+ Interval::make(Some(case.0), Some(case.1))?.not()?,
+ Interval::make(Some(case.2), Some(case.3))?
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn intersect_test() -> Result<()> {
+ let possible_cases = vec![
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make::<i64>(None, None)?,
+ Interval::make(Some(1000_i64), None)?,
+ ),
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(1000_i64))?,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(2000_i64))?,
+ Interval::make(Some(1000_i64), Some(2000_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), Some(2000_i64))?,
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(Some(1000_i64), Some(2000_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), Some(2000_i64))?,
+ Interval::make(Some(1000_i64), Some(1500_i64))?,
+ Interval::make(Some(1000_i64), Some(1500_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), Some(2000_i64))?,
+ Interval::make(Some(500_i64), Some(1500_i64))?,
+ Interval::make(Some(1000_i64), Some(1500_i64))?,
+ ),
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make::<i64>(None, None)?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(None, Some(2000_u64))?,
+ Interval::make(Some(500_u64), None)?,
+ Interval::make(Some(500_u64), Some(2000_u64))?,
+ ),
+ (
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ Interval::make(Some(0_u64), None)?,
+ Interval::make(Some(0_u64), Some(0_u64))?,
+ ),
+ (
+ Interval::make(Some(1000.0_f32), None)?,
+ Interval::make(None, Some(1000.0_f32))?,
+ Interval::make(Some(1000.0_f32), Some(1000.0_f32))?,
+ ),
+ (
+ Interval::make(Some(1000.0_f32), Some(1500.0_f32))?,
+ Interval::make(Some(0.0_f32), Some(1500.0_f32))?,
+ Interval::make(Some(1000.0_f32), Some(1500.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-1000.0_f64), Some(1500.0_f64))?,
+ Interval::make(Some(-1500.0_f64), Some(2000.0_f64))?,
+ Interval::make(Some(-1000.0_f64), Some(1500.0_f64))?,
+ ),
+ (
+ Interval::make(Some(16.0_f64), Some(32.0_f64))?,
+ Interval::make(Some(32.0_f64), Some(64.0_f64))?,
+ Interval::make(Some(32.0_f64), Some(32.0_f64))?,
+ ),
+ ];
+ for (first, second, expected) in possible_cases {
+ assert_eq!(first.intersect(second)?.unwrap(), expected)
+ }
+
+ let empty_cases = vec![
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(999_i64))?,
+ ),
+ (
+ Interval::make(Some(1500_i64), Some(2000_i64))?,
+ Interval::make(Some(1000_i64), Some(1499_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ Interval::make(Some(2000_i64), Some(3000_i64))?,
+ ),
+ (
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(1.0))),
+ prev_value(ScalarValue::Float32(Some(1.0))),
+ )?,
+ Interval::make(Some(1.0_f32), Some(1.0_f32))?,
+ ),
+ (
+ Interval::try_new(
+ next_value(ScalarValue::Float32(Some(1.0))),
+ next_value(ScalarValue::Float32(Some(1.0))),
+ )?,
+ Interval::make(Some(1.0_f32), Some(1.0_f32))?,
+ ),
+ ];
+ for (first, second) in empty_cases {
+ assert_eq!(first.intersect(second)?, None)
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_contains() -> Result<()> {
+ let possible_cases = vec![
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make::<i64>(None, None)?,
+ Interval::CERTAINLY_TRUE,
+ ),
+ (
+ Interval::make(Some(1500_i64), Some(2000_i64))?,
+ Interval::make(Some(1501_i64), Some(1999_i64))?,
+ Interval::CERTAINLY_TRUE,
+ ),
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make::<i64>(None, None)?,
+ Interval::UNCERTAIN,
+ ),
+ (
+ Interval::make(Some(1000_i64), Some(2000_i64))?,
+ Interval::make(Some(500), Some(1500_i64))?,
+ Interval::UNCERTAIN,
+ ),
+ (
+ Interval::make(Some(16.0), Some(32.0))?,
+ Interval::make(Some(32.0), Some(64.0))?,
+ Interval::UNCERTAIN,
+ ),
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(0_i64))?,
+ Interval::CERTAINLY_FALSE,
+ ),
+ (
+ Interval::make(Some(1500_i64), Some(2000_i64))?,
+ Interval::make(Some(1000_i64), Some(1499_i64))?,
+ Interval::CERTAINLY_FALSE,
+ ),
+ (
+ Interval::try_new(
+ prev_value(ScalarValue::Float32(Some(1.0))),
+ prev_value(ScalarValue::Float32(Some(1.0))),
+ )?,
+ Interval::make(Some(1.0_f32), Some(1.0_f32))?,
+ Interval::CERTAINLY_FALSE,
+ ),
+ (
+ Interval::try_new(
+ next_value(ScalarValue::Float32(Some(1.0))),
+ next_value(ScalarValue::Float32(Some(1.0))),
+ )?,
+ Interval::make(Some(1.0_f32), Some(1.0_f32))?,
+ Interval::CERTAINLY_FALSE,
+ ),
+ ];
+ for (first, second, expected) in possible_cases {
+ assert_eq!(first.contains(second)?, expected)
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_add() -> Result<()> {
+ let cases = vec![
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(None, Some(200_i64))?,
+ Interval::make(None, Some(400_i64))?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(200_i64), None)?,
+ Interval::make(Some(300_i64), None)?,
+ ),
+ (
+ Interval::make(None, Some(200_i64))?,
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(None, Some(400_i64))?,
+ ),
+ (
+ Interval::make(Some(200_i64), None)?,
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(300_i64), None)?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(-300_i64), Some(150_i64))?,
+ Interval::make(Some(-200_i64), Some(350_i64))?,
+ ),
+ (
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(Some(11_f32), Some(11_f32))?,
+ Interval::make(Some(f32::MAX), None)?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(f32::MIN))?,
+ Interval::make(Some(-10_f32), Some(10_f32))?,
+ // Since rounding mode is up, the result would be much greater than f32::MIN
+ // (f32::MIN = -3.4_028_235e38, the result is -3.4_028_233e38)
+ Interval::make(
+ None,
+ Some(-340282330000000000000000000000000000000.0_f32),
+ )?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(f32::MIN))?,
+ Interval::make(Some(-10_f32), Some(-10_f32))?,
+ Interval::make(None, Some(f32::MIN))?,
+ ),
+ (
+ Interval::make(Some(1.0), Some(f32::MAX))?,
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(Some(f32::MAX), None)?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(f32::MIN))?,
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(Some(-0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(100_f64), None)?,
+ Interval::make(None, Some(200_f64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(None, Some(100_f64))?,
+ Interval::make(None, Some(200_f64))?,
+ Interval::make(None, Some(300_f64))?,
+ ),
+ ];
+ for case in cases {
+ let result = case.0.add(case.1)?;
+ if case.0.data_type().is_floating() {
+ assert!(
+ result.lower().is_null() && case.2.lower().is_null()
+ || result.lower().le(case.2.lower())
+ );
+ assert!(
+ result.upper().is_null() && case.2.upper().is_null()
+ || result.upper().ge(case.2.upper())
+ );
+ } else {
+ assert_eq!(result, case.2);
+ }
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_sub() -> Result<()> {
+ let cases = vec![
+ (
+ Interval::make(Some(i32::MAX), Some(i32::MAX))?,
+ Interval::make(Some(11_i32), Some(11_i32))?,
+ Interval::make(Some(i32::MAX - 11), Some(i32::MAX - 11))?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(None, Some(200_i64))?,
+ Interval::make(Some(-100_i64), None)?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(200_i64), None)?,
+ Interval::make(None, Some(0_i64))?,
+ ),
+ (
+ Interval::make(None, Some(200_i64))?,
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(None, Some(100_i64))?,
+ ),
+ (
+ Interval::make(Some(200_i64), None)?,
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(0_i64), None)?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(-300_i64), Some(150_i64))?,
+ Interval::make(Some(-50_i64), Some(500_i64))?,
+ ),
+ (
+ Interval::make(Some(i64::MIN), Some(i64::MIN))?,
+ Interval::make(Some(-10_i64), Some(-10_i64))?,
+ Interval::make(Some(i64::MIN + 10), Some(i64::MIN + 10))?,
+ ),
+ (
+ Interval::make(Some(1), Some(i64::MAX))?,
+ Interval::make(Some(i64::MAX), Some(i64::MAX))?,
+ Interval::make(Some(1 - i64::MAX), Some(0))?,
+ ),
+ (
+ Interval::make(Some(i64::MIN), Some(i64::MIN))?,
+ Interval::make(Some(i64::MAX), Some(i64::MAX))?,
+ Interval::make(None, Some(i64::MIN))?,
+ ),
+ (
+ Interval::make(Some(2_u32), Some(10_u32))?,
+ Interval::make(Some(4_u32), Some(6_u32))?,
+ Interval::make(None, Some(6_u32))?,
+ ),
+ (
+ Interval::make(Some(2_u32), Some(10_u32))?,
+ Interval::make(Some(20_u32), Some(30_u32))?,
+ Interval::make(None, Some(0_u32))?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(f32::MIN))?,
+ Interval::make(Some(-10_f32), Some(10_f32))?,
+ // Since rounding mode is up, the result would be much larger than f32::MIN
+ // (f32::MIN = -3.4_028_235e38, the result is -3.4_028_233e38)
+ Interval::make(
+ None,
+ Some(-340282330000000000000000000000000000000.0_f32),
+ )?,
+ ),
+ (
+ Interval::make(Some(100_f64), None)?,
+ Interval::make(None, Some(200_f64))?,
+ Interval::make(Some(-100_f64), None)?,
+ ),
+ (
+ Interval::make(None, Some(100_f64))?,
+ Interval::make(None, Some(200_f64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ ];
+ for case in cases {
+ let result = case.0.sub(case.1)?;
+ if case.0.data_type().is_floating() {
+ assert!(
+ result.lower().is_null() && case.2.lower().is_null()
+ || result.lower().le(case.2.lower())
+ );
+ assert!(
+ result.upper().is_null() && case.2.upper().is_null()
+ || result.upper().ge(case.2.upper(),)
+ );
+ } else {
+ assert_eq!(result, case.2);
+ }
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_mul() -> Result<()> {
+ let cases = vec![
+ (
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(None, Some(2_i64))?,
+ Interval::make(None, Some(4_i64))?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(2_i64), None)?,
+ Interval::make(Some(2_i64), None)?,
+ ),
+ (
+ Interval::make(None, Some(2_i64))?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(None, Some(4_i64))?,
+ ),
+ (
+ Interval::make(Some(2_i64), None)?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(2_i64), None)?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(-3_i64), Some(15_i64))?,
+ Interval::make(Some(-6_i64), Some(30_i64))?,
+ ),
+ (
+ Interval::make(Some(-0.0), Some(0.0))?,
+ Interval::make(None, Some(0.0))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(f32::MIN))?,
+ Interval::make(Some(-10_f32), Some(10_f32))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(1_u32), Some(2_u32))?,
+ Interval::make(Some(0_u32), Some(1_u32))?,
+ Interval::make(Some(0_u32), Some(2_u32))?,
+ ),
+ (
+ Interval::make(None, Some(2_u32))?,
+ Interval::make(Some(0_u32), Some(1_u32))?,
+ Interval::make(None, Some(2_u32))?,
+ ),
+ (
+ Interval::make(None, Some(2_u32))?,
+ Interval::make(Some(1_u32), Some(2_u32))?,
+ Interval::make(None, Some(4_u32))?,
+ ),
+ (
+ Interval::make(None, Some(2_u32))?,
+ Interval::make(Some(1_u32), None)?,
+ Interval::make::<u32>(None, None)?,
+ ),
+ (
+ Interval::make::<u32>(None, None)?,
+ Interval::make(Some(0_u32), None)?,
+ Interval::make::<u32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(Some(11_f32), Some(11_f32))?,
+ Interval::make(Some(f32::MAX), None)?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(f32::MIN))?,
+ Interval::make(Some(-10_f32), Some(-10_f32))?,
+ Interval::make(Some(f32::MAX), None)?,
+ ),
+ (
+ Interval::make(Some(1.0), Some(f32::MAX))?,
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(Some(f32::MAX), None)?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(f32::MIN))?,
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(None, Some(f32::MIN))?,
+ ),
+ (
+ Interval::make(Some(-0.0_f32), Some(0.0_f32))?,
+ Interval::make(Some(f32::MAX), None)?,
+ Interval::make::<f32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ Interval::make(Some(f32::MAX), None)?,
+ Interval::make(Some(0.0_f32), None)?,
+ ),
+ (
+ Interval::make(Some(1_f64), None)?,
+ Interval::make(None, Some(2_f64))?,
+ Interval::make::<f64>(None, None)?,
+ ),
+ (
+ Interval::make(None, Some(1_f64))?,
+ Interval::make(None, Some(2_f64))?,
+ Interval::make::<f64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(-0.0_f64), Some(-0.0_f64))?,
+ Interval::make(Some(1_f64), Some(2_f64))?,
+ Interval::make(Some(-0.0_f64), Some(-0.0_f64))?,
+ ),
+ (
+ Interval::make(Some(0.0_f64), Some(0.0_f64))?,
+ Interval::make(Some(1_f64), Some(2_f64))?,
+ Interval::make(Some(0.0_f64), Some(0.0_f64))?,
+ ),
+ (
+ Interval::make(Some(-0.0_f64), Some(0.0_f64))?,
+ Interval::make(Some(1_f64), Some(2_f64))?,
+ Interval::make(Some(-0.0_f64), Some(0.0_f64))?,
+ ),
+ (
+ Interval::make(Some(-0.0_f64), Some(1.0_f64))?,
+ Interval::make(Some(1_f64), Some(2_f64))?,
+ Interval::make(Some(-0.0_f64), Some(2.0_f64))?,
+ ),
+ (
+ Interval::make(Some(0.0_f64), Some(1.0_f64))?,
+ Interval::make(Some(1_f64), Some(2_f64))?,
+ Interval::make(Some(0.0_f64), Some(2.0_f64))?,
+ ),
+ (
+ Interval::make(Some(-0.0_f64), Some(1.0_f64))?,
+ Interval::make(Some(-1_f64), Some(2_f64))?,
+ Interval::make(Some(-1.0_f64), Some(2.0_f64))?,
+ ),
+ (
+ Interval::make::<f64>(None, None)?,
+ Interval::make(Some(-0.0_f64), Some(0.0_f64))?,
+ Interval::make::<f64>(None, None)?,
+ ),
+ (
+ Interval::make::<f64>(None, Some(10.0_f64))?,
+ Interval::make(Some(-0.0_f64), Some(0.0_f64))?,
+ Interval::make::<f64>(None, None)?,
+ ),
+ ];
+ for case in cases {
+ let result = case.0.mul(case.1)?;
+ if case.0.data_type().is_floating() {
+ assert!(
+ result.lower().is_null() && case.2.lower().is_null()
+ || result.lower().le(case.2.lower())
+ );
+ assert!(
+ result.upper().is_null() && case.2.upper().is_null()
+ || result.upper().ge(case.2.upper())
+ );
+ } else {
+ assert_eq!(result, case.2);
+ }
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_div() -> Result<()> {
+ let cases = vec![
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(50_i64), Some(200_i64))?,
+ ),
+ (
+ Interval::make(Some(-200_i64), Some(-100_i64))?,
+ Interval::make(Some(-2_i64), Some(-1_i64))?,
+ Interval::make(Some(50_i64), Some(200_i64))?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(-2_i64), Some(-1_i64))?,
+ Interval::make(Some(-200_i64), Some(-50_i64))?,
+ ),
+ (
+ Interval::make(Some(-200_i64), Some(-100_i64))?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(-200_i64), Some(-50_i64))?,
+ ),
+ (
+ Interval::make(Some(-200_i64), Some(100_i64))?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(-200_i64), Some(100_i64))?,
+ ),
+ (
+ Interval::make(Some(-100_i64), Some(200_i64))?,
+ Interval::make(Some(1_i64), Some(2_i64))?,
+ Interval::make(Some(-100_i64), Some(200_i64))?,
+ ),
+ (
+ Interval::make(Some(10_i64), Some(20_i64))?,
+ Interval::make::<i64>(None, None)?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(-100_i64), Some(200_i64))?,
+ Interval::make(Some(-1_i64), Some(2_i64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(-100_i64), Some(200_i64))?,
+ Interval::make(Some(-2_i64), Some(1_i64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(0_i64), Some(1_i64))?,
+ Interval::make(Some(100_i64), None)?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(None, Some(0_i64))?,
+ Interval::make(None, Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(0_i64), Some(0_i64))?,
+ Interval::make::<i64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(1_i64))?,
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(0_i64), Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(1_i64))?,
+ Interval::make(Some(100_i64), Some(200_i64))?,
+ Interval::make(Some(0_i64), Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(1_u32), Some(2_u32))?,
+ Interval::make(Some(0_u32), Some(0_u32))?,
+ Interval::make::<u32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(10_u32), Some(20_u32))?,
+ Interval::make(None, Some(2_u32))?,
+ Interval::make(Some(5_u32), None)?,
+ ),
+ (
+ Interval::make(Some(10_u32), Some(20_u32))?,
+ Interval::make(Some(0_u32), Some(2_u32))?,
+ Interval::make(Some(5_u32), None)?,
+ ),
+ (
+ Interval::make(Some(10_u32), Some(20_u32))?,
+ Interval::make(Some(0_u32), Some(0_u32))?,
+ Interval::make::<u32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(12_u64), Some(48_u64))?,
+ Interval::make(Some(10_u64), Some(20_u64))?,
+ Interval::make(Some(0_u64), Some(4_u64))?,
+ ),
+ (
+ Interval::make(Some(12_u64), Some(48_u64))?,
+ Interval::make(None, Some(2_u64))?,
+ Interval::make(Some(6_u64), None)?,
+ ),
+ (
+ Interval::make(Some(12_u64), Some(48_u64))?,
+ Interval::make(Some(0_u64), Some(2_u64))?,
+ Interval::make(Some(6_u64), None)?,
+ ),
+ (
+ Interval::make(None, Some(48_u64))?,
+ Interval::make(Some(0_u64), Some(2_u64))?,
+ Interval::make::<u64>(None, None)?,
+ ),
+ (
+ Interval::make(Some(f32::MAX), Some(f32::MAX))?,
+ Interval::make(Some(-0.1_f32), Some(0.1_f32))?,
+ Interval::make::<f32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), None)?,
+ Interval::make(Some(0.1_f32), Some(0.1_f32))?,
+ Interval::make::<f32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(-10.0_f32), Some(10.0_f32))?,
+ Interval::make(Some(-0.1_f32), Some(-0.1_f32))?,
+ Interval::make(Some(-100.0_f32), Some(100.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-10.0_f32), Some(f32::MAX))?,
+ Interval::make::<f32>(None, None)?,
+ Interval::make::<f32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(f32::MIN), Some(10.0_f32))?,
+ Interval::make(Some(1.0_f32), None)?,
+ Interval::make(Some(f32::MIN), Some(10.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-0.0_f32), Some(0.0_f32))?,
+ Interval::make(Some(f32::MAX), None)?,
+ Interval::make(Some(-0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-0.0_f32), Some(0.0_f32))?,
+ Interval::make(None, Some(-0.0_f32))?,
+ Interval::make::<f32>(None, None)?,
+ ),
+ (
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ Interval::make(Some(f32::MAX), None)?,
+ Interval::make(Some(0.0_f32), Some(0.0_f32))?,
+ ),
+ (
+ Interval::make(Some(1.0_f32), Some(2.0_f32))?,
+ Interval::make(Some(0.0_f32), Some(4.0_f32))?,
+ Interval::make(Some(0.25_f32), None)?,
+ ),
+ (
+ Interval::make(Some(1.0_f32), Some(2.0_f32))?,
+ Interval::make(Some(-4.0_f32), Some(-0.0_f32))?,
+ Interval::make(None, Some(-0.25_f32))?,
+ ),
+ (
+ Interval::make(Some(-4.0_f64), Some(2.0_f64))?,
+ Interval::make(Some(10.0_f64), Some(20.0_f64))?,
+ Interval::make(Some(-0.4_f64), Some(0.2_f64))?,
+ ),
+ (
+ Interval::make(Some(-0.0_f64), Some(-0.0_f64))?,
+ Interval::make(None, Some(-0.0_f64))?,
+ Interval::make(Some(0.0_f64), None)?,
+ ),
+ (
+ Interval::make(Some(1.0_f64), Some(2.0_f64))?,
+ Interval::make::<f64>(None, None)?,
+ Interval::make(Some(0.0_f64), None)?,
+ ),
+ ];
+ for case in cases {
+ let result = case.0.div(case.1)?;
+ if case.0.data_type().is_floating() {
+ assert!(
+ result.lower().is_null() && case.2.lower().is_null()
+ || result.lower().le(case.2.lower())
+ );
+ assert!(
+ result.upper().is_null() && case.2.upper().is_null()
+ || result.upper().ge(case.2.upper())
+ );
+ } else {
+ assert_eq!(result, case.2);
+ }
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_cardinality_of_intervals() -> Result<()> {
+ // In IEEE 754 standard for floating-point arithmetic, if we keep the sign and exponent fields same,
+ // we can represent 4503599627370496+1 different numbers by changing the mantissa
+ // (4503599627370496 = 2^52, since there are 52 bits in mantissa, and 2^23 = 8388608 for f32).
+ // TODO: Add tests for non-exponential boundary aligned intervals too.
+ let distinct_f64 = 4503599627370497;
+ let distinct_f32 = 8388609;
+ let intervals = [
+ Interval::make(Some(0.25_f64), Some(0.50_f64))?,
+ Interval::make(Some(0.5_f64), Some(1.0_f64))?,
+ Interval::make(Some(1.0_f64), Some(2.0_f64))?,
+ Interval::make(Some(32.0_f64), Some(64.0_f64))?,
+ Interval::make(Some(-0.50_f64), Some(-0.25_f64))?,
+ Interval::make(Some(-32.0_f64), Some(-16.0_f64))?,
+ ];
+ for interval in intervals {
+ assert_eq!(interval.cardinality().unwrap(), distinct_f64);
+ }
+
+ let intervals = [
+ Interval::make(Some(0.25_f32), Some(0.50_f32))?,
+ Interval::make(Some(-1_f32), Some(-0.5_f32))?,
+ ];
+ for interval in intervals {
+ assert_eq!(interval.cardinality().unwrap(), distinct_f32);
+ }
+
+ // The regular logarithmic distribution of floating-point numbers are
+ // only applicable outside of the `(-phi, phi)` interval where `phi`
+ // denotes the largest positive subnormal floating-point number. Since
+ // the following intervals include such subnormal points, we cannot use
+ // a simple powers-of-two type formula for our expectations. Therefore,
+ // we manually supply the actual expected cardinality.
+ let interval = Interval::make(Some(-0.0625), Some(0.0625))?;
+ assert_eq!(interval.cardinality().unwrap(), 9178336040581070850);
+
+ let interval = Interval::try_new(
+ ScalarValue::UInt64(Some(u64::MIN + 1)),
+ ScalarValue::UInt64(Some(u64::MAX)),
+ )?;
+ assert_eq!(interval.cardinality().unwrap(), u64::MAX);
+
+ let interval = Interval::try_new(
+ ScalarValue::Int64(Some(i64::MIN + 1)),
+ ScalarValue::Int64(Some(i64::MAX)),
+ )?;
+ assert_eq!(interval.cardinality().unwrap(), u64::MAX);
+
+ let interval = Interval::try_new(
+ ScalarValue::Float32(Some(-0.0_f32)),
+ ScalarValue::Float32(Some(0.0_f32)),
+ )?;
+ assert_eq!(interval.cardinality().unwrap(), 2);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_satisfy_comparison() -> Result<()> {
+ let cases = vec![
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(1000_i64))?,
+ true,
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(1000_i64))?,
+ ),
+ (
+ Interval::make(None, Some(1000_i64))?,
+ Interval::make(Some(1000_i64), None)?,
+ true,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(1000_i64))?,
+ false,
+ Interval::make(Some(1000_i64), None)?,
+ Interval::make(None, Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(1500_i64))?,
+ true,
+ Interval::make(Some(500_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(1500_i64))?,
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ true,
+ Interval::make(Some(500_i64), Some(1500_i64))?,
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(1500_i64))?,
+ false,
+ Interval::make(Some(501_i64), Some(1000_i64))?,
+ Interval::make(Some(500_i64), Some(999_i64))?,
+ ),
+ (
+ Interval::make(Some(500_i64), Some(1500_i64))?,
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ false,
+ Interval::make(Some(500_i64), Some(1500_i64))?,
+ Interval::make(Some(0_i64), Some(1000_i64))?,
+ ),
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ false,
+ Interval::make(Some(2_i64), None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ ),
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ true,
+ Interval::make(Some(1_i64), None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make::<i64>(None, None)?,
+ false,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make(None, Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make::<i64>(None, None)?,
+ true,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make(None, Some(1_i64))?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make::<i64>(None, None)?,
+ false,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make(None, Some(0_i64))?,
+ ),
+ (
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make::<i64>(None, None)?,
+ true,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ Interval::make(None, Some(1_i64))?,
+ ),
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ false,
+ Interval::make(Some(2_i64), None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ ),
+ (
+ Interval::make::<i64>(None, None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ true,
+ Interval::make(Some(1_i64), None)?,
+ Interval::make(Some(1_i64), Some(1_i64))?,
+ ),
+ (
+ Interval::make(Some(-1000.0_f32), Some(1000.0_f32))?,
+ Interval::make(Some(-500.0_f32), Some(500.0_f32))?,
+ false,
+ Interval::try_new(
+ next_value(ScalarValue::Float32(Some(-500.0))),
+ ScalarValue::Float32(Some(1000.0)),
+ )?,
+ Interval::make(Some(-500_f32), Some(500.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-500.0_f32), Some(500.0_f32))?,
+ Interval::make(Some(-1000.0_f32), Some(1000.0_f32))?,
+ true,
+ Interval::make(Some(-500.0_f32), Some(500.0_f32))?,
+ Interval::make(Some(-1000.0_f32), Some(500.0_f32))?,
+ ),
+ (
+ Interval::make(Some(-500.0_f32), Some(500.0_f32))?,
+ Interval::make(Some(-1000.0_f32), Some(1000.0_f32))?,
+ false,
+ Interval::make(Some(-500.0_f32), Some(500.0_f32))?,
+ Interval::try_new(
+ ScalarValue::Float32(Some(-1000.0_f32)),
+ prev_value(ScalarValue::Float32(Some(500.0_f32))),
+ )?,
+ ),
+ (
+ Interval::make(Some(-1000.0_f64), Some(1000.0_f64))?,
+ Interval::make(Some(-500.0_f64), Some(500.0_f64))?,
+ true,
+ Interval::make(Some(-500.0_f64), Some(1000.0_f64))?,
+ Interval::make(Some(-500.0_f64), Some(500.0_f64))?,
+ ),
+ ];
+ for (first, second, includes_endpoints, left_modified, right_modified) in cases {
+ assert_eq!(
+ satisfy_greater(&first, &second, !includes_endpoints)?.unwrap(),
+ (left_modified, right_modified)
+ );
+ }
+
+ let infeasible_cases = vec![
+ (
+ Interval::make(None, Some(1000_i64))?,
+ Interval::make(Some(1000_i64), None)?,
+ false,
+ ),
+ (
+ Interval::make(Some(-1000.0_f32), Some(1000.0_f32))?,
+ Interval::make(Some(1500.0_f32), Some(2000.0_f32))?,
+ false,
+ ),
+ ];
+ for (first, second, includes_endpoints) in infeasible_cases {
+ assert_eq!(satisfy_greater(&first, &second, !includes_endpoints)?, None);
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_interval_display() {
+ let interval = Interval::make(Some(0.25_f32), Some(0.50_f32)).unwrap();
+ assert_eq!(format!("{}", interval), "[0.25, 0.5]");
+
+ let interval = Interval::try_new(
+ ScalarValue::Float32(Some(f32::NEG_INFINITY)),
+ ScalarValue::Float32(Some(f32::INFINITY)),
+ )
+ .unwrap();
+ assert_eq!(format!("{}", interval), "[NULL, NULL]");
+ }
+
+ macro_rules! capture_mode_change {
+ ($TYPE:ty) => {
+ paste::item! {
+ capture_mode_change_helper!([<capture_mode_change_ $TYPE>],
+ [<create_interval_ $TYPE>],
+ $TYPE);
+ }
+ };
+ }
+
+ macro_rules! capture_mode_change_helper {
+ ($TEST_FN_NAME:ident, $CREATE_FN_NAME:ident, $TYPE:ty) => {
+ fn $CREATE_FN_NAME(lower: $TYPE, upper: $TYPE) -> Interval {
+ Interval::try_new(
+ ScalarValue::try_from(Some(lower as $TYPE)).unwrap(),
+ ScalarValue::try_from(Some(upper as $TYPE)).unwrap(),
+ )
+ .unwrap()
+ }
+
+ fn $TEST_FN_NAME(input: ($TYPE, $TYPE), expect_low: bool, expect_high: bool) {
+ assert!(expect_low || expect_high);
+ let interval1 = $CREATE_FN_NAME(input.0, input.0);
+ let interval2 = $CREATE_FN_NAME(input.1, input.1);
+ let result = interval1.add(&interval2).unwrap();
+ let without_fe = $CREATE_FN_NAME(input.0 + input.1, input.0 + input.1);
+ assert!(
+ (!expect_low || result.lower < without_fe.lower)
+ && (!expect_high || result.upper > without_fe.upper)
+ );
+ }
+ };
+ }
+
+ capture_mode_change!(f32);
+ capture_mode_change!(f64);
+
+ #[cfg(all(
+ any(target_arch = "x86_64", target_arch = "aarch64"),
+ not(target_os = "windows")
+ ))]
+ #[test]
+ fn test_add_intervals_lower_affected_f32() {
+ // Lower is affected
+ let lower = f32::from_bits(1073741887); //1000000000000000000000000111111
+ let upper = f32::from_bits(1098907651); //1000001100000000000000000000011
+ capture_mode_change_f32((lower, upper), true, false);
+
+ // Upper is affected
+ let lower = f32::from_bits(1072693248); //111111111100000000000000000000
+ let upper = f32::from_bits(715827883); //101010101010101010101010101011
+ capture_mode_change_f32((lower, upper), false, true);
+
+ // Lower is affected
+ let lower = 1.0; // 0x3FF0000000000000
+ let upper = 0.3; // 0x3FD3333333333333
+ capture_mode_change_f64((lower, upper), true, false);
+
+ // Upper is affected
+ let lower = 1.4999999999999998; // 0x3FF7FFFFFFFFFFFF
+ let upper = 0.000_000_000_000_000_022_044_604_925_031_31; // 0x3C796A6B413BB21F
+ capture_mode_change_f64((lower, upper), false, true);
+ }
+
+ #[cfg(any(
+ not(any(target_arch = "x86_64", target_arch = "aarch64")),
+ target_os = "windows"
+ ))]
+ #[test]
+ fn test_next_impl_add_intervals_f64() {
+ let lower = 1.5;
+ let upper = 1.5;
+ capture_mode_change_f64((lower, upper), true, true);
+
+ let lower = 1.5;
+ let upper = 1.5;
+ capture_mode_change_f32((lower, upper), true, true);
+ }
+}
diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs
index 21c0d750a3..b9976f90c5 100644
--- a/datafusion/expr/src/lib.rs
+++ b/datafusion/expr/src/lib.rs
@@ -26,10 +26,20 @@
//! The [expr_fn] module contains functions for creating expressions.
mod accumulator;
-pub mod aggregate_function;
-pub mod array_expressions;
mod built_in_function;
mod columnar_value;
+mod literal;
+mod nullif;
+mod operator;
+mod partition_evaluator;
+mod signature;
+mod table_source;
+mod udaf;
+mod udf;
+mod udwf;
+
+pub mod aggregate_function;
+pub mod array_expressions;
pub mod conditional_expressions;
pub mod expr;
pub mod expr_fn;
@@ -37,19 +47,11 @@ pub mod expr_rewriter;
pub mod expr_schema;
pub mod field_util;
pub mod function;
-mod literal;
+pub mod interval_arithmetic;
pub mod logical_plan;
-mod nullif;
-mod operator;
-mod partition_evaluator;
-mod signature;
pub mod struct_expressions;
-mod table_source;
pub mod tree_node;
pub mod type_coercion;
-mod udaf;
-mod udf;
-mod udwf;
pub mod utils;
pub mod window_frame;
pub mod window_function;
diff --git a/datafusion/expr/src/type_coercion/binary.rs b/datafusion/expr/src/type_coercion/binary.rs
index cf93d15e23..9ccddbfce0 100644
--- a/datafusion/expr/src/type_coercion/binary.rs
+++ b/datafusion/expr/src/type_coercion/binary.rs
@@ -782,7 +782,6 @@ fn temporal_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataTyp
use arrow::datatypes::TimeUnit::*;
match (lhs_type, rhs_type) {
- // interval +/-
(Interval(_), Interval(_)) => Some(Interval(MonthDayNano)),
(Date64, Date32) | (Date32, Date64) => Some(Date64),
(Timestamp(_, None), Date32) | (Date32, Timestamp(_, None)) => {
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 947a6f6070..ad64625f7f 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -21,9 +21,11 @@ use std::ops::Not;
use super::or_in_list_simplifier::OrInListSimplifier;
use super::utils::*;
-
use crate::analyzer::type_coercion::TypeCoercionRewriter;
+use crate::simplify_expressions::guarantees::GuaranteeRewriter;
use crate::simplify_expressions::regex::simplify_regex_expr;
+use crate::simplify_expressions::SimplifyInfo;
+
use arrow::{
array::new_null_array,
datatypes::{DataType, Field, Schema},
@@ -37,18 +39,15 @@ use datafusion_common::{
use datafusion_common::{
exec_err, internal_err, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue,
};
-use datafusion_expr::expr::{InList, InSubquery, ScalarFunction};
use datafusion_expr::{
and, expr, lit, or, BinaryExpr, BuiltinScalarFunction, Case, ColumnarValue, Expr,
Like, Volatility,
};
-use datafusion_physical_expr::{
- create_physical_expr, execution_props::ExecutionProps, intervals::NullableInterval,
+use datafusion_expr::{
+ expr::{InList, InSubquery, ScalarFunction},
+ interval_arithmetic::NullableInterval,
};
-
-use crate::simplify_expressions::SimplifyInfo;
-
-use crate::simplify_expressions::guarantees::GuaranteeRewriter;
+use datafusion_physical_expr::{create_physical_expr, execution_props::ExecutionProps};
/// This structure handles API for expression simplification
pub struct ExprSimplifier<S> {
@@ -178,9 +177,9 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
/// ```rust
/// use arrow::datatypes::{DataType, Field, Schema};
/// use datafusion_expr::{col, lit, Expr};
+ /// use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};
/// use datafusion_common::{Result, ScalarValue, ToDFSchema};
/// use datafusion_physical_expr::execution_props::ExecutionProps;
- /// use datafusion_physical_expr::intervals::{Interval, NullableInterval};
/// use datafusion_optimizer::simplify_expressions::{
/// ExprSimplifier, SimplifyContext};
///
@@ -207,7 +206,7 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
/// (
/// col("x"),
/// NullableInterval::NotNull {
- /// values: Interval::make(Some(3_i64), Some(5_i64), (false, false)),
+ /// values: Interval::make(Some(3_i64), Some(5_i64)).unwrap()
/// }
/// ),
/// // y = 3
@@ -1300,26 +1299,25 @@ mod tests {
sync::Arc,
};
+ use super::*;
use crate::simplify_expressions::{
utils::for_test::{cast_to_int64_expr, now_expr, to_timestamp_expr},
SimplifyContext,
};
-
- use super::*;
use crate::test::test_table_scan_with_name;
+
use arrow::{
array::{ArrayRef, Int32Array},
datatypes::{DataType, Field, Schema},
};
- use chrono::{DateTime, TimeZone, Utc};
use datafusion_common::{assert_contains, cast::as_int32_array, DFField, ToDFSchema};
- use datafusion_expr::*;
+ use datafusion_expr::{interval_arithmetic::Interval, *};
use datafusion_physical_expr::{
- execution_props::ExecutionProps,
- functions::make_scalar_function,
- intervals::{Interval, NullableInterval},
+ execution_props::ExecutionProps, functions::make_scalar_function,
};
+ use chrono::{DateTime, TimeZone, Utc};
+
// ------------------------------
// --- ExprSimplifier tests -----
// ------------------------------
@@ -3281,7 +3279,7 @@ mod tests {
(
col("c3"),
NullableInterval::NotNull {
- values: Interval::make(Some(0_i64), Some(2_i64), (false, false)),
+ values: Interval::make(Some(0_i64), Some(2_i64)).unwrap(),
},
),
(
@@ -3301,19 +3299,23 @@ mod tests {
(
col("c3"),
NullableInterval::MaybeNull {
- values: Interval::make(Some(0_i64), Some(2_i64), (false, false)),
+ values: Interval::make(Some(0_i64), Some(2_i64)).unwrap(),
},
),
(
col("c4"),
NullableInterval::MaybeNull {
- values: Interval::make(Some(9_u32), Some(9_u32), (false, false)),
+ values: Interval::make(Some(9_u32), Some(9_u32)).unwrap(),
},
),
(
col("c1"),
NullableInterval::NotNull {
- values: Interval::make(Some("d"), Some("f"), (false, false)),
+ values: Interval::try_new(
+ ScalarValue::Utf8(Some("d".to_string())),
+ ScalarValue::Utf8(Some("f".to_string())),
+ )
+ .unwrap(),
},
),
];
diff --git a/datafusion/optimizer/src/simplify_expressions/guarantees.rs b/datafusion/optimizer/src/simplify_expressions/guarantees.rs
index 0204698571..3cfaae858e 100644
--- a/datafusion/optimizer/src/simplify_expressions/guarantees.rs
+++ b/datafusion/optimizer/src/simplify_expressions/guarantees.rs
@@ -18,11 +18,12 @@
//! Simplifier implementation for [`ExprSimplifier::with_guarantees()`]
//!
//! [`ExprSimplifier::with_guarantees()`]: crate::simplify_expressions::expr_simplifier::ExprSimplifier::with_guarantees
-use datafusion_common::{tree_node::TreeNodeRewriter, DataFusionError, Result};
-use datafusion_expr::{expr::InList, lit, Between, BinaryExpr, Expr};
+
use std::{borrow::Cow, collections::HashMap};
-use datafusion_physical_expr::intervals::{Interval, IntervalBound, NullableInterval};
+use datafusion_common::{tree_node::TreeNodeRewriter, DataFusionError, Result};
+use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};
+use datafusion_expr::{expr::InList, lit, Between, BinaryExpr, Expr};
/// Rewrite expressions to incorporate guarantees.
///
@@ -82,10 +83,7 @@ impl<'a> TreeNodeRewriter for GuaranteeRewriter<'a> {
high.as_ref(),
) {
let expr_interval = NullableInterval::NotNull {
- values: Interval::new(
- IntervalBound::new(low.clone(), false),
- IntervalBound::new(high.clone(), false),
- ),
+ values: Interval::try_new(low.clone(), high.clone())?,
};
let contains = expr_interval.contains(*interval)?;
@@ -146,12 +144,8 @@ impl<'a> TreeNodeRewriter for GuaranteeRewriter<'a> {
// Columns (if interval is collapsed to a single value)
Expr::Column(_) => {
- if let Some(col_interval) = self.guarantees.get(&expr) {
- if let Some(value) = col_interval.single_value() {
- Ok(lit(value))
- } else {
- Ok(expr)
- }
+ if let Some(interval) = self.guarantees.get(&expr) {
+ Ok(interval.single_value().map_or(expr, lit))
} else {
Ok(expr)
}
@@ -215,7 +209,7 @@ mod tests {
(
col("x"),
NullableInterval::NotNull {
- values: Default::default(),
+ values: Interval::make_unbounded(&DataType::Boolean).unwrap(),
},
),
];
@@ -262,18 +256,18 @@ mod tests {
#[test]
fn test_inequalities_non_null_bounded() {
let guarantees = vec![
- // x ∈ (1, 3] (not null)
+ // x ∈ [1, 3] (not null)
(
col("x"),
NullableInterval::NotNull {
- values: Interval::make(Some(1_i32), Some(3_i32), (true, false)),
+ values: Interval::make(Some(1_i32), Some(3_i32)).unwrap(),
},
),
- // s.y ∈ (1, 3] (not null)
+ // s.y ∈ [1, 3] (not null)
(
col("s").field("y"),
NullableInterval::NotNull {
- values: Interval::make(Some(1_i32), Some(3_i32), (true, false)),
+ values: Interval::make(Some(1_i32), Some(3_i32)).unwrap(),
},
),
];
@@ -282,18 +276,16 @@ mod tests {
// (original_expr, expected_simplification)
let simplified_cases = &[
- (col("x").lt_eq(lit(1)), false),
- (col("s").field("y").lt_eq(lit(1)), false),
+ (col("x").lt(lit(0)), false),
+ (col("s").field("y").lt(lit(0)), false),
(col("x").lt_eq(lit(3)), true),
(col("x").gt(lit(3)), false),
- (col("x").gt(lit(1)), true),
+ (col("x").gt(lit(0)), true),
(col("x").eq(lit(0)), false),
(col("x").not_eq(lit(0)), true),
- (col("x").between(lit(2), lit(5)), true),
- (col("x").between(lit(2), lit(3)), true),
+ (col("x").between(lit(0), lit(5)), true),
(col("x").between(lit(5), lit(10)), false),
- (col("x").not_between(lit(2), lit(5)), false),
- (col("x").not_between(lit(2), lit(3)), false),
+ (col("x").not_between(lit(0), lit(5)), false),
(col("x").not_between(lit(5), lit(10)), true),
(
Expr::BinaryExpr(BinaryExpr {
@@ -334,10 +326,11 @@ mod tests {
(
col("x"),
NullableInterval::NotNull {
- values: Interval::new(
- IntervalBound::new(ScalarValue::Date32(Some(18628)), false),
- IntervalBound::make_unbounded(DataType::Date32).unwrap(),
- ),
+ values: Interval::try_new(
+ ScalarValue::Date32(Some(18628)),
+ ScalarValue::Date32(None),
+ )
+ .unwrap(),
},
),
];
@@ -412,7 +405,11 @@ mod tests {
(
col("x"),
NullableInterval::MaybeNull {
- values: Interval::make(Some("abc"), Some("def"), (true, false)),
+ values: Interval::try_new(
+ ScalarValue::Utf8(Some("abc".to_string())),
+ ScalarValue::Utf8(Some("def".to_string())),
+ )
+ .unwrap(),
},
),
];
@@ -485,11 +482,15 @@ mod tests {
#[test]
fn test_in_list() {
let guarantees = vec![
- // x ∈ [1, 10) (not null)
+ // x ∈ [1, 10] (not null)
(
col("x"),
NullableInterval::NotNull {
- values: Interval::make(Some(1_i32), Some(10_i32), (false, true)),
+ values: Interval::try_new(
+ ScalarValue::Int32(Some(1)),
+ ScalarValue::Int32(Some(10)),
+ )
+ .unwrap(),
},
),
];
@@ -501,8 +502,8 @@ mod tests {
let cases = &[
// x IN (9, 11) => x IN (9)
("x", vec![9, 11], false, vec![9]),
- // x IN (10, 2) => x IN (2)
- ("x", vec![10, 2], false, vec![2]),
+ // x IN (10, 2) => x IN (10, 2)
+ ("x", vec![10, 2], false, vec![10, 2]),
// x NOT IN (9, 11) => x NOT IN (9)
("x", vec![9, 11], true, vec![9]),
// x NOT IN (0, 22) => x NOT IN ()
diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
index caa812d075..d237c68657 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -34,13 +34,16 @@ path = "src/lib.rs"
[features]
crypto_expressions = ["md-5", "sha2", "blake2", "blake3"]
-default = ["crypto_expressions", "regex_expressions", "unicode_expressions", "encoding_expressions"]
+default = ["crypto_expressions", "regex_expressions", "unicode_expressions", "encoding_expressions",
+]
encoding_expressions = ["base64", "hex"]
regex_expressions = ["regex"]
unicode_expressions = ["unicode-segmentation"]
[dependencies]
-ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
+ahash = { version = "0.8", default-features = false, features = [
+ "runtime-rng",
+] }
arrow = { workspace = true }
arrow-array = { workspace = true }
arrow-buffer = { workspace = true }
@@ -57,7 +60,6 @@ hashbrown = { version = "0.14", features = ["raw"] }
hex = { version = "0.4", optional = true }
indexmap = { workspace = true }
itertools = { version = "0.12", features = ["use_std"] }
-libc = "0.2.140"
log = { workspace = true }
md-5 = { version = "^0.10.0", optional = true }
paste = "^1.0"
diff --git a/datafusion/physical-expr/src/analysis.rs b/datafusion/physical-expr/src/analysis.rs
index 93c24014fd..dc12bdf46a 100644
--- a/datafusion/physical-expr/src/analysis.rs
+++ b/datafusion/physical-expr/src/analysis.rs
@@ -21,8 +21,7 @@ use std::fmt::Debug;
use std::sync::Arc;
use crate::expressions::Column;
-use crate::intervals::cp_solver::PropagationResult;
-use crate::intervals::{cardinality_ratio, ExprIntervalGraph, Interval, IntervalBound};
+use crate::intervals::cp_solver::{ExprIntervalGraph, PropagationResult};
use crate::utils::collect_columns;
use crate::PhysicalExpr;
@@ -31,6 +30,7 @@ use datafusion_common::stats::Precision;
use datafusion_common::{
internal_err, ColumnStatistics, DataFusionError, Result, ScalarValue,
};
+use datafusion_expr::interval_arithmetic::{cardinality_ratio, Interval};
/// The shared context used during the analysis of an expression. Includes
/// the boundaries for all known columns.
@@ -92,22 +92,18 @@ impl ExprBoundaries {
) -> Result<Self> {
let field = &schema.fields()[col_index];
let empty_field = ScalarValue::try_from(field.data_type())?;
- let interval = Interval::new(
- IntervalBound::new_closed(
- col_stats
- .min_value
- .get_value()
- .cloned()
- .unwrap_or(empty_field.clone()),
- ),
- IntervalBound::new_closed(
- col_stats
- .max_value
- .get_value()
- .cloned()
- .unwrap_or(empty_field),
- ),
- );
+ let interval = Interval::try_new(
+ col_stats
+ .min_value
+ .get_value()
+ .cloned()
+ .unwrap_or(empty_field.clone()),
+ col_stats
+ .max_value
+ .get_value()
+ .cloned()
+ .unwrap_or(empty_field),
+ )?;
let column = Column::new(field.name(), col_index);
Ok(ExprBoundaries {
column,
@@ -135,47 +131,44 @@ impl ExprBoundaries {
pub fn analyze(
expr: &Arc<dyn PhysicalExpr>,
context: AnalysisContext,
+ schema: &Schema,
) -> Result<AnalysisContext> {
let target_boundaries = context.boundaries;
- let mut graph = ExprIntervalGraph::try_new(expr.clone())?;
+ let mut graph = ExprIntervalGraph::try_new(expr.clone(), schema)?;
- let columns: Vec<Arc<dyn PhysicalExpr>> = collect_columns(expr)
+ let columns = collect_columns(expr)
.into_iter()
- .map(|c| Arc::new(c) as Arc<dyn PhysicalExpr>)
- .collect();
-
- let target_expr_and_indices: Vec<(Arc<dyn PhysicalExpr>, usize)> =
- graph.gather_node_indices(columns.as_slice());
-
- let mut target_indices_and_boundaries: Vec<(usize, Interval)> =
- target_expr_and_indices
- .iter()
- .filter_map(|(expr, i)| {
- target_boundaries.iter().find_map(|bound| {
- expr.as_any()
- .downcast_ref::<Column>()
- .filter(|expr_column| bound.column.eq(*expr_column))
- .map(|_| (*i, bound.interval.clone()))
- })
+ .map(|c| Arc::new(c) as _)
+ .collect::<Vec<_>>();
+
+ let target_expr_and_indices = graph.gather_node_indices(columns.as_slice());
+
+ let mut target_indices_and_boundaries = target_expr_and_indices
+ .iter()
+ .filter_map(|(expr, i)| {
+ target_boundaries.iter().find_map(|bound| {
+ expr.as_any()
+ .downcast_ref::<Column>()
+ .filter(|expr_column| bound.column.eq(*expr_column))
+ .map(|_| (*i, bound.interval.clone()))
})
- .collect();
- Ok(
- match graph.update_ranges(&mut target_indices_and_boundaries)? {
- PropagationResult::Success => shrink_boundaries(
- expr,
- graph,
- target_boundaries,
- target_expr_and_indices,
- )?,
- PropagationResult::Infeasible => {
- AnalysisContext::new(target_boundaries).with_selectivity(0.0)
- }
- PropagationResult::CannotPropagate => {
- AnalysisContext::new(target_boundaries).with_selectivity(1.0)
- }
- },
- )
+ })
+ .collect::<Vec<_>>();
+
+ match graph
+ .update_ranges(&mut target_indices_and_boundaries, Interval::CERTAINLY_TRUE)?
+ {
+ PropagationResult::Success => {
+ shrink_boundaries(graph, target_boundaries, target_expr_and_indices)
+ }
+ PropagationResult::Infeasible => {
+ Ok(AnalysisContext::new(target_boundaries).with_selectivity(0.0))
+ }
+ PropagationResult::CannotPropagate => {
+ Ok(AnalysisContext::new(target_boundaries).with_selectivity(1.0))
+ }
+ }
}
/// If the `PropagationResult` indicates success, this function calculates the
@@ -183,8 +176,7 @@ pub fn analyze(
/// Following this, it constructs and returns a new `AnalysisContext` with the
/// updated parameters.
fn shrink_boundaries(
- expr: &Arc<dyn PhysicalExpr>,
- mut graph: ExprIntervalGraph,
+ graph: ExprIntervalGraph,
mut target_boundaries: Vec<ExprBoundaries>,
target_expr_and_indices: Vec<(Arc<dyn PhysicalExpr>, usize)>,
) -> Result<AnalysisContext> {
@@ -199,20 +191,12 @@ fn shrink_boundaries(
};
}
});
- let graph_nodes = graph.gather_node_indices(&[expr.clone()]);
- let Some((_, root_index)) = graph_nodes.get(0) else {
- return internal_err!(
- "The ExprIntervalGraph under investigation does not have any nodes."
- );
- };
- let final_result = graph.get_interval(*root_index);
-
- let selectivity = calculate_selectivity(
- &final_result.lower.value,
- &final_result.upper.value,
- &target_boundaries,
- &initial_boundaries,
- )?;
+
+ let selectivity = calculate_selectivity(&target_boundaries, &initial_boundaries);
+
+ if !(0.0..=1.0).contains(&selectivity) {
+ return internal_err!("Selectivity is out of limit: {}", selectivity);
+ }
Ok(AnalysisContext::new(target_boundaries).with_selectivity(selectivity))
}
@@ -220,33 +204,17 @@ fn shrink_boundaries(
/// This function calculates the filter predicate's selectivity by comparing
/// the initial and pruned column boundaries. Selectivity is defined as the
/// ratio of rows in a table that satisfy the filter's predicate.
-///
-/// An exact propagation result at the root, i.e. `[true, true]` or `[false, false]`,
-/// leads to early exit (returning a selectivity value of either 1.0 or 0.0). In such
-/// a case, `[true, true]` indicates that all data values satisfy the predicate (hence,
-/// selectivity is 1.0), and `[false, false]` suggests that no data value meets the
-/// predicate (therefore, selectivity is 0.0).
fn calculate_selectivity(
- lower_value: &ScalarValue,
- upper_value: &ScalarValue,
target_boundaries: &[ExprBoundaries],
initial_boundaries: &[ExprBoundaries],
-) -> Result<f64> {
- match (lower_value, upper_value) {
- (ScalarValue::Boolean(Some(true)), ScalarValue::Boolean(Some(true))) => Ok(1.0),
- (ScalarValue::Boolean(Some(false)), ScalarValue::Boolean(Some(false))) => Ok(0.0),
- _ => {
- // Since the intervals are assumed uniform and the values
- // are not correlated, we need to multiply the selectivities
- // of multiple columns to get the overall selectivity.
- target_boundaries.iter().enumerate().try_fold(
- 1.0,
- |acc, (i, ExprBoundaries { interval, .. })| {
- let temp =
- cardinality_ratio(&initial_boundaries[i].interval, interval)?;
- Ok(acc * temp)
- },
- )
- }
- }
+) -> f64 {
+ // Since the intervals are assumed uniform and the values
+ // are not correlated, we need to multiply the selectivities
+ // of multiple columns to get the overall selectivity.
+ initial_boundaries
+ .iter()
+ .zip(target_boundaries.iter())
+ .fold(1.0, |acc, (initial, target)| {
+ acc * cardinality_ratio(&initial.interval, &target.interval)
+ })
}
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
index 0a05a479e5..9c7fdd2e81 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -23,8 +23,8 @@ use std::{any::Any, sync::Arc};
use crate::array_expressions::{
array_append, array_concat, array_has_all, array_prepend,
};
+use crate::expressions::datum::{apply, apply_cmp};
use crate::intervals::cp_solver::{propagate_arithmetic, propagate_comparison};
-use crate::intervals::{apply_operator, Interval};
use crate::physical_expr::down_cast_any_ref;
use crate::sort_properties::SortProperties;
use crate::PhysicalExpr;
@@ -38,12 +38,13 @@ use arrow::compute::kernels::comparison::regexp_is_match_utf8_scalar;
use arrow::compute::kernels::concat_elements::concat_elements_utf8;
use arrow::datatypes::*;
use arrow::record_batch::RecordBatch;
+
use datafusion_common::cast::as_boolean_array;
use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue};
+use datafusion_expr::interval_arithmetic::{apply_operator, Interval};
use datafusion_expr::type_coercion::binary::get_result_type;
use datafusion_expr::{ColumnarValue, Operator};
-use crate::expressions::datum::{apply, apply_cmp};
use kernels::{
bitwise_and_dyn, bitwise_and_dyn_scalar, bitwise_or_dyn, bitwise_or_dyn_scalar,
bitwise_shift_left_dyn, bitwise_shift_left_dyn_scalar, bitwise_shift_right_dyn,
@@ -338,32 +339,102 @@ impl PhysicalExpr for BinaryExpr {
&self,
interval: &Interval,
children: &[&Interval],
- ) -> Result<Vec<Option<Interval>>> {
+ ) -> Result<Option<Vec<Interval>>> {
// Get children intervals.
let left_interval = children[0];
let right_interval = children[1];
- let (left, right) = if self.op.is_logic_operator() {
- // TODO: Currently, this implementation only supports the AND operator
- // and does not require any further propagation. In the future,
- // upon adding support for additional logical operators, this
- // method will require modification to support propagating the
- // changes accordingly.
- return Ok(vec![]);
- } else if self.op.is_comparison_operator() {
- if interval == &Interval::CERTAINLY_FALSE {
- // TODO: We will handle strictly false clauses by negating
- // the comparison operator (e.g. GT to LE, LT to GE)
- // once open/closed intervals are supported.
- return Ok(vec![]);
+ if self.op.eq(&Operator::And) {
+ if interval.eq(&Interval::CERTAINLY_TRUE) {
+ // A certainly true logical conjunction can only derive from possibly
+ // true operands. Otherwise, we prove infeasability.
+ Ok((!left_interval.eq(&Interval::CERTAINLY_FALSE)
+ && !right_interval.eq(&Interval::CERTAINLY_FALSE))
+ .then(|| vec![Interval::CERTAINLY_TRUE, Interval::CERTAINLY_TRUE]))
+ } else if interval.eq(&Interval::CERTAINLY_FALSE) {
+ // If the logical conjunction is certainly false, one of the
+ // operands must be false. However, it's not always possible to
+ // determine which operand is false, leading to different scenarios.
+
+ // If one operand is certainly true and the other one is uncertain,
+ // then the latter must be certainly false.
+ if left_interval.eq(&Interval::CERTAINLY_TRUE)
+ && right_interval.eq(&Interval::UNCERTAIN)
+ {
+ Ok(Some(vec![
+ Interval::CERTAINLY_TRUE,
+ Interval::CERTAINLY_FALSE,
+ ]))
+ } else if right_interval.eq(&Interval::CERTAINLY_TRUE)
+ && left_interval.eq(&Interval::UNCERTAIN)
+ {
+ Ok(Some(vec![
+ Interval::CERTAINLY_FALSE,
+ Interval::CERTAINLY_TRUE,
+ ]))
+ }
+ // If both children are uncertain, or if one is certainly false,
+ // we cannot conclusively refine their intervals. In this case,
+ // propagation does not result in any interval changes.
+ else {
+ Ok(Some(vec![]))
+ }
+ } else {
+ // An uncertain logical conjunction result can not shrink the
+ // end-points of its children.
+ Ok(Some(vec![]))
+ }
+ } else if self.op.eq(&Operator::Or) {
+ if interval.eq(&Interval::CERTAINLY_FALSE) {
+ // A certainly false logical conjunction can only derive from certainly
+ // false operands. Otherwise, we prove infeasability.
+ Ok((!left_interval.eq(&Interval::CERTAINLY_TRUE)
+ && !right_interval.eq(&Interval::CERTAINLY_TRUE))
+ .then(|| vec![Interval::CERTAINLY_FALSE, Interval::CERTAINLY_FALSE]))
+ } else if interval.eq(&Interval::CERTAINLY_TRUE) {
+ // If the logical disjunction is certainly true, one of the
+ // operands must be true. However, it's not always possible to
+ // determine which operand is true, leading to different scenarios.
+
+ // If one operand is certainly false and the other one is uncertain,
+ // then the latter must be certainly true.
+ if left_interval.eq(&Interval::CERTAINLY_FALSE)
+ && right_interval.eq(&Interval::UNCERTAIN)
+ {
+ Ok(Some(vec![
+ Interval::CERTAINLY_FALSE,
+ Interval::CERTAINLY_TRUE,
+ ]))
+ } else if right_interval.eq(&Interval::CERTAINLY_FALSE)
+ && left_interval.eq(&Interval::UNCERTAIN)
+ {
+ Ok(Some(vec![
+ Interval::CERTAINLY_TRUE,
+ Interval::CERTAINLY_FALSE,
+ ]))
+ }
+ // If both children are uncertain, or if one is certainly true,
+ // we cannot conclusively refine their intervals. In this case,
+ // propagation does not result in any interval changes.
+ else {
+ Ok(Some(vec![]))
+ }
+ } else {
+ // An uncertain logical disjunction result can not shrink the
+ // end-points of its children.
+ Ok(Some(vec![]))
}
- // Propagate the comparison operator.
- propagate_comparison(&self.op, left_interval, right_interval)?
+ } else if self.op.is_comparison_operator() {
+ Ok(
+ propagate_comparison(&self.op, interval, left_interval, right_interval)?
+ .map(|(left, right)| vec![left, right]),
+ )
} else {
- // Propagate the arithmetic operator.
- propagate_arithmetic(&self.op, interval, left_interval, right_interval)?
- };
- Ok(vec![left, right])
+ Ok(
+ propagate_arithmetic(&self.op, interval, left_interval, right_interval)?
+ .map(|(left, right)| vec![left, right]),
+ )
+ }
}
fn dyn_hash(&self, state: &mut dyn Hasher) {
@@ -380,7 +451,7 @@ impl PhysicalExpr for BinaryExpr {
Operator::Minus => left_child.sub(right_child),
Operator::Gt | Operator::GtEq => left_child.gt_or_gteq(right_child),
Operator::Lt | Operator::LtEq => right_child.gt_or_gteq(left_child),
- Operator::And => left_child.and(right_child),
+ Operator::And | Operator::Or => left_child.and_or(right_child),
_ => SortProperties::Unordered,
}
}
diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs
index cbc82cc776..b718b5017c 100644
--- a/datafusion/physical-expr/src/expressions/cast.rs
+++ b/datafusion/physical-expr/src/expressions/cast.rs
@@ -20,18 +20,16 @@ use std::fmt;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
-use crate::intervals::Interval;
use crate::physical_expr::down_cast_any_ref;
use crate::sort_properties::SortProperties;
use crate::PhysicalExpr;
-use arrow::compute;
-use arrow::compute::{kernels, CastOptions};
+use arrow::compute::{can_cast_types, kernels, CastOptions};
use arrow::datatypes::{DataType, Schema};
use arrow::record_batch::RecordBatch;
-use compute::can_cast_types;
use datafusion_common::format::DEFAULT_FORMAT_OPTIONS;
use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue};
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_expr::ColumnarValue;
const DEFAULT_CAST_OPTIONS: CastOptions<'static> = CastOptions {
@@ -129,13 +127,13 @@ impl PhysicalExpr for CastExpr {
&self,
interval: &Interval,
children: &[&Interval],
- ) -> Result<Vec<Option<Interval>>> {
+ ) -> Result<Option<Vec<Interval>>> {
let child_interval = children[0];
// Get child's datatype:
- let cast_type = child_interval.get_datatype()?;
- Ok(vec![Some(
- interval.cast_to(&cast_type, &self.cast_options)?,
- )])
+ let cast_type = child_interval.data_type();
+ Ok(Some(
+ vec![interval.cast_to(&cast_type, &self.cast_options)?],
+ ))
}
fn dyn_hash(&self, state: &mut dyn Hasher) {
@@ -226,6 +224,7 @@ pub fn cast(
mod tests {
use super::*;
use crate::expressions::col;
+
use arrow::{
array::{
Array, Decimal128Array, Float32Array, Float64Array, Int16Array, Int32Array,
@@ -234,6 +233,7 @@ mod tests {
},
datatypes::*,
};
+
use datafusion_common::Result;
// runs an end-to-end test of physical type cast
diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs
index 65b3479411..a59fd1ae3f 100644
--- a/datafusion/physical-expr/src/expressions/negative.rs
+++ b/datafusion/physical-expr/src/expressions/negative.rs
@@ -17,25 +17,26 @@
//! Negation (-) expression
-use crate::intervals::Interval;
+use std::any::Any;
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
+
use crate::physical_expr::down_cast_any_ref;
use crate::sort_properties::SortProperties;
use crate::PhysicalExpr;
+
use arrow::{
compute::kernels::numeric::neg_wrapping,
datatypes::{DataType, Schema},
record_batch::RecordBatch,
};
use datafusion_common::{internal_err, DataFusionError, Result};
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_expr::{
type_coercion::{is_interval, is_null, is_signed_numeric},
ColumnarValue,
};
-use std::any::Any;
-use std::hash::{Hash, Hasher};
-use std::sync::Arc;
-
/// Negative expression
#[derive(Debug, Hash)]
pub struct NegativeExpr {
@@ -108,10 +109,10 @@ impl PhysicalExpr for NegativeExpr {
/// It replaces the upper and lower bounds after multiplying them with -1.
/// Ex: `(a, b]` => `[-b, -a)`
fn evaluate_bounds(&self, children: &[&Interval]) -> Result<Interval> {
- Ok(Interval::new(
- children[0].upper.negate()?,
- children[0].lower.negate()?,
- ))
+ Interval::try_new(
+ children[0].upper().arithmetic_negate()?,
+ children[0].lower().arithmetic_negate()?,
+ )
}
/// Returns a new [`Interval`] of a NegativeExpr that has the existing `interval` given that
@@ -120,12 +121,16 @@ impl PhysicalExpr for NegativeExpr {
&self,
interval: &Interval,
children: &[&Interval],
- ) -> Result<Vec<Option<Interval>>> {
+ ) -> Result<Option<Vec<Interval>>> {
let child_interval = children[0];
- let negated_interval =
- Interval::new(interval.upper.negate()?, interval.lower.negate()?);
+ let negated_interval = Interval::try_new(
+ interval.upper().arithmetic_negate()?,
+ interval.lower().arithmetic_negate()?,
+ )?;
- Ok(vec![child_interval.intersect(negated_interval)?])
+ Ok(child_interval
+ .intersect(negated_interval)?
+ .map(|result| vec![result]))
}
/// The ordering of a [`NegativeExpr`] is simply the reverse of its child.
@@ -167,14 +172,14 @@ pub fn negative(
#[cfg(test)]
mod tests {
use super::*;
- use crate::{
- expressions::{col, Column},
- intervals::Interval,
- };
+ use crate::expressions::{col, Column};
+
use arrow::array::*;
use arrow::datatypes::*;
use arrow_schema::DataType::{Float32, Float64, Int16, Int32, Int64, Int8};
- use datafusion_common::{cast::as_primitive_array, Result};
+ use datafusion_common::cast::as_primitive_array;
+ use datafusion_common::Result;
+
use paste::paste;
macro_rules! test_array_negative_op {
@@ -218,8 +223,8 @@ mod tests {
let negative_expr = NegativeExpr {
arg: Arc::new(Column::new("a", 0)),
};
- let child_interval = Interval::make(Some(-2), Some(1), (true, false));
- let negative_expr_interval = Interval::make(Some(-1), Some(2), (false, true));
+ let child_interval = Interval::make(Some(-2), Some(1))?;
+ let negative_expr_interval = Interval::make(Some(-1), Some(2))?;
assert_eq!(
negative_expr.evaluate_bounds(&[&child_interval])?,
negative_expr_interval
@@ -232,10 +237,9 @@ mod tests {
let negative_expr = NegativeExpr {
arg: Arc::new(Column::new("a", 0)),
};
- let original_child_interval = Interval::make(Some(-2), Some(3), (false, false));
- let negative_expr_interval = Interval::make(Some(0), Some(4), (true, false));
- let after_propagation =
- vec![Some(Interval::make(Some(-2), Some(0), (false, true)))];
+ let original_child_interval = Interval::make(Some(-2), Some(3))?;
+ let negative_expr_interval = Interval::make(Some(0), Some(4))?;
+ let after_propagation = Some(vec![Interval::make(Some(-2), Some(0))?]);
assert_eq!(
negative_expr.propagate_constraints(
&negative_expr_interval,
diff --git a/datafusion/physical-expr/src/intervals/cp_solver.rs b/datafusion/physical-expr/src/intervals/cp_solver.rs
index e7515341c5..5064ad8d5c 100644
--- a/datafusion/physical-expr/src/intervals/cp_solver.rs
+++ b/datafusion/physical-expr/src/intervals/cp_solver.rs
@@ -24,15 +24,13 @@ use std::sync::Arc;
use super::utils::{
convert_duration_type_to_interval, convert_interval_type_to_duration, get_inverse_op,
};
-use super::IntervalBound;
use crate::expressions::Literal;
-use crate::intervals::interval_aritmetic::{apply_operator, Interval};
use crate::utils::{build_dag, ExprTreeNode};
use crate::PhysicalExpr;
-use arrow_schema::DataType;
-use datafusion_common::{DataFusionError, Result, ScalarValue};
-use datafusion_expr::type_coercion::binary::get_result_type;
+use arrow_schema::{DataType, Schema};
+use datafusion_common::{internal_err, DataFusionError, Result};
+use datafusion_expr::interval_arithmetic::{apply_operator, satisfy_greater, Interval};
use datafusion_expr::Operator;
use petgraph::graph::NodeIndex;
@@ -148,7 +146,7 @@ pub enum PropagationResult {
}
/// This is a node in the DAEG; it encapsulates a reference to the actual
-/// [PhysicalExpr] as well as an interval containing expression bounds.
+/// [`PhysicalExpr`] as well as an interval containing expression bounds.
#[derive(Clone, Debug)]
pub struct ExprIntervalGraphNode {
expr: Arc<dyn PhysicalExpr>,
@@ -163,11 +161,9 @@ impl Display for ExprIntervalGraphNode {
impl ExprIntervalGraphNode {
/// Constructs a new DAEG node with an [-∞, ∞] range.
- pub fn new(expr: Arc<dyn PhysicalExpr>) -> Self {
- ExprIntervalGraphNode {
- expr,
- interval: Interval::default(),
- }
+ pub fn new_unbounded(expr: Arc<dyn PhysicalExpr>, dt: &DataType) -> Result<Self> {
+ Interval::make_unbounded(dt)
+ .map(|interval| ExprIntervalGraphNode { expr, interval })
}
/// Constructs a new DAEG node with the given range.
@@ -180,26 +176,24 @@ impl ExprIntervalGraphNode {
&self.interval
}
- /// This function creates a DAEG node from Datafusion's [ExprTreeNode]
+ /// This function creates a DAEG node from Datafusion's [`ExprTreeNode`]
/// object. Literals are created with definite, singleton intervals while
/// any other expression starts with an indefinite interval ([-∞, ∞]).
- pub fn make_node(node: &ExprTreeNode<NodeIndex>) -> ExprIntervalGraphNode {
+ pub fn make_node(node: &ExprTreeNode<NodeIndex>, schema: &Schema) -> Result<Self> {
let expr = node.expression().clone();
if let Some(literal) = expr.as_any().downcast_ref::<Literal>() {
let value = literal.value();
- let interval = Interval::new(
- IntervalBound::new_closed(value.clone()),
- IntervalBound::new_closed(value.clone()),
- );
- ExprIntervalGraphNode::new_with_interval(expr, interval)
+ Interval::try_new(value.clone(), value.clone())
+ .map(|interval| Self::new_with_interval(expr, interval))
} else {
- ExprIntervalGraphNode::new(expr)
+ expr.data_type(schema)
+ .and_then(|dt| Self::new_unbounded(expr, &dt))
}
}
}
impl PartialEq for ExprIntervalGraphNode {
- fn eq(&self, other: &ExprIntervalGraphNode) -> bool {
+ fn eq(&self, other: &Self) -> bool {
self.expr.eq(&other.expr)
}
}
@@ -216,16 +210,23 @@ impl PartialEq for ExprIntervalGraphNode {
/// - For minus operation, specifically, we would first do
/// - [xL, xU] <- ([yL, yU] + [pL, pU]) ∩ [xL, xU], and then
/// - [yL, yU] <- ([xL, xU] - [pL, pU]) ∩ [yL, yU].
+/// - For multiplication operation, specifically, we would first do
+/// - [xL, xU] <- ([pL, pU] / [yL, yU]) ∩ [xL, xU], and then
+/// - [yL, yU] <- ([pL, pU] / [xL, xU]) ∩ [yL, yU].
+/// - For division operation, specifically, we would first do
+/// - [xL, xU] <- ([yL, yU] * [pL, pU]) ∩ [xL, xU], and then
+/// - [yL, yU] <- ([xL, xU] / [pL, pU]) ∩ [yL, yU].
pub fn propagate_arithmetic(
op: &Operator,
parent: &Interval,
left_child: &Interval,
right_child: &Interval,
-) -> Result<(Option<Interval>, Option<Interval>)> {
- let inverse_op = get_inverse_op(*op);
- match (left_child.get_datatype()?, right_child.get_datatype()?) {
- // If we have a child whose type is a time interval (i.e. DataType::Interval), we need special handling
- // since timestamp differencing results in a Duration type.
+) -> Result<Option<(Interval, Interval)>> {
+ let inverse_op = get_inverse_op(*op)?;
+ match (left_child.data_type(), right_child.data_type()) {
+ // If we have a child whose type is a time interval (i.e. DataType::Interval),
+ // we need special handling since timestamp differencing results in a
+ // Duration type.
(DataType::Timestamp(..), DataType::Interval(_)) => {
propagate_time_interval_at_right(
left_child,
@@ -250,87 +251,109 @@ pub fn propagate_arithmetic(
.intersect(left_child)?
{
// Left is feasible:
- Some(value) => {
+ Some(value) => Ok(
// Propagate to the right using the new left.
- let right =
- propagate_right(&value, parent, right_child, op, &inverse_op)?;
-
- // Return intervals for both children:
- Ok((Some(value), right))
- }
+ propagate_right(&value, parent, right_child, op, &inverse_op)?
+ .map(|right| (value, right)),
+ ),
// If the left child is infeasible, short-circuit.
- None => Ok((None, None)),
+ None => Ok(None),
}
}
}
}
-/// This function provides a target parent interval for comparison operators.
-/// If we have expression > 0, expression must have the range (0, ∞).
-/// If we have expression >= 0, expression must have the range [0, ∞).
-/// If we have expression < 0, expression must have the range (-∞, 0).
-/// If we have expression <= 0, expression must have the range (-∞, 0].
-fn comparison_operator_target(
- left_datatype: &DataType,
- op: &Operator,
- right_datatype: &DataType,
-) -> Result<Interval> {
- let datatype = get_result_type(left_datatype, &Operator::Minus, right_datatype)?;
- let unbounded = IntervalBound::make_unbounded(&datatype)?;
- let zero = ScalarValue::new_zero(&datatype)?;
- Ok(match *op {
- Operator::GtEq => Interval::new(IntervalBound::new_closed(zero), unbounded),
- Operator::Gt => Interval::new(IntervalBound::new_open(zero), unbounded),
- Operator::LtEq => Interval::new(unbounded, IntervalBound::new_closed(zero)),
- Operator::Lt => Interval::new(unbounded, IntervalBound::new_open(zero)),
- Operator::Eq => Interval::new(
- IntervalBound::new_closed(zero.clone()),
- IntervalBound::new_closed(zero),
- ),
- _ => unreachable!(),
- })
-}
-
-/// This function propagates constraints arising from comparison operators.
-/// The main idea is that we can analyze an inequality like x > y through the
-/// equivalent inequality x - y > 0. Assuming that x and y has ranges [xL, xU]
-/// and [yL, yU], we simply apply constraint propagation across [xL, xU],
-/// [yL, yH] and [0, ∞]. Specifically, we would first do
-/// - [xL, xU] <- ([yL, yU] + [0, ∞]) ∩ [xL, xU], and then
-/// - [yL, yU] <- ([xL, xU] - [0, ∞]) ∩ [yL, yU].
+/// This function refines intervals `left_child` and `right_child` by applying
+/// comparison propagation through `parent` via operation. The main idea is
+/// that we can shrink ranges of variables x and y using parent interval p.
+/// Two intervals can be ordered in 6 ways for a Gt `>` operator:
+/// ```text
+/// (1): Infeasible, short-circuit
+/// left: | ================ |
+/// right: | ======================== |
+///
+/// (2): Update both interval
+/// left: | ====================== |
+/// right: | ====================== |
+/// |
+/// V
+/// left: | ======= |
+/// right: | ======= |
+///
+/// (3): Update left interval
+/// left: | ============================== |
+/// right: | ========== |
+/// |
+/// V
+/// left: | ===================== |
+/// right: | ========== |
+///
+/// (4): Update right interval
+/// left: | ========== |
+/// right: | =========================== |
+/// |
+/// V
+/// left: | ========== |
+/// right | ================== |
+///
+/// (5): No change
+/// left: | ============================ |
+/// right: | =================== |
+///
+/// (6): No change
+/// left: | ==================== |
+/// right: | =============== |
+///
+/// -inf --------------------------------------------------------------- +inf
+/// ```
pub fn propagate_comparison(
op: &Operator,
+ parent: &Interval,
left_child: &Interval,
right_child: &Interval,
-) -> Result<(Option<Interval>, Option<Interval>)> {
- let left_type = left_child.get_datatype()?;
- let right_type = right_child.get_datatype()?;
- let parent = comparison_operator_target(&left_type, op, &right_type)?;
- match (&left_type, &right_type) {
- // We can not compare a Duration type with a time interval type
- // without a reference timestamp unless the latter has a zero month field.
- (DataType::Interval(_), DataType::Duration(_)) => {
- propagate_comparison_to_time_interval_at_left(
- left_child,
- &parent,
- right_child,
- )
+) -> Result<Option<(Interval, Interval)>> {
+ if parent == &Interval::CERTAINLY_TRUE {
+ match op {
+ Operator::Eq => left_child.intersect(right_child).map(|result| {
+ result.map(|intersection| (intersection.clone(), intersection))
+ }),
+ Operator::Gt => satisfy_greater(left_child, right_child, true),
+ Operator::GtEq => satisfy_greater(left_child, right_child, false),
+ Operator::Lt => satisfy_greater(right_child, left_child, true)
+ .map(|t| t.map(reverse_tuple)),
+ Operator::LtEq => satisfy_greater(right_child, left_child, false)
+ .map(|t| t.map(reverse_tuple)),
+ _ => internal_err!(
+ "The operator must be a comparison operator to propagate intervals"
+ ),
}
- (DataType::Duration(_), DataType::Interval(_)) => {
- propagate_comparison_to_time_interval_at_left(
- left_child,
- &parent,
- right_child,
- )
+ } else if parent == &Interval::CERTAINLY_FALSE {
+ match op {
+ Operator::Eq => {
+ // TODO: Propagation is not possible until we support interval sets.
+ Ok(None)
+ }
+ Operator::Gt => satisfy_greater(right_child, left_child, false),
+ Operator::GtEq => satisfy_greater(right_child, left_child, true),
+ Operator::Lt => satisfy_greater(left_child, right_child, false)
+ .map(|t| t.map(reverse_tuple)),
+ Operator::LtEq => satisfy_greater(left_child, right_child, true)
+ .map(|t| t.map(reverse_tuple)),
+ _ => internal_err!(
+ "The operator must be a comparison operator to propagate intervals"
+ ),
}
- _ => propagate_arithmetic(&Operator::Minus, &parent, left_child, right_child),
+ } else {
+ // Uncertainty cannot change any end-point of the intervals.
+ Ok(None)
}
}
impl ExprIntervalGraph {
- pub fn try_new(expr: Arc<dyn PhysicalExpr>) -> Result<Self> {
+ pub fn try_new(expr: Arc<dyn PhysicalExpr>, schema: &Schema) -> Result<Self> {
// Build the full graph:
- let (root, graph) = build_dag(expr, &ExprIntervalGraphNode::make_node)?;
+ let (root, graph) =
+ build_dag(expr, &|node| ExprIntervalGraphNode::make_node(node, schema))?;
Ok(Self { graph, root })
}
@@ -383,7 +406,7 @@ impl ExprIntervalGraph {
//
// ```
- /// This function associates stable node indices with [PhysicalExpr]s so
+ /// This function associates stable node indices with [`PhysicalExpr`]s so
/// that we can match `Arc<dyn PhysicalExpr>` and NodeIndex objects during
/// membership tests.
pub fn gather_node_indices(
@@ -437,6 +460,33 @@ impl ExprIntervalGraph {
nodes
}
+ /// Updates intervals for all expressions in the DAEG by successive
+ /// bottom-up and top-down traversals.
+ pub fn update_ranges(
+ &mut self,
+ leaf_bounds: &mut [(usize, Interval)],
+ given_range: Interval,
+ ) -> Result<PropagationResult> {
+ self.assign_intervals(leaf_bounds);
+ let bounds = self.evaluate_bounds()?;
+ // There are three possible cases to consider:
+ // (1) given_range ⊇ bounds => Nothing to propagate
+ // (2) ∅ ⊂ (given_range ∩ bounds) ⊂ bounds => Can propagate
+ // (3) Disjoint sets => Infeasible
+ if given_range.contains(bounds)? == Interval::CERTAINLY_TRUE {
+ // First case:
+ Ok(PropagationResult::CannotPropagate)
+ } else if bounds.contains(&given_range)? != Interval::CERTAINLY_FALSE {
+ // Second case:
+ let result = self.propagate_constraints(given_range);
+ self.update_intervals(leaf_bounds);
+ result
+ } else {
+ // Third case:
+ Ok(PropagationResult::Infeasible)
+ }
+ }
+
/// This function assigns given ranges to expressions in the DAEG.
/// The argument `assignments` associates indices of sought expressions
/// with their corresponding new ranges.
@@ -466,34 +516,43 @@ impl ExprIntervalGraph {
/// # Examples
///
/// ```
- /// use std::sync::Arc;
- /// use datafusion_common::ScalarValue;
- /// use datafusion_expr::Operator;
- /// use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal};
- /// use datafusion_physical_expr::intervals::{Interval, IntervalBound, ExprIntervalGraph};
- /// use datafusion_physical_expr::PhysicalExpr;
- /// let expr = Arc::new(BinaryExpr::new(
- /// Arc::new(Column::new("gnz", 0)),
- /// Operator::Plus,
- /// Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),
- /// ));
- /// let mut graph = ExprIntervalGraph::try_new(expr).unwrap();
- /// // Do it once, while constructing.
- /// let node_indices = graph
+ /// use arrow::datatypes::DataType;
+ /// use arrow::datatypes::Field;
+ /// use arrow::datatypes::Schema;
+ /// use datafusion_common::ScalarValue;
+ /// use datafusion_expr::interval_arithmetic::Interval;
+ /// use datafusion_expr::Operator;
+ /// use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal};
+ /// use datafusion_physical_expr::intervals::cp_solver::ExprIntervalGraph;
+ /// use datafusion_physical_expr::PhysicalExpr;
+ /// use std::sync::Arc;
+ ///
+ /// let expr = Arc::new(BinaryExpr::new(
+ /// Arc::new(Column::new("gnz", 0)),
+ /// Operator::Plus,
+ /// Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),
+ /// ));
+ ///
+ /// let schema = Schema::new(vec![Field::new("gnz".to_string(), DataType::Int32, true)]);
+ ///
+ /// let mut graph = ExprIntervalGraph::try_new(expr, &schema).unwrap();
+ /// // Do it once, while constructing.
+ /// let node_indices = graph
/// .gather_node_indices(&[Arc::new(Column::new("gnz", 0))]);
- /// let left_index = node_indices.get(0).unwrap().1;
- /// // Provide intervals for leaf variables (here, there is only one).
- /// let intervals = vec![(
+ /// let left_index = node_indices.get(0).unwrap().1;
+ ///
+ /// // Provide intervals for leaf variables (here, there is only one).
+ /// let intervals = vec![(
/// left_index,
- /// Interval::make(Some(10), Some(20), (true, true)),
- /// )];
- /// // Evaluate bounds for the composite expression:
- /// graph.assign_intervals(&intervals);
- /// assert_eq!(
- /// graph.evaluate_bounds().unwrap(),
- /// &Interval::make(Some(20), Some(30), (true, true)),
- /// )
+ /// Interval::make(Some(10), Some(20)).unwrap(),
+ /// )];
///
+ /// // Evaluate bounds for the composite expression:
+ /// graph.assign_intervals(&intervals);
+ /// assert_eq!(
+ /// graph.evaluate_bounds().unwrap(),
+ /// &Interval::make(Some(20), Some(30)).unwrap(),
+ /// )
/// ```
pub fn evaluate_bounds(&mut self) -> Result<&Interval> {
let mut dfs = DfsPostOrder::new(&self.graph, self.root);
@@ -505,7 +564,7 @@ impl ExprIntervalGraph {
// If the current expression is a leaf, its interval should already
// be set externally, just continue with the evaluation procedure:
if !children_intervals.is_empty() {
- // Reverse to align with [PhysicalExpr]'s children:
+ // Reverse to align with `PhysicalExpr`'s children:
children_intervals.reverse();
self.graph[node].interval =
self.graph[node].expr.evaluate_bounds(&children_intervals)?;
@@ -516,8 +575,19 @@ impl ExprIntervalGraph {
/// Updates/shrinks bounds for leaf expressions using interval arithmetic
/// via a top-down traversal.
- fn propagate_constraints(&mut self) -> Result<PropagationResult> {
+ fn propagate_constraints(
+ &mut self,
+ given_range: Interval,
+ ) -> Result<PropagationResult> {
let mut bfs = Bfs::new(&self.graph, self.root);
+
+ // Adjust the root node with the given range:
+ if let Some(interval) = self.graph[self.root].interval.intersect(given_range)? {
+ self.graph[self.root].interval = interval;
+ } else {
+ return Ok(PropagationResult::Infeasible);
+ }
+
while let Some(node) = bfs.next(&self.graph) {
let neighbors = self.graph.neighbors_directed(node, Outgoing);
let mut children = neighbors.collect::<Vec<_>>();
@@ -526,7 +596,7 @@ impl ExprIntervalGraph {
if children.is_empty() {
continue;
}
- // Reverse to align with [PhysicalExpr]'s children:
+ // Reverse to align with `PhysicalExpr`'s children:
children.reverse();
let children_intervals = children
.iter()
@@ -536,164 +606,132 @@ impl ExprIntervalGraph {
let propagated_intervals = self.graph[node]
.expr
.propagate_constraints(node_interval, &children_intervals)?;
- for (child, interval) in children.into_iter().zip(propagated_intervals) {
- if let Some(interval) = interval {
+ if let Some(propagated_intervals) = propagated_intervals {
+ for (child, interval) in children.into_iter().zip(propagated_intervals) {
self.graph[child].interval = interval;
- } else {
- // The constraint is infeasible, report:
- return Ok(PropagationResult::Infeasible);
}
+ } else {
+ // The constraint is infeasible, report:
+ return Ok(PropagationResult::Infeasible);
}
}
Ok(PropagationResult::Success)
}
- /// Updates intervals for all expressions in the DAEG by successive
- /// bottom-up and top-down traversals.
- pub fn update_ranges(
- &mut self,
- leaf_bounds: &mut [(usize, Interval)],
- ) -> Result<PropagationResult> {
- self.assign_intervals(leaf_bounds);
- let bounds = self.evaluate_bounds()?;
- if bounds == &Interval::CERTAINLY_FALSE {
- Ok(PropagationResult::Infeasible)
- } else if bounds == &Interval::UNCERTAIN {
- let result = self.propagate_constraints();
- self.update_intervals(leaf_bounds);
- result
- } else {
- Ok(PropagationResult::CannotPropagate)
- }
- }
-
/// Returns the interval associated with the node at the given `index`.
pub fn get_interval(&self, index: usize) -> Interval {
self.graph[NodeIndex::new(index)].interval.clone()
}
}
-/// During the propagation of [`Interval`] values on an [`ExprIntervalGraph`], if there exists a `timestamp - timestamp`
-/// operation, the result would be of type `Duration`. However, we may encounter a situation where a time interval
-/// is involved in an arithmetic operation with a `Duration` type. This function offers special handling for such cases,
-/// where the time interval resides on the left side of the operation.
+/// This is a subfunction of the `propagate_arithmetic` function that propagates to the right child.
+fn propagate_right(
+ left: &Interval,
+ parent: &Interval,
+ right: &Interval,
+ op: &Operator,
+ inverse_op: &Operator,
+) -> Result<Option<Interval>> {
+ match op {
+ Operator::Minus => apply_operator(op, left, parent),
+ Operator::Plus => apply_operator(inverse_op, parent, left),
+ Operator::Divide => apply_operator(op, left, parent),
+ Operator::Multiply => apply_operator(inverse_op, parent, left),
+ _ => internal_err!("Interval arithmetic does not support the operator {}", op),
+ }?
+ .intersect(right)
+}
+
+/// During the propagation of [`Interval`] values on an [`ExprIntervalGraph`],
+/// if there exists a `timestamp - timestamp` operation, the result would be
+/// of type `Duration`. However, we may encounter a situation where a time interval
+/// is involved in an arithmetic operation with a `Duration` type. This function
+/// offers special handling for such cases, where the time interval resides on
+/// the left side of the operation.
fn propagate_time_interval_at_left(
left_child: &Interval,
right_child: &Interval,
parent: &Interval,
op: &Operator,
inverse_op: &Operator,
-) -> Result<(Option<Interval>, Option<Interval>)> {
+) -> Result<Option<(Interval, Interval)>> {
// We check if the child's time interval(s) has a non-zero month or day field(s).
// If so, we return it as is without propagating. Otherwise, we first convert
- // the time intervals to the Duration type, then propagate, and then convert the bounds to time intervals again.
- if let Some(duration) = convert_interval_type_to_duration(left_child) {
+ // the time intervals to the `Duration` type, then propagate, and then convert
+ // the bounds to time intervals again.
+ let result = if let Some(duration) = convert_interval_type_to_duration(left_child) {
match apply_operator(inverse_op, parent, right_child)?.intersect(duration)? {
Some(value) => {
+ let left = convert_duration_type_to_interval(&value);
let right = propagate_right(&value, parent, right_child, op, inverse_op)?;
- let new_interval = convert_duration_type_to_interval(&value);
- Ok((new_interval, right))
+ match (left, right) {
+ (Some(left), Some(right)) => Some((left, right)),
+ _ => None,
+ }
}
- None => Ok((None, None)),
+ None => None,
}
} else {
- let right = propagate_right(left_child, parent, right_child, op, inverse_op)?;
- Ok((Some(left_child.clone()), right))
- }
+ propagate_right(left_child, parent, right_child, op, inverse_op)?
+ .map(|right| (left_child.clone(), right))
+ };
+ Ok(result)
}
-/// During the propagation of [`Interval`] values on an [`ExprIntervalGraph`], if there exists a `timestamp - timestamp`
-/// operation, the result would be of type `Duration`. However, we may encounter a situation where a time interval
-/// is involved in an arithmetic operation with a `Duration` type. This function offers special handling for such cases,
-/// where the time interval resides on the right side of the operation.
+/// During the propagation of [`Interval`] values on an [`ExprIntervalGraph`],
+/// if there exists a `timestamp - timestamp` operation, the result would be
+/// of type `Duration`. However, we may encounter a situation where a time interval
+/// is involved in an arithmetic operation with a `Duration` type. This function
+/// offers special handling for such cases, where the time interval resides on
+/// the right side of the operation.
fn propagate_time_interval_at_right(
left_child: &Interval,
right_child: &Interval,
parent: &Interval,
op: &Operator,
inverse_op: &Operator,
-) -> Result<(Option<Interval>, Option<Interval>)> {
+) -> Result<Option<(Interval, Interval)>> {
// We check if the child's time interval(s) has a non-zero month or day field(s).
// If so, we return it as is without propagating. Otherwise, we first convert
- // the time intervals to the Duration type, then propagate, and then convert the bounds to time intervals again.
- if let Some(duration) = convert_interval_type_to_duration(right_child) {
+ // the time intervals to the `Duration` type, then propagate, and then convert
+ // the bounds to time intervals again.
+ let result = if let Some(duration) = convert_interval_type_to_duration(right_child) {
match apply_operator(inverse_op, parent, &duration)?.intersect(left_child)? {
Some(value) => {
- let right =
- propagate_right(left_child, parent, &duration, op, inverse_op)?;
- let right =
- right.and_then(|right| convert_duration_type_to_interval(&right));
- Ok((Some(value), right))
+ propagate_right(left_child, parent, &duration, op, inverse_op)?
+ .and_then(|right| convert_duration_type_to_interval(&right))
+ .map(|right| (value, right))
}
- None => Ok((None, None)),
+ None => None,
}
} else {
- match apply_operator(inverse_op, parent, right_child)?.intersect(left_child)? {
- Some(value) => Ok((Some(value), Some(right_child.clone()))),
- None => Ok((None, None)),
- }
- }
+ apply_operator(inverse_op, parent, right_child)?
+ .intersect(left_child)?
+ .map(|value| (value, right_child.clone()))
+ };
+ Ok(result)
}
-/// This is a subfunction of the `propagate_arithmetic` function that propagates to the right child.
-fn propagate_right(
- left: &Interval,
- parent: &Interval,
- right: &Interval,
- op: &Operator,
- inverse_op: &Operator,
-) -> Result<Option<Interval>> {
- match op {
- Operator::Minus => apply_operator(op, left, parent),
- Operator::Plus => apply_operator(inverse_op, parent, left),
- _ => unreachable!(),
- }?
- .intersect(right)
-}
-
-/// Converts the `time interval` (as the left child) to duration, then performs the propagation rule for comparison operators.
-pub fn propagate_comparison_to_time_interval_at_left(
- left_child: &Interval,
- parent: &Interval,
- right_child: &Interval,
-) -> Result<(Option<Interval>, Option<Interval>)> {
- if let Some(converted) = convert_interval_type_to_duration(left_child) {
- propagate_arithmetic(&Operator::Minus, parent, &converted, right_child)
- } else {
- Err(DataFusionError::Internal(
- "Interval type has a non-zero month field, cannot compare with a Duration type".to_string(),
- ))
- }
-}
-
-/// Converts the `time interval` (as the right child) to duration, then performs the propagation rule for comparison operators.
-pub fn propagate_comparison_to_time_interval_at_right(
- left_child: &Interval,
- parent: &Interval,
- right_child: &Interval,
-) -> Result<(Option<Interval>, Option<Interval>)> {
- if let Some(converted) = convert_interval_type_to_duration(right_child) {
- propagate_arithmetic(&Operator::Minus, parent, left_child, &converted)
- } else {
- Err(DataFusionError::Internal(
- "Interval type has a non-zero month field, cannot compare with a Duration type".to_string(),
- ))
- }
+fn reverse_tuple<T, U>((first, second): (T, U)) -> (U, T) {
+ (second, first)
}
#[cfg(test)]
mod tests {
use super::*;
- use itertools::Itertools;
-
use crate::expressions::{BinaryExpr, Column};
use crate::intervals::test_utils::gen_conjunctive_numerical_expr;
+
use arrow::datatypes::TimeUnit;
+ use arrow_schema::{DataType, Field};
use datafusion_common::ScalarValue;
+
+ use itertools::Itertools;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use rstest::*;
+ #[allow(clippy::too_many_arguments)]
fn experiment(
expr: Arc<dyn PhysicalExpr>,
exprs_with_interval: (Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>),
@@ -702,6 +740,7 @@ mod tests {
left_expected: Interval,
right_expected: Interval,
result: PropagationResult,
+ schema: &Schema,
) -> Result<()> {
let col_stats = vec![
(exprs_with_interval.0.clone(), left_interval),
@@ -711,7 +750,7 @@ mod tests {
(exprs_with_interval.0.clone(), left_expected),
(exprs_with_interval.1.clone(), right_expected),
];
- let mut graph = ExprIntervalGraph::try_new(expr)?;
+ let mut graph = ExprIntervalGraph::try_new(expr, schema)?;
let expr_indexes = graph
.gather_node_indices(&col_stats.iter().map(|(e, _)| e.clone()).collect_vec());
@@ -726,14 +765,37 @@ mod tests {
.map(|((_, interval), (_, index))| (*index, interval.clone()))
.collect_vec();
- let exp_result = graph.update_ranges(&mut col_stat_nodes[..])?;
+ let exp_result =
+ graph.update_ranges(&mut col_stat_nodes[..], Interval::CERTAINLY_TRUE)?;
assert_eq!(exp_result, result);
col_stat_nodes.iter().zip(expected_nodes.iter()).for_each(
|((_, calculated_interval_node), (_, expected))| {
// NOTE: These randomized tests only check for conservative containment,
// not openness/closedness of endpoints.
- assert!(calculated_interval_node.lower.value <= expected.lower.value);
- assert!(calculated_interval_node.upper.value >= expected.upper.value);
+
+ // Calculated bounds are relaxed by 1 to cover all strict and
+ // and non-strict comparison cases since we have only closed bounds.
+ let one = ScalarValue::new_one(&expected.data_type()).unwrap();
+ assert!(
+ calculated_interval_node.lower()
+ <= &expected.lower().add(&one).unwrap(),
+ "{}",
+ format!(
+ "Calculated {} must be less than or equal {}",
+ calculated_interval_node.lower(),
+ expected.lower()
+ )
+ );
+ assert!(
+ calculated_interval_node.upper()
+ >= &expected.upper().sub(&one).unwrap(),
+ "{}",
+ format!(
+ "Calculated {} must be greater than or equal {}",
+ calculated_interval_node.upper(),
+ expected.upper()
+ )
+ );
},
);
Ok(())
@@ -773,12 +835,24 @@ mod tests {
experiment(
expr,
- (left_col, right_col),
- Interval::make(left_given.0, left_given.1, (true, true)),
- Interval::make(right_given.0, right_given.1, (true, true)),
- Interval::make(left_expected.0, left_expected.1, (true, true)),
- Interval::make(right_expected.0, right_expected.1, (true, true)),
+ (left_col.clone(), right_col.clone()),
+ Interval::make(left_given.0, left_given.1).unwrap(),
+ Interval::make(right_given.0, right_given.1).unwrap(),
+ Interval::make(left_expected.0, left_expected.1).unwrap(),
+ Interval::make(right_expected.0, right_expected.1).unwrap(),
PropagationResult::Success,
+ &Schema::new(vec![
+ Field::new(
+ left_col.as_any().downcast_ref::<Column>().unwrap().name(),
+ DataType::$SCALAR,
+ true,
+ ),
+ Field::new(
+ right_col.as_any().downcast_ref::<Column>().unwrap().name(),
+ DataType::$SCALAR,
+ true,
+ ),
+ ]),
)
}
};
@@ -802,12 +876,24 @@ mod tests {
let expr = Arc::new(BinaryExpr::new(left_and_1, Operator::Gt, right_col.clone()));
experiment(
expr,
- (left_col, right_col),
- Interval::make(Some(10), Some(20), (true, true)),
- Interval::make(Some(100), None, (true, true)),
- Interval::make(Some(10), Some(20), (true, true)),
- Interval::make(Some(100), None, (true, true)),
+ (left_col.clone(), right_col.clone()),
+ Interval::make(Some(10_i32), Some(20_i32))?,
+ Interval::make(Some(100), None)?,
+ Interval::make(Some(10), Some(20))?,
+ Interval::make(Some(100), None)?,
PropagationResult::Infeasible,
+ &Schema::new(vec![
+ Field::new(
+ left_col.as_any().downcast_ref::<Column>().unwrap().name(),
+ DataType::Int32,
+ true,
+ ),
+ Field::new(
+ right_col.as_any().downcast_ref::<Column>().unwrap().name(),
+ DataType::Int32,
+ true,
+ ),
+ ]),
)
}
@@ -1112,7 +1198,14 @@ mod tests {
Arc::new(Column::new("b", 1)),
));
let expr = Arc::new(BinaryExpr::new(left_expr, Operator::Gt, right_expr));
- let mut graph = ExprIntervalGraph::try_new(expr).unwrap();
+ let mut graph = ExprIntervalGraph::try_new(
+ expr,
+ &Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ ]),
+ )
+ .unwrap();
// Define a test leaf node.
let leaf_node = Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
@@ -1151,7 +1244,16 @@ mod tests {
Arc::new(Column::new("z", 1)),
));
let expr = Arc::new(BinaryExpr::new(left_expr, Operator::Gt, right_expr));
- let mut graph = ExprIntervalGraph::try_new(expr).unwrap();
+ let mut graph = ExprIntervalGraph::try_new(
+ expr,
+ &Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("y", DataType::Int32, true),
+ Field::new("z", DataType::Int32, true),
+ ]),
+ )
+ .unwrap();
// Define a test leaf node.
let leaf_node = Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
@@ -1190,7 +1292,15 @@ mod tests {
Arc::new(Column::new("z", 1)),
));
let expr = Arc::new(BinaryExpr::new(left_expr, Operator::Gt, right_expr));
- let mut graph = ExprIntervalGraph::try_new(expr).unwrap();
+ let mut graph = ExprIntervalGraph::try_new(
+ expr,
+ &Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("z", DataType::Int32, true),
+ ]),
+ )
+ .unwrap();
// Define a test leaf node.
let leaf_node = Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
@@ -1213,9 +1323,9 @@ mod tests {
fn test_gather_node_indices_cannot_provide() -> Result<()> {
// Expression: a@0 + 1 + b@1 > y@0 - z@1 -> provide a@0 + b@1
// TODO: We expect nodes a@0 and b@1 to be pruned, and intervals to be provided from the a@0 + b@1 node.
- // However, we do not have an exact node for a@0 + b@1 due to the binary tree structure of the expressions.
- // Pruning and interval providing for BinaryExpr expressions are more challenging without exact matches.
- // Currently, we only support exact matches for BinaryExprs, but we plan to extend support beyond exact matches in the future.
+ // However, we do not have an exact node for a@0 + b@1 due to the binary tree structure of the expressions.
+ // Pruning and interval providing for BinaryExpr expressions are more challenging without exact matches.
+ // Currently, we only support exact matches for BinaryExprs, but we plan to extend support beyond exact matches in the future.
let left_expr = Arc::new(BinaryExpr::new(
Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
@@ -1232,7 +1342,16 @@ mod tests {
Arc::new(Column::new("z", 1)),
));
let expr = Arc::new(BinaryExpr::new(left_expr, Operator::Gt, right_expr));
- let mut graph = ExprIntervalGraph::try_new(expr).unwrap();
+ let mut graph = ExprIntervalGraph::try_new(
+ expr,
+ &Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("y", DataType::Int32, true),
+ Field::new("z", DataType::Int32, true),
+ ]),
+ )
+ .unwrap();
// Define a test leaf node.
let leaf_node = Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
@@ -1257,80 +1376,51 @@ mod tests {
Operator::Plus,
Arc::new(Literal::new(ScalarValue::new_interval_mdn(0, 1, 321))),
);
- let parent = Interval::new(
- IntervalBound::new(
- // 15.10.2020 - 10:11:12.000_000_321 AM
- ScalarValue::TimestampNanosecond(Some(1_602_756_672_000_000_321), None),
- false,
- ),
- IntervalBound::new(
- // 16.10.2020 - 10:11:12.000_000_321 AM
- ScalarValue::TimestampNanosecond(Some(1_602_843_072_000_000_321), None),
- false,
- ),
- );
- let left_child = Interval::new(
- IntervalBound::new(
- // 10.10.2020 - 10:11:12 AM
- ScalarValue::TimestampNanosecond(Some(1_602_324_672_000_000_000), None),
- false,
- ),
- IntervalBound::new(
- // 20.10.2020 - 10:11:12 AM
- ScalarValue::TimestampNanosecond(Some(1_603_188_672_000_000_000), None),
- false,
- ),
- );
- let right_child = Interval::new(
- IntervalBound::new(
- // 1 day 321 ns
- ScalarValue::IntervalMonthDayNano(Some(0x1_0000_0000_0000_0141)),
- false,
- ),
- IntervalBound::new(
- // 1 day 321 ns
- ScalarValue::IntervalMonthDayNano(Some(0x1_0000_0000_0000_0141)),
- false,
- ),
- );
+ let parent = Interval::try_new(
+ // 15.10.2020 - 10:11:12.000_000_321 AM
+ ScalarValue::TimestampNanosecond(Some(1_602_756_672_000_000_321), None),
+ // 16.10.2020 - 10:11:12.000_000_321 AM
+ ScalarValue::TimestampNanosecond(Some(1_602_843_072_000_000_321), None),
+ )?;
+ let left_child = Interval::try_new(
+ // 10.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampNanosecond(Some(1_602_324_672_000_000_000), None),
+ // 20.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampNanosecond(Some(1_603_188_672_000_000_000), None),
+ )?;
+ let right_child = Interval::try_new(
+ // 1 day 321 ns
+ ScalarValue::IntervalMonthDayNano(Some(0x1_0000_0000_0000_0141)),
+ // 1 day 321 ns
+ ScalarValue::IntervalMonthDayNano(Some(0x1_0000_0000_0000_0141)),
+ )?;
let children = vec![&left_child, &right_child];
- let result = expression.propagate_constraints(&parent, &children)?;
+ let result = expression
+ .propagate_constraints(&parent, &children)?
+ .unwrap();
assert_eq!(
- Some(Interval::new(
- // 14.10.2020 - 10:11:12 AM
- IntervalBound::new(
+ vec![
+ Interval::try_new(
+ // 14.10.2020 - 10:11:12 AM
ScalarValue::TimestampNanosecond(
Some(1_602_670_272_000_000_000),
None
),
- false,
- ),
- // 15.10.2020 - 10:11:12 AM
- IntervalBound::new(
+ // 15.10.2020 - 10:11:12 AM
ScalarValue::TimestampNanosecond(
Some(1_602_756_672_000_000_000),
None
),
- false,
- ),
- )),
- result[0]
- );
- assert_eq!(
- Some(Interval::new(
- // 1 day 321 ns in Duration type
- IntervalBound::new(
+ )?,
+ Interval::try_new(
+ // 1 day 321 ns in Duration type
ScalarValue::IntervalMonthDayNano(Some(0x1_0000_0000_0000_0141)),
- false,
- ),
- // 1 day 321 ns in Duration type
- IntervalBound::new(
+ // 1 day 321 ns in Duration type
ScalarValue::IntervalMonthDayNano(Some(0x1_0000_0000_0000_0141)),
- false,
- ),
- )),
- result[1]
+ )?
+ ],
+ result
);
Ok(())
@@ -1343,206 +1433,216 @@ mod tests {
Operator::Plus,
Arc::new(Column::new("ts_column", 0)),
);
- let parent = Interval::new(
- IntervalBound::new(
- // 15.10.2020 - 10:11:12 AM
- ScalarValue::TimestampMillisecond(Some(1_602_756_672_000), None),
- false,
- ),
- IntervalBound::new(
- // 16.10.2020 - 10:11:12 AM
- ScalarValue::TimestampMillisecond(Some(1_602_843_072_000), None),
- false,
- ),
- );
- let right_child = Interval::new(
- IntervalBound::new(
- // 10.10.2020 - 10:11:12 AM
- ScalarValue::TimestampMillisecond(Some(1_602_324_672_000), None),
- false,
- ),
- IntervalBound::new(
- // 20.10.2020 - 10:11:12 AM
- ScalarValue::TimestampMillisecond(Some(1_603_188_672_000), None),
- false,
- ),
- );
- let left_child = Interval::new(
- IntervalBound::new(
- // 2 days
- ScalarValue::IntervalDayTime(Some(172_800_000)),
- false,
- ),
- IntervalBound::new(
- // 10 days
- ScalarValue::IntervalDayTime(Some(864_000_000)),
- false,
- ),
- );
+ let parent = Interval::try_new(
+ // 15.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampMillisecond(Some(1_602_756_672_000), None),
+ // 16.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampMillisecond(Some(1_602_843_072_000), None),
+ )?;
+ let right_child = Interval::try_new(
+ // 10.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampMillisecond(Some(1_602_324_672_000), None),
+ // 20.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampMillisecond(Some(1_603_188_672_000), None),
+ )?;
+ let left_child = Interval::try_new(
+ // 2 days
+ ScalarValue::IntervalDayTime(Some(172_800_000)),
+ // 10 days
+ ScalarValue::IntervalDayTime(Some(864_000_000)),
+ )?;
let children = vec![&left_child, &right_child];
- let result = expression.propagate_constraints(&parent, &children)?;
+ let result = expression
+ .propagate_constraints(&parent, &children)?
+ .unwrap();
assert_eq!(
- Some(Interval::new(
- // 10.10.2020 - 10:11:12 AM
- IntervalBound::new(
- ScalarValue::TimestampMillisecond(Some(1_602_324_672_000), None),
- false,
- ),
- // 14.10.2020 - 10:11:12 AM
- IntervalBound::new(
- ScalarValue::TimestampMillisecond(Some(1_602_670_272_000), None),
- false,
- )
- )),
- result[1]
- );
- assert_eq!(
- Some(Interval::new(
- IntervalBound::new(
+ vec![
+ Interval::try_new(
// 2 days
ScalarValue::IntervalDayTime(Some(172_800_000)),
- false,
- ),
- IntervalBound::new(
// 6 days
ScalarValue::IntervalDayTime(Some(518_400_000)),
- false,
- ),
- )),
- result[0]
+ )?,
+ Interval::try_new(
+ // 10.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampMillisecond(Some(1_602_324_672_000), None),
+ // 14.10.2020 - 10:11:12 AM
+ ScalarValue::TimestampMillisecond(Some(1_602_670_272_000), None),
+ )?
+ ],
+ result
);
Ok(())
}
#[test]
- fn test_propagate_comparison() {
+ fn test_propagate_comparison() -> Result<()> {
// In the examples below:
// `left` is unbounded: [?, ?],
// `right` is known to be [1000,1000]
- // so `left` < `right` results in no new knowledge of `right` but knowing that `left` is now < 1000:` [?, 1000)
- let left = Interval::new(
- IntervalBound::make_unbounded(DataType::Int64).unwrap(),
- IntervalBound::make_unbounded(DataType::Int64).unwrap(),
- );
- let right = Interval::new(
- IntervalBound::new(ScalarValue::Int64(Some(1000)), false),
- IntervalBound::new(ScalarValue::Int64(Some(1000)), false),
- );
+ // so `left` < `right` results in no new knowledge of `right` but knowing that `left` is now < 1000:` [?, 999]
+ let left = Interval::make_unbounded(&DataType::Int64)?;
+ let right = Interval::make(Some(1000_i64), Some(1000_i64))?;
assert_eq!(
- (
- Some(Interval::new(
- IntervalBound::make_unbounded(DataType::Int64).unwrap(),
- IntervalBound::new(ScalarValue::Int64(Some(1000)), true)
- )),
- Some(Interval::new(
- IntervalBound::new(ScalarValue::Int64(Some(1000)), false),
- IntervalBound::new(ScalarValue::Int64(Some(1000)), false)
- )),
- ),
- propagate_comparison(&Operator::Lt, &left, &right).unwrap()
+ (Some((
+ Interval::make(None, Some(999_i64))?,
+ Interval::make(Some(1000_i64), Some(1000_i64))?,
+ ))),
+ propagate_comparison(
+ &Operator::Lt,
+ &Interval::CERTAINLY_TRUE,
+ &left,
+ &right
+ )?
);
- let left = Interval::new(
- IntervalBound::make_unbounded(DataType::Timestamp(
- TimeUnit::Nanosecond,
- None,
- ))
- .unwrap(),
- IntervalBound::make_unbounded(DataType::Timestamp(
- TimeUnit::Nanosecond,
- None,
- ))
- .unwrap(),
- );
- let right = Interval::new(
- IntervalBound::new(ScalarValue::TimestampNanosecond(Some(1000), None), false),
- IntervalBound::new(ScalarValue::TimestampNanosecond(Some(1000), None), false),
- );
+ let left =
+ Interval::make_unbounded(&DataType::Timestamp(TimeUnit::Nanosecond, None))?;
+ let right = Interval::try_new(
+ ScalarValue::TimestampNanosecond(Some(1000), None),
+ ScalarValue::TimestampNanosecond(Some(1000), None),
+ )?;
assert_eq!(
- (
- Some(Interval::new(
- IntervalBound::make_unbounded(DataType::Timestamp(
+ (Some((
+ Interval::try_new(
+ ScalarValue::try_from(&DataType::Timestamp(
TimeUnit::Nanosecond,
None
))
.unwrap(),
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(Some(1000), None),
- true
- )
- )),
- Some(Interval::new(
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(Some(1000), None),
- false
- ),
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(Some(1000), None),
- false
- )
- )),
- ),
- propagate_comparison(&Operator::Lt, &left, &right).unwrap()
+ ScalarValue::TimestampNanosecond(Some(999), None),
+ )?,
+ Interval::try_new(
+ ScalarValue::TimestampNanosecond(Some(1000), None),
+ ScalarValue::TimestampNanosecond(Some(1000), None),
+ )?
+ ))),
+ propagate_comparison(
+ &Operator::Lt,
+ &Interval::CERTAINLY_TRUE,
+ &left,
+ &right
+ )?
);
- let left = Interval::new(
- IntervalBound::make_unbounded(DataType::Timestamp(
- TimeUnit::Nanosecond,
- Some("+05:00".into()),
- ))
- .unwrap(),
- IntervalBound::make_unbounded(DataType::Timestamp(
- TimeUnit::Nanosecond,
- Some("+05:00".into()),
- ))
- .unwrap(),
- );
- let right = Interval::new(
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(Some(1000), Some("+05:00".into())),
- false,
- ),
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(Some(1000), Some("+05:00".into())),
- false,
- ),
- );
+ let left = Interval::make_unbounded(&DataType::Timestamp(
+ TimeUnit::Nanosecond,
+ Some("+05:00".into()),
+ ))?;
+ let right = Interval::try_new(
+ ScalarValue::TimestampNanosecond(Some(1000), Some("+05:00".into())),
+ ScalarValue::TimestampNanosecond(Some(1000), Some("+05:00".into())),
+ )?;
assert_eq!(
- (
- Some(Interval::new(
- IntervalBound::make_unbounded(DataType::Timestamp(
+ (Some((
+ Interval::try_new(
+ ScalarValue::try_from(&DataType::Timestamp(
TimeUnit::Nanosecond,
Some("+05:00".into()),
))
.unwrap(),
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(
- Some(1000),
- Some("+05:00".into())
- ),
- true
- )
- )),
- Some(Interval::new(
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(
- Some(1000),
- Some("+05:00".into())
- ),
- false
- ),
- IntervalBound::new(
- ScalarValue::TimestampNanosecond(
- Some(1000),
- Some("+05:00".into())
- ),
- false
- )
- )),
- ),
- propagate_comparison(&Operator::Lt, &left, &right).unwrap()
+ ScalarValue::TimestampNanosecond(Some(999), Some("+05:00".into())),
+ )?,
+ Interval::try_new(
+ ScalarValue::TimestampNanosecond(Some(1000), Some("+05:00".into())),
+ ScalarValue::TimestampNanosecond(Some(1000), Some("+05:00".into())),
+ )?
+ ))),
+ propagate_comparison(
+ &Operator::Lt,
+ &Interval::CERTAINLY_TRUE,
+ &left,
+ &right
+ )?
);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_propagate_or() -> Result<()> {
+ let expr = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Or,
+ Arc::new(Column::new("b", 1)),
+ ));
+ let parent = Interval::CERTAINLY_FALSE;
+ let children_set = vec![
+ vec![&Interval::CERTAINLY_FALSE, &Interval::UNCERTAIN],
+ vec![&Interval::UNCERTAIN, &Interval::CERTAINLY_FALSE],
+ vec![&Interval::CERTAINLY_FALSE, &Interval::CERTAINLY_FALSE],
+ vec![&Interval::UNCERTAIN, &Interval::UNCERTAIN],
+ ];
+ for children in children_set {
+ assert_eq!(
+ expr.propagate_constraints(&parent, &children)?.unwrap(),
+ vec![Interval::CERTAINLY_FALSE, Interval::CERTAINLY_FALSE],
+ );
+ }
+
+ let parent = Interval::CERTAINLY_FALSE;
+ let children_set = vec![
+ vec![&Interval::CERTAINLY_TRUE, &Interval::UNCERTAIN],
+ vec![&Interval::UNCERTAIN, &Interval::CERTAINLY_TRUE],
+ ];
+ for children in children_set {
+ assert_eq!(expr.propagate_constraints(&parent, &children)?, None,);
+ }
+
+ let parent = Interval::CERTAINLY_TRUE;
+ let children = vec![&Interval::CERTAINLY_FALSE, &Interval::UNCERTAIN];
+ assert_eq!(
+ expr.propagate_constraints(&parent, &children)?.unwrap(),
+ vec![Interval::CERTAINLY_FALSE, Interval::CERTAINLY_TRUE]
+ );
+
+ let parent = Interval::CERTAINLY_TRUE;
+ let children = vec![&Interval::UNCERTAIN, &Interval::UNCERTAIN];
+ assert_eq!(
+ expr.propagate_constraints(&parent, &children)?.unwrap(),
+ // Empty means unchanged intervals.
+ vec![]
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_propagate_certainly_false_and() -> Result<()> {
+ let expr = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::And,
+ Arc::new(Column::new("b", 1)),
+ ));
+ let parent = Interval::CERTAINLY_FALSE;
+ let children_and_results_set = vec![
+ (
+ vec![&Interval::CERTAINLY_TRUE, &Interval::UNCERTAIN],
+ vec![Interval::CERTAINLY_TRUE, Interval::CERTAINLY_FALSE],
+ ),
+ (
+ vec![&Interval::UNCERTAIN, &Interval::CERTAINLY_TRUE],
+ vec![Interval::CERTAINLY_FALSE, Interval::CERTAINLY_TRUE],
+ ),
+ (
+ vec![&Interval::UNCERTAIN, &Interval::UNCERTAIN],
+ // Empty means unchanged intervals.
+ vec![],
+ ),
+ (
+ vec![&Interval::CERTAINLY_FALSE, &Interval::UNCERTAIN],
+ vec![],
+ ),
+ ];
+ for (children, result) in children_and_results_set {
+ assert_eq!(
+ expr.propagate_constraints(&parent, &children)?.unwrap(),
+ result
+ );
+ }
+
+ Ok(())
}
}
diff --git a/datafusion/physical-expr/src/intervals/interval_aritmetic.rs b/datafusion/physical-expr/src/intervals/interval_aritmetic.rs
deleted file mode 100644
index 4b81adfbb1..0000000000
--- a/datafusion/physical-expr/src/intervals/interval_aritmetic.rs
+++ /dev/null
@@ -1,1886 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-//http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Interval arithmetic library
-
-use std::borrow::Borrow;
-use std::fmt::{self, Display, Formatter};
-use std::ops::{AddAssign, SubAssign};
-
-use crate::aggregate::min_max::{max, min};
-use crate::intervals::rounding::{alter_fp_rounding_mode, next_down, next_up};
-
-use arrow::compute::{cast_with_options, CastOptions};
-use arrow::datatypes::DataType;
-use arrow_array::ArrowNativeTypeOp;
-use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue};
-use datafusion_expr::type_coercion::binary::get_result_type;
-use datafusion_expr::Operator;
-
-/// This type represents a single endpoint of an [`Interval`]. An
-/// endpoint can be open (does not include the endpoint) or closed
-/// (includes the endpoint).
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct IntervalBound {
- pub value: ScalarValue,
- /// If true, interval does not include `value`
- pub open: bool,
-}
-
-impl IntervalBound {
- /// Creates a new `IntervalBound` object using the given value.
- pub const fn new(value: ScalarValue, open: bool) -> IntervalBound {
- IntervalBound { value, open }
- }
-
- /// Creates a new "open" interval (does not include the `value`
- /// bound)
- pub const fn new_open(value: ScalarValue) -> IntervalBound {
- IntervalBound::new(value, true)
- }
-
- /// Creates a new "closed" interval (includes the `value`
- /// bound)
- pub const fn new_closed(value: ScalarValue) -> IntervalBound {
- IntervalBound::new(value, false)
- }
-
- /// This convenience function creates an unbounded interval endpoint.
- pub fn make_unbounded<T: Borrow<DataType>>(data_type: T) -> Result<Self> {
- ScalarValue::try_from(data_type.borrow()).map(|v| IntervalBound::new(v, true))
- }
-
- /// This convenience function returns the data type associated with this
- /// `IntervalBound`.
- pub fn get_datatype(&self) -> DataType {
- self.value.data_type()
- }
-
- /// This convenience function checks whether the `IntervalBound` represents
- /// an unbounded interval endpoint.
- pub fn is_unbounded(&self) -> bool {
- self.value.is_null()
- }
-
- /// This function casts the `IntervalBound` to the given data type.
- pub(crate) fn cast_to(
- &self,
- data_type: &DataType,
- cast_options: &CastOptions,
- ) -> Result<IntervalBound> {
- cast_scalar_value(&self.value, data_type, cast_options)
- .map(|value| IntervalBound::new(value, self.open))
- }
-
- /// Returns a new bound with a negated value, if any, and the same open/closed.
- /// For example negating `[5` would return `[-5`, or `-1)` would return `1)`.
- pub fn negate(&self) -> Result<IntervalBound> {
- self.value.arithmetic_negate().map(|value| IntervalBound {
- value,
- open: self.open,
- })
- }
-
- /// This function adds the given `IntervalBound` to this `IntervalBound`.
- /// The result is unbounded if either is; otherwise, their values are
- /// added. The result is closed if both original bounds are closed, or open
- /// otherwise.
- pub fn add<const UPPER: bool, T: Borrow<IntervalBound>>(
- &self,
- other: T,
- ) -> Result<IntervalBound> {
- let rhs = other.borrow();
- if self.is_unbounded() || rhs.is_unbounded() {
- return IntervalBound::make_unbounded(get_result_type(
- &self.get_datatype(),
- &Operator::Plus,
- &rhs.get_datatype(),
- )?);
- }
- match self.get_datatype() {
- DataType::Float64 | DataType::Float32 => {
- alter_fp_rounding_mode::<UPPER, _>(&self.value, &rhs.value, |lhs, rhs| {
- lhs.add(rhs)
- })
- }
- _ => self.value.add(&rhs.value),
- }
- .map(|v| IntervalBound::new(v, self.open || rhs.open))
- }
-
- /// This function subtracts the given `IntervalBound` from `self`.
- /// The result is unbounded if either is; otherwise, their values are
- /// subtracted. The result is closed if both original bounds are closed,
- /// or open otherwise.
- pub fn sub<const UPPER: bool, T: Borrow<IntervalBound>>(
- &self,
- other: T,
- ) -> Result<IntervalBound> {
- let rhs = other.borrow();
- if self.is_unbounded() || rhs.is_unbounded() {
- return IntervalBound::make_unbounded(get_result_type(
- &self.get_datatype(),
- &Operator::Minus,
- &rhs.get_datatype(),
- )?);
- }
- match self.get_datatype() {
- DataType::Float64 | DataType::Float32 => {
- alter_fp_rounding_mode::<UPPER, _>(&self.value, &rhs.value, |lhs, rhs| {
- lhs.sub(rhs)
- })
- }
- _ => self.value.sub(&rhs.value),
- }
- .map(|v| IntervalBound::new(v, self.open || rhs.open))
- }
-
- /// This function chooses one of the given `IntervalBound`s according to
- /// the given function `decide`. The result is unbounded if both are. If
- /// only one of the arguments is unbounded, the other one is chosen by
- /// default. If neither is unbounded, the function `decide` is used.
- pub fn choose(
- first: &IntervalBound,
- second: &IntervalBound,
- decide: fn(&ScalarValue, &ScalarValue) -> Result<ScalarValue>,
- ) -> Result<IntervalBound> {
- Ok(if first.is_unbounded() {
- second.clone()
- } else if second.is_unbounded() {
- first.clone()
- } else if first.value != second.value {
- let chosen = decide(&first.value, &second.value)?;
- if chosen.eq(&first.value) {
- first.clone()
- } else {
- second.clone()
- }
- } else {
- IntervalBound::new(second.value.clone(), first.open || second.open)
- })
- }
-}
-
-impl Display for IntervalBound {
- fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
- write!(f, "IntervalBound [{}]", self.value)
- }
-}
-
-/// This type represents an interval, which is used to calculate reliable
-/// bounds for expressions:
-///
-/// * An *open* interval does not include the endpoint and is written using a
-/// `(` or `)`.
-///
-/// * A *closed* interval does include the endpoint and is written using `[` or
-/// `]`.
-///
-/// * If the interval's `lower` and/or `upper` bounds are not known, they are
-/// called *unbounded* endpoint and represented using a `NULL` and written using
-/// `∞`.
-///
-/// # Examples
-///
-/// A `Int64` `Interval` of `[10, 20)` represents the values `10, 11, ... 18,
-/// 19` (includes 10, but does not include 20).
-///
-/// A `Int64` `Interval` of `[10, ∞)` represents a value known to be either
-/// `10` or higher.
-///
-/// An `Interval` of `(-∞, ∞)` represents that the range is entirely unknown.
-///
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Interval {
- pub lower: IntervalBound,
- pub upper: IntervalBound,
-}
-
-impl Default for Interval {
- fn default() -> Self {
- Interval::new(
- IntervalBound::new(ScalarValue::Null, true),
- IntervalBound::new(ScalarValue::Null, true),
- )
- }
-}
-
-impl Display for Interval {
- fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
- write!(
- f,
- "{}{}, {}{}",
- if self.lower.open { "(" } else { "[" },
- self.lower.value,
- self.upper.value,
- if self.upper.open { ")" } else { "]" }
- )
- }
-}
-
-impl Interval {
- /// Creates a new interval object using the given bounds.
- ///
- /// # Boolean intervals need special handling
- ///
- /// For boolean intervals, having an open false lower bound is equivalent to
- /// having a true closed lower bound. Similarly, open true upper bound is
- /// equivalent to having a false closed upper bound. Also for boolean
- /// intervals, having an unbounded left endpoint is equivalent to having a
- /// false closed lower bound, while having an unbounded right endpoint is
- /// equivalent to having a true closed upper bound. Therefore; input
- /// parameters to construct an Interval can have different types, but they
- /// all result in `[false, false]`, `[false, true]` or `[true, true]`.
- pub fn new(lower: IntervalBound, upper: IntervalBound) -> Interval {
- // Boolean intervals need a special handling.
- if let ScalarValue::Boolean(_) = lower.value {
- let standardized_lower = match lower.value {
- ScalarValue::Boolean(None) if lower.open => {
- ScalarValue::Boolean(Some(false))
- }
- ScalarValue::Boolean(Some(false)) if lower.open => {
- ScalarValue::Boolean(Some(true))
- }
- // The rest may include some invalid interval cases. The validation of
- // interval construction parameters will be implemented later.
- // For now, let's return them unchanged.
- _ => lower.value,
- };
- let standardized_upper = match upper.value {
- ScalarValue::Boolean(None) if upper.open => {
- ScalarValue::Boolean(Some(true))
- }
- ScalarValue::Boolean(Some(true)) if upper.open => {
- ScalarValue::Boolean(Some(false))
- }
- _ => upper.value,
- };
- Interval {
- lower: IntervalBound::new(standardized_lower, false),
- upper: IntervalBound::new(standardized_upper, false),
- }
- } else {
- Interval { lower, upper }
- }
- }
-
- pub fn make<T>(lower: Option<T>, upper: Option<T>, open: (bool, bool)) -> Interval
- where
- ScalarValue: From<Option<T>>,
- {
- Interval::new(
- IntervalBound::new(ScalarValue::from(lower), open.0),
- IntervalBound::new(ScalarValue::from(upper), open.1),
- )
- }
-
- /// Casts this interval to `data_type` using `cast_options`.
- pub(crate) fn cast_to(
- &self,
- data_type: &DataType,
- cast_options: &CastOptions,
- ) -> Result<Interval> {
- let lower = self.lower.cast_to(data_type, cast_options)?;
- let upper = self.upper.cast_to(data_type, cast_options)?;
- Ok(Interval::new(lower, upper))
- }
-
- /// This function returns the data type of this interval. If both endpoints
- /// do not have the same data type, returns an error.
- pub fn get_datatype(&self) -> Result<DataType> {
- let lower_type = self.lower.get_datatype();
- let upper_type = self.upper.get_datatype();
- if lower_type == upper_type {
- Ok(lower_type)
- } else {
- internal_err!(
- "Interval bounds have different types: {lower_type} != {upper_type}"
- )
- }
- }
-
- /// Decide if this interval is certainly greater than, possibly greater than,
- /// or can't be greater than `other` by returning [true, true],
- /// [false, true] or [false, false] respectively.
- pub(crate) fn gt<T: Borrow<Interval>>(&self, other: T) -> Interval {
- let rhs = other.borrow();
- let flags = if !self.upper.is_unbounded()
- && !rhs.lower.is_unbounded()
- && self.upper.value <= rhs.lower.value
- {
- // Values in this interval are certainly less than or equal to those
- // in the given interval.
- (false, false)
- } else if !self.lower.is_unbounded()
- && !rhs.upper.is_unbounded()
- && self.lower.value >= rhs.upper.value
- && (self.lower.value > rhs.upper.value || self.lower.open || rhs.upper.open)
- {
- // Values in this interval are certainly greater than those in the
- // given interval.
- (true, true)
- } else {
- // All outcomes are possible.
- (false, true)
- };
-
- Interval::make(Some(flags.0), Some(flags.1), (false, false))
- }
-
- /// Decide if this interval is certainly greater than or equal to, possibly greater than
- /// or equal to, or can't be greater than or equal to `other` by returning [true, true],
- /// [false, true] or [false, false] respectively.
- pub(crate) fn gt_eq<T: Borrow<Interval>>(&self, other: T) -> Interval {
- let rhs = other.borrow();
- let flags = if !self.lower.is_unbounded()
- && !rhs.upper.is_unbounded()
- && self.lower.value >= rhs.upper.value
- {
- // Values in this interval are certainly greater than or equal to those
- // in the given interval.
- (true, true)
- } else if !self.upper.is_unbounded()
- && !rhs.lower.is_unbounded()
- && self.upper.value <= rhs.lower.value
- && (self.upper.value < rhs.lower.value || self.upper.open || rhs.lower.open)
- {
- // Values in this interval are certainly less than those in the
- // given interval.
- (false, false)
- } else {
- // All outcomes are possible.
- (false, true)
- };
-
- Interval::make(Some(flags.0), Some(flags.1), (false, false))
- }
-
- /// Decide if this interval is certainly less than, possibly less than,
- /// or can't be less than `other` by returning [true, true],
- /// [false, true] or [false, false] respectively.
- pub(crate) fn lt<T: Borrow<Interval>>(&self, other: T) -> Interval {
- other.borrow().gt(self)
- }
-
- /// Decide if this interval is certainly less than or equal to, possibly
- /// less than or equal to, or can't be less than or equal to `other` by returning
- /// [true, true], [false, true] or [false, false] respectively.
- pub(crate) fn lt_eq<T: Borrow<Interval>>(&self, other: T) -> Interval {
- other.borrow().gt_eq(self)
- }
-
- /// Decide if this interval is certainly equal to, possibly equal to,
- /// or can't be equal to `other` by returning [true, true],
- /// [false, true] or [false, false] respectively.
- pub(crate) fn equal<T: Borrow<Interval>>(&self, other: T) -> Interval {
- let rhs = other.borrow();
- let flags = if !self.lower.is_unbounded()
- && (self.lower.value == self.upper.value)
- && (rhs.lower.value == rhs.upper.value)
- && (self.lower.value == rhs.lower.value)
- {
- (true, true)
- } else if self.gt(rhs) == Interval::CERTAINLY_TRUE
- || self.lt(rhs) == Interval::CERTAINLY_TRUE
- {
- (false, false)
- } else {
- (false, true)
- };
-
- Interval::make(Some(flags.0), Some(flags.1), (false, false))
- }
-
- /// Compute the logical conjunction of this (boolean) interval with the given boolean interval.
- pub(crate) fn and<T: Borrow<Interval>>(&self, other: T) -> Result<Interval> {
- let rhs = other.borrow();
- match (
- &self.lower.value,
- &self.upper.value,
- &rhs.lower.value,
- &rhs.upper.value,
- ) {
- (
- ScalarValue::Boolean(Some(self_lower)),
- ScalarValue::Boolean(Some(self_upper)),
- ScalarValue::Boolean(Some(other_lower)),
- ScalarValue::Boolean(Some(other_upper)),
- ) => {
- let lower = *self_lower && *other_lower;
- let upper = *self_upper && *other_upper;
-
- Ok(Interval {
- lower: IntervalBound::new(ScalarValue::Boolean(Some(lower)), false),
- upper: IntervalBound::new(ScalarValue::Boolean(Some(upper)), false),
- })
- }
- _ => internal_err!("Incompatible types for logical conjunction"),
- }
- }
-
- /// Compute the logical negation of this (boolean) interval.
- pub(crate) fn not(&self) -> Result<Self> {
- if !matches!(self.get_datatype()?, DataType::Boolean) {
- return internal_err!(
- "Cannot apply logical negation to non-boolean interval"
- );
- }
- if self == &Interval::CERTAINLY_TRUE {
- Ok(Interval::CERTAINLY_FALSE)
- } else if self == &Interval::CERTAINLY_FALSE {
- Ok(Interval::CERTAINLY_TRUE)
- } else {
- Ok(Interval::UNCERTAIN)
- }
- }
-
- /// Compute the intersection of the interval with the given interval.
- /// If the intersection is empty, return None.
- pub(crate) fn intersect<T: Borrow<Interval>>(
- &self,
- other: T,
- ) -> Result<Option<Interval>> {
- let rhs = other.borrow();
- // If it is evident that the result is an empty interval,
- // do not make any calculation and directly return None.
- if (!self.lower.is_unbounded()
- && !rhs.upper.is_unbounded()
- && self.lower.value > rhs.upper.value)
- || (!self.upper.is_unbounded()
- && !rhs.lower.is_unbounded()
- && self.upper.value < rhs.lower.value)
- {
- // This None value signals an empty interval.
- return Ok(None);
- }
-
- let lower = IntervalBound::choose(&self.lower, &rhs.lower, max)?;
- let upper = IntervalBound::choose(&self.upper, &rhs.upper, min)?;
-
- let non_empty = lower.is_unbounded()
- || upper.is_unbounded()
- || lower.value != upper.value
- || (!lower.open && !upper.open);
- Ok(non_empty.then_some(Interval::new(lower, upper)))
- }
-
- /// Decide if this interval is certainly contains, possibly contains,
- /// or can't can't `other` by returning [true, true],
- /// [false, true] or [false, false] respectively.
- pub fn contains<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
- match self.intersect(other.borrow())? {
- Some(intersection) => {
- // Need to compare with same bounds close-ness.
- if intersection.close_bounds() == other.borrow().clone().close_bounds() {
- Ok(Interval::CERTAINLY_TRUE)
- } else {
- Ok(Interval::UNCERTAIN)
- }
- }
- None => Ok(Interval::CERTAINLY_FALSE),
- }
- }
-
- /// Add the given interval (`other`) to this interval. Say we have
- /// intervals [a1, b1] and [a2, b2], then their sum is [a1 + a2, b1 + b2].
- /// Note that this represents all possible values the sum can take if
- /// one can choose single values arbitrarily from each of the operands.
- pub fn add<T: Borrow<Interval>>(&self, other: T) -> Result<Interval> {
- let rhs = other.borrow();
- Ok(Interval::new(
- self.lower.add::<false, _>(&rhs.lower)?,
- self.upper.add::<true, _>(&rhs.upper)?,
- ))
- }
-
- /// Subtract the given interval (`other`) from this interval. Say we have
- /// intervals [a1, b1] and [a2, b2], then their sum is [a1 - b2, b1 - a2].
- /// Note that this represents all possible values the difference can take
- /// if one can choose single values arbitrarily from each of the operands.
- pub fn sub<T: Borrow<Interval>>(&self, other: T) -> Result<Interval> {
- let rhs = other.borrow();
- Ok(Interval::new(
- self.lower.sub::<false, _>(&rhs.upper)?,
- self.upper.sub::<true, _>(&rhs.lower)?,
- ))
- }
-
- pub const CERTAINLY_FALSE: Interval = Interval {
- lower: IntervalBound::new_closed(ScalarValue::Boolean(Some(false))),
- upper: IntervalBound::new_closed(ScalarValue::Boolean(Some(false))),
- };
-
- pub const UNCERTAIN: Interval = Interval {
- lower: IntervalBound::new_closed(ScalarValue::Boolean(Some(false))),
- upper: IntervalBound::new_closed(ScalarValue::Boolean(Some(true))),
- };
-
- pub const CERTAINLY_TRUE: Interval = Interval {
- lower: IntervalBound::new_closed(ScalarValue::Boolean(Some(true))),
- upper: IntervalBound::new_closed(ScalarValue::Boolean(Some(true))),
- };
-
- /// Returns the cardinality of this interval, which is the number of all
- /// distinct points inside it. This function returns `None` if:
- /// - The interval is unbounded from either side, or
- /// - Cardinality calculations for the datatype in question is not
- /// implemented yet, or
- /// - An overflow occurs during the calculation.
- ///
- /// This function returns an error if the given interval is malformed.
- pub fn cardinality(&self) -> Result<Option<u64>> {
- let data_type = self.get_datatype()?;
- if data_type.is_integer() {
- Ok(self.upper.value.distance(&self.lower.value).map(|diff| {
- calculate_cardinality_based_on_bounds(
- self.lower.open,
- self.upper.open,
- diff as u64,
- )
- }))
- }
- // Ordering floating-point numbers according to their binary representations
- // coincide with their natural ordering. Therefore, we can consider their
- // binary representations as "indices" and subtract them. For details, see:
- // https://stackoverflow.com/questions/8875064/how-many-distinct-floating-point-numbers-in-a-specific-range
- else if data_type.is_floating() {
- match (&self.lower.value, &self.upper.value) {
- (
- ScalarValue::Float32(Some(lower)),
- ScalarValue::Float32(Some(upper)),
- ) => {
- // Negative numbers are sorted in the reverse order. To always have a positive difference after the subtraction,
- // we perform following transformation:
- let lower_bits = lower.to_bits() as i32;
- let upper_bits = upper.to_bits() as i32;
- let transformed_lower =
- lower_bits ^ ((lower_bits >> 31) & 0x7fffffff);
- let transformed_upper =
- upper_bits ^ ((upper_bits >> 31) & 0x7fffffff);
- let Ok(count) = transformed_upper.sub_checked(transformed_lower)
- else {
- return Ok(None);
- };
- Ok(Some(calculate_cardinality_based_on_bounds(
- self.lower.open,
- self.upper.open,
- count as u64,
- )))
- }
- (
- ScalarValue::Float64(Some(lower)),
- ScalarValue::Float64(Some(upper)),
- ) => {
- let lower_bits = lower.to_bits() as i64;
- let upper_bits = upper.to_bits() as i64;
- let transformed_lower =
- lower_bits ^ ((lower_bits >> 63) & 0x7fffffffffffffff);
- let transformed_upper =
- upper_bits ^ ((upper_bits >> 63) & 0x7fffffffffffffff);
- let Ok(count) = transformed_upper.sub_checked(transformed_lower)
- else {
- return Ok(None);
- };
- Ok(Some(calculate_cardinality_based_on_bounds(
- self.lower.open,
- self.upper.open,
- count as u64,
- )))
- }
- _ => Ok(None),
- }
- } else {
- // Cardinality calculations are not implemented for this data type yet:
- Ok(None)
- }
- }
-
- /// This function "closes" this interval; i.e. it modifies the endpoints so
- /// that we end up with the narrowest possible closed interval containing
- /// the original interval.
- pub fn close_bounds(mut self) -> Interval {
- if self.lower.open {
- // Get next value
- self.lower.value = next_value::<true>(self.lower.value);
- self.lower.open = false;
- }
-
- if self.upper.open {
- // Get previous value
- self.upper.value = next_value::<false>(self.upper.value);
- self.upper.open = false;
- }
-
- self
- }
-}
-
-trait OneTrait: Sized + std::ops::Add + std::ops::Sub {
- fn one() -> Self;
-}
-
-macro_rules! impl_OneTrait{
- ($($m:ty),*) => {$( impl OneTrait for $m { fn one() -> Self { 1 as $m } })*}
-}
-impl_OneTrait! {u8, u16, u32, u64, i8, i16, i32, i64}
-
-/// This function either increments or decrements its argument, depending on the `INC` value.
-/// If `true`, it increments; otherwise it decrements the argument.
-fn increment_decrement<const INC: bool, T: OneTrait + SubAssign + AddAssign>(
- mut val: T,
-) -> T {
- if INC {
- val.add_assign(T::one());
- } else {
- val.sub_assign(T::one());
- }
- val
-}
-
-macro_rules! check_infinite_bounds {
- ($value:expr, $val:expr, $type:ident, $inc:expr) => {
- if ($val == $type::MAX && $inc) || ($val == $type::MIN && !$inc) {
- return $value;
- }
- };
-}
-
-/// This function returns the next/previous value depending on the `ADD` value.
-/// If `true`, it returns the next value; otherwise it returns the previous value.
-fn next_value<const INC: bool>(value: ScalarValue) -> ScalarValue {
- use ScalarValue::*;
- match value {
- Float32(Some(val)) => {
- let new_float = if INC { next_up(val) } else { next_down(val) };
- Float32(Some(new_float))
- }
- Float64(Some(val)) => {
- let new_float = if INC { next_up(val) } else { next_down(val) };
- Float64(Some(new_float))
- }
- Int8(Some(val)) => {
- check_infinite_bounds!(value, val, i8, INC);
- Int8(Some(increment_decrement::<INC, i8>(val)))
- }
- Int16(Some(val)) => {
- check_infinite_bounds!(value, val, i16, INC);
- Int16(Some(increment_decrement::<INC, i16>(val)))
- }
- Int32(Some(val)) => {
- check_infinite_bounds!(value, val, i32, INC);
- Int32(Some(increment_decrement::<INC, i32>(val)))
- }
- Int64(Some(val)) => {
- check_infinite_bounds!(value, val, i64, INC);
- Int64(Some(increment_decrement::<INC, i64>(val)))
- }
- UInt8(Some(val)) => {
- check_infinite_bounds!(value, val, u8, INC);
- UInt8(Some(increment_decrement::<INC, u8>(val)))
- }
- UInt16(Some(val)) => {
- check_infinite_bounds!(value, val, u16, INC);
- UInt16(Some(increment_decrement::<INC, u16>(val)))
- }
- UInt32(Some(val)) => {
- check_infinite_bounds!(value, val, u32, INC);
- UInt32(Some(increment_decrement::<INC, u32>(val)))
- }
- UInt64(Some(val)) => {
- check_infinite_bounds!(value, val, u64, INC);
- UInt64(Some(increment_decrement::<INC, u64>(val)))
- }
- _ => value, // Unsupported datatypes
- }
-}
-
-/// This function computes the selectivity of an operation by computing the
-/// cardinality ratio of the given input/output intervals. If this can not be
-/// calculated for some reason, it returns `1.0` meaning fullly selective (no
-/// filtering).
-pub fn cardinality_ratio(
- initial_interval: &Interval,
- final_interval: &Interval,
-) -> Result<f64> {
- Ok(
- match (
- final_interval.cardinality()?,
- initial_interval.cardinality()?,
- ) {
- (Some(final_interval), Some(initial_interval)) => {
- final_interval as f64 / initial_interval as f64
- }
- _ => 1.0,
- },
- )
-}
-
-pub fn apply_operator(op: &Operator, lhs: &Interval, rhs: &Interval) -> Result<Interval> {
- match *op {
- Operator::Eq => Ok(lhs.equal(rhs)),
- Operator::NotEq => Ok(lhs.equal(rhs).not()?),
- Operator::Gt => Ok(lhs.gt(rhs)),
- Operator::GtEq => Ok(lhs.gt_eq(rhs)),
- Operator::Lt => Ok(lhs.lt(rhs)),
- Operator::LtEq => Ok(lhs.lt_eq(rhs)),
- Operator::And => lhs.and(rhs),
- Operator::Plus => lhs.add(rhs),
- Operator::Minus => lhs.sub(rhs),
- _ => Ok(Interval::default()),
- }
-}
-
-/// Cast scalar value to the given data type using an arrow kernel.
-fn cast_scalar_value(
- value: &ScalarValue,
- data_type: &DataType,
- cast_options: &CastOptions,
-) -> Result<ScalarValue> {
- let cast_array = cast_with_options(&value.to_array()?, data_type, cast_options)?;
- ScalarValue::try_from_array(&cast_array, 0)
-}
-
-/// This function calculates the final cardinality result by inspecting the endpoints of the interval.
-fn calculate_cardinality_based_on_bounds(
- lower_open: bool,
- upper_open: bool,
- diff: u64,
-) -> u64 {
- match (lower_open, upper_open) {
- (false, false) => diff + 1,
- (true, true) => diff - 1,
- _ => diff,
- }
-}
-
-/// An [Interval] that also tracks null status using a boolean interval.
-///
-/// This represents values that may be in a particular range or be null.
-///
-/// # Examples
-///
-/// ```
-/// use arrow::datatypes::DataType;
-/// use datafusion_physical_expr::intervals::{Interval, NullableInterval};
-/// use datafusion_common::ScalarValue;
-///
-/// // [1, 2) U {NULL}
-/// NullableInterval::MaybeNull {
-/// values: Interval::make(Some(1), Some(2), (false, true)),
-/// };
-///
-/// // (0, ∞)
-/// NullableInterval::NotNull {
-/// values: Interval::make(Some(0), None, (true, true)),
-/// };
-///
-/// // {NULL}
-/// NullableInterval::Null { datatype: DataType::Int32 };
-///
-/// // {4}
-/// NullableInterval::from(ScalarValue::Int32(Some(4)));
-/// ```
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum NullableInterval {
- /// The value is always null in this interval
- ///
- /// This is typed so it can be used in physical expressions, which don't do
- /// type coercion.
- Null { datatype: DataType },
- /// The value may or may not be null in this interval. If it is non null its value is within
- /// the specified values interval
- MaybeNull { values: Interval },
- /// The value is definitely not null in this interval and is within values
- NotNull { values: Interval },
-}
-
-impl Default for NullableInterval {
- fn default() -> Self {
- NullableInterval::MaybeNull {
- values: Interval::default(),
- }
- }
-}
-
-impl Display for NullableInterval {
- fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
- match self {
- Self::Null { .. } => write!(f, "NullableInterval: {{NULL}}"),
- Self::MaybeNull { values } => {
- write!(f, "NullableInterval: {} U {{NULL}}", values)
- }
- Self::NotNull { values } => write!(f, "NullableInterval: {}", values),
- }
- }
-}
-
-impl From<ScalarValue> for NullableInterval {
- /// Create an interval that represents a single value.
- fn from(value: ScalarValue) -> Self {
- if value.is_null() {
- Self::Null {
- datatype: value.data_type(),
- }
- } else {
- Self::NotNull {
- values: Interval::new(
- IntervalBound::new(value.clone(), false),
- IntervalBound::new(value, false),
- ),
- }
- }
- }
-}
-
-impl NullableInterval {
- /// Get the values interval, or None if this interval is definitely null.
- pub fn values(&self) -> Option<&Interval> {
- match self {
- Self::Null { .. } => None,
- Self::MaybeNull { values } | Self::NotNull { values } => Some(values),
- }
- }
-
- /// Get the data type
- pub fn get_datatype(&self) -> Result<DataType> {
- match self {
- Self::Null { datatype } => Ok(datatype.clone()),
- Self::MaybeNull { values } | Self::NotNull { values } => {
- values.get_datatype()
- }
- }
- }
-
- /// Return true if the value is definitely true (and not null).
- pub fn is_certainly_true(&self) -> bool {
- match self {
- Self::Null { .. } | Self::MaybeNull { .. } => false,
- Self::NotNull { values } => values == &Interval::CERTAINLY_TRUE,
- }
- }
-
- /// Return true if the value is definitely false (and not null).
- pub fn is_certainly_false(&self) -> bool {
- match self {
- Self::Null { .. } => false,
- Self::MaybeNull { .. } => false,
- Self::NotNull { values } => values == &Interval::CERTAINLY_FALSE,
- }
- }
-
- /// Perform logical negation on a boolean nullable interval.
- fn not(&self) -> Result<Self> {
- match self {
- Self::Null { datatype } => Ok(Self::Null {
- datatype: datatype.clone(),
- }),
- Self::MaybeNull { values } => Ok(Self::MaybeNull {
- values: values.not()?,
- }),
- Self::NotNull { values } => Ok(Self::NotNull {
- values: values.not()?,
- }),
- }
- }
-
- /// Apply the given operator to this interval and the given interval.
- ///
- /// # Examples
- ///
- /// ```
- /// use datafusion_common::ScalarValue;
- /// use datafusion_expr::Operator;
- /// use datafusion_physical_expr::intervals::{Interval, NullableInterval};
- ///
- /// // 4 > 3 -> true
- /// let lhs = NullableInterval::from(ScalarValue::Int32(Some(4)));
- /// let rhs = NullableInterval::from(ScalarValue::Int32(Some(3)));
- /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
- /// assert_eq!(result, NullableInterval::from(ScalarValue::Boolean(Some(true))));
- ///
- /// // [1, 3) > NULL -> NULL
- /// let lhs = NullableInterval::NotNull {
- /// values: Interval::make(Some(1), Some(3), (false, true)),
- /// };
- /// let rhs = NullableInterval::from(ScalarValue::Int32(None));
- /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
- /// assert_eq!(result.single_value(), Some(ScalarValue::Boolean(None)));
- ///
- /// // [1, 3] > [2, 4] -> [false, true]
- /// let lhs = NullableInterval::NotNull {
- /// values: Interval::make(Some(1), Some(3), (false, false)),
- /// };
- /// let rhs = NullableInterval::NotNull {
- /// values: Interval::make(Some(2), Some(4), (false, false)),
- /// };
- /// let result = lhs.apply_operator(&Operator::Gt, &rhs).unwrap();
- /// // Both inputs are valid (non-null), so result must be non-null
- /// assert_eq!(result, NullableInterval::NotNull {
- /// // Uncertain whether inequality is true or false
- /// values: Interval::UNCERTAIN,
- /// });
- ///
- /// ```
- pub fn apply_operator(&self, op: &Operator, rhs: &Self) -> Result<Self> {
- match op {
- Operator::IsDistinctFrom => {
- let values = match (self, rhs) {
- // NULL is distinct from NULL -> False
- (Self::Null { .. }, Self::Null { .. }) => Interval::CERTAINLY_FALSE,
- // x is distinct from y -> x != y,
- // if at least one of them is never null.
- (Self::NotNull { .. }, _) | (_, Self::NotNull { .. }) => {
- let lhs_values = self.values();
- let rhs_values = rhs.values();
- match (lhs_values, rhs_values) {
- (Some(lhs_values), Some(rhs_values)) => {
- lhs_values.equal(rhs_values).not()?
- }
- (Some(_), None) | (None, Some(_)) => Interval::CERTAINLY_TRUE,
- (None, None) => unreachable!("Null case handled above"),
- }
- }
- _ => Interval::UNCERTAIN,
- };
- // IsDistinctFrom never returns null.
- Ok(Self::NotNull { values })
- }
- Operator::IsNotDistinctFrom => self
- .apply_operator(&Operator::IsDistinctFrom, rhs)
- .map(|i| i.not())?,
- _ => {
- if let (Some(left_values), Some(right_values)) =
- (self.values(), rhs.values())
- {
- let values = apply_operator(op, left_values, right_values)?;
- match (self, rhs) {
- (Self::NotNull { .. }, Self::NotNull { .. }) => {
- Ok(Self::NotNull { values })
- }
- _ => Ok(Self::MaybeNull { values }),
- }
- } else if op.is_comparison_operator() {
- Ok(Self::Null {
- datatype: DataType::Boolean,
- })
- } else {
- Ok(Self::Null {
- datatype: self.get_datatype()?,
- })
- }
- }
- }
- }
-
- /// Determine if this interval contains the given interval. Returns a boolean
- /// interval that is [true, true] if this interval is a superset of the
- /// given interval, [false, false] if this interval is disjoint from the
- /// given interval, and [false, true] otherwise.
- pub fn contains<T: Borrow<Self>>(&self, other: T) -> Result<Self> {
- let rhs = other.borrow();
- if let (Some(left_values), Some(right_values)) = (self.values(), rhs.values()) {
- let values = left_values.contains(right_values)?;
- match (self, rhs) {
- (Self::NotNull { .. }, Self::NotNull { .. }) => {
- Ok(Self::NotNull { values })
- }
- _ => Ok(Self::MaybeNull { values }),
- }
- } else {
- Ok(Self::Null {
- datatype: DataType::Boolean,
- })
- }
- }
-
- /// If the interval has collapsed to a single value, return that value.
- ///
- /// Otherwise returns None.
- ///
- /// # Examples
- ///
- /// ```
- /// use datafusion_common::ScalarValue;
- /// use datafusion_physical_expr::intervals::{Interval, NullableInterval};
- ///
- /// let interval = NullableInterval::from(ScalarValue::Int32(Some(4)));
- /// assert_eq!(interval.single_value(), Some(ScalarValue::Int32(Some(4))));
- ///
- /// let interval = NullableInterval::from(ScalarValue::Int32(None));
- /// assert_eq!(interval.single_value(), Some(ScalarValue::Int32(None)));
- ///
- /// let interval = NullableInterval::MaybeNull {
- /// values: Interval::make(Some(1), Some(4), (false, true)),
- /// };
- /// assert_eq!(interval.single_value(), None);
- /// ```
- pub fn single_value(&self) -> Option<ScalarValue> {
- match self {
- Self::Null { datatype } => {
- Some(ScalarValue::try_from(datatype).unwrap_or(ScalarValue::Null))
- }
- Self::MaybeNull { values } | Self::NotNull { values }
- if values.lower.value == values.upper.value
- && !values.lower.is_unbounded() =>
- {
- Some(values.lower.value.clone())
- }
- _ => None,
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::next_value;
- use crate::intervals::{Interval, IntervalBound};
- use arrow_schema::DataType;
- use datafusion_common::{Result, ScalarValue};
-
- fn open_open<T>(lower: Option<T>, upper: Option<T>) -> Interval
- where
- ScalarValue: From<Option<T>>,
- {
- Interval::make(lower, upper, (true, true))
- }
-
- fn open_closed<T>(lower: Option<T>, upper: Option<T>) -> Interval
- where
- ScalarValue: From<Option<T>>,
- {
- Interval::make(lower, upper, (true, false))
- }
-
- fn closed_open<T>(lower: Option<T>, upper: Option<T>) -> Interval
- where
- ScalarValue: From<Option<T>>,
- {
- Interval::make(lower, upper, (false, true))
- }
-
- fn closed_closed<T>(lower: Option<T>, upper: Option<T>) -> Interval
- where
- ScalarValue: From<Option<T>>,
- {
- Interval::make(lower, upper, (false, false))
- }
-
- #[test]
- fn intersect_test() -> Result<()> {
- let possible_cases = vec![
- (Some(1000_i64), None, None, None, Some(1000_i64), None),
- (None, Some(1000_i64), None, None, None, Some(1000_i64)),
- (None, None, Some(1000_i64), None, Some(1000_i64), None),
- (None, None, None, Some(1000_i64), None, Some(1000_i64)),
- (
- Some(1000_i64),
- None,
- Some(1000_i64),
- None,
- Some(1000_i64),
- None,
- ),
- (
- None,
- Some(1000_i64),
- Some(999_i64),
- Some(1002_i64),
- Some(999_i64),
- Some(1000_i64),
- ),
- (None, None, None, None, None, None),
- ];
-
- for case in possible_cases {
- assert_eq!(
- open_open(case.0, case.1).intersect(open_open(case.2, case.3))?,
- Some(open_open(case.4, case.5))
- )
- }
-
- let empty_cases = vec![
- (None, Some(1000_i64), Some(1001_i64), None),
- (Some(1001_i64), None, None, Some(1000_i64)),
- (None, Some(1000_i64), Some(1001_i64), Some(1002_i64)),
- (Some(1001_i64), Some(1002_i64), None, Some(1000_i64)),
- ];
-
- for case in empty_cases {
- assert_eq!(
- open_open(case.0, case.1).intersect(open_open(case.2, case.3))?,
- None
- )
- }
-
- Ok(())
- }
-
- #[test]
- fn gt_test() {
- let cases = vec![
- (Some(1000_i64), None, None, None, false, true),
- (None, Some(1000_i64), None, None, false, true),
- (None, None, Some(1000_i64), None, false, true),
- (None, None, None, Some(1000_i64), false, true),
- (None, Some(1000_i64), Some(1000_i64), None, false, false),
- (None, Some(1000_i64), Some(1001_i64), None, false, false),
- (Some(1000_i64), None, Some(1000_i64), None, false, true),
- (
- None,
- Some(1000_i64),
- Some(1001_i64),
- Some(1002_i64),
- false,
- false,
- ),
- (
- None,
- Some(1000_i64),
- Some(999_i64),
- Some(1002_i64),
- false,
- true,
- ),
- (
- Some(1002_i64),
- None,
- Some(999_i64),
- Some(1002_i64),
- true,
- true,
- ),
- (
- Some(1003_i64),
- None,
- Some(999_i64),
- Some(1002_i64),
- true,
- true,
- ),
- (None, None, None, None, false, true),
- ];
-
- for case in cases {
- assert_eq!(
- open_open(case.0, case.1).gt(open_open(case.2, case.3)),
- closed_closed(Some(case.4), Some(case.5))
- );
- }
- }
-
- #[test]
- fn lt_test() {
- let cases = vec![
- (Some(1000_i64), None, None, None, false, true),
- (None, Some(1000_i64), None, None, false, true),
- (None, None, Some(1000_i64), None, false, true),
- (None, None, None, Some(1000_i64), false, true),
- (None, Some(1000_i64), Some(1000_i64), None, true, true),
- (None, Some(1000_i64), Some(1001_i64), None, true, true),
- (Some(1000_i64), None, Some(1000_i64), None, false, true),
- (
- None,
- Some(1000_i64),
- Some(1001_i64),
- Some(1002_i64),
- true,
- true,
- ),
- (
- None,
- Some(1000_i64),
- Some(999_i64),
- Some(1002_i64),
- false,
- true,
- ),
- (None, None, None, None, false, true),
- ];
-
- for case in cases {
- assert_eq!(
- open_open(case.0, case.1).lt(open_open(case.2, case.3)),
- closed_closed(Some(case.4), Some(case.5))
- );
- }
- }
-
- #[test]
- fn and_test() -> Result<()> {
- let cases = vec![
- (false, true, false, false, false, false),
- (false, false, false, true, false, false),
- (false, true, false, true, false, true),
- (false, true, true, true, false, true),
- (false, false, false, false, false, false),
- (true, true, true, true, true, true),
- ];
-
- for case in cases {
- assert_eq!(
- open_open(Some(case.0), Some(case.1))
- .and(open_open(Some(case.2), Some(case.3)))?,
- open_open(Some(case.4), Some(case.5))
- );
- }
- Ok(())
- }
-
- #[test]
- fn add_test() -> Result<()> {
- let cases = vec![
- (Some(1000_i64), None, None, None, None, None),
- (None, Some(1000_i64), None, None, None, None),
- (None, None, Some(1000_i64), None, None, None),
- (None, None, None, Some(1000_i64), None, None),
- (
- Some(1000_i64),
- None,
- Some(1000_i64),
- None,
- Some(2000_i64),
- None,
- ),
- (
- None,
- Some(1000_i64),
- Some(999_i64),
- Some(1002_i64),
- None,
- Some(2002_i64),
- ),
- (None, Some(1000_i64), Some(1000_i64), None, None, None),
- (
- Some(2001_i64),
- Some(1_i64),
- Some(1005_i64),
- Some(-999_i64),
- Some(3006_i64),
- Some(-998_i64),
- ),
- (None, None, None, None, None, None),
- ];
-
- for case in cases {
- assert_eq!(
- open_open(case.0, case.1).add(open_open(case.2, case.3))?,
- open_open(case.4, case.5)
- );
- }
- Ok(())
- }
-
- #[test]
- fn sub_test() -> Result<()> {
- let cases = vec![
- (Some(1000_i64), None, None, None, None, None),
- (None, Some(1000_i64), None, None, None, None),
- (None, None, Some(1000_i64), None, None, None),
- (None, None, None, Some(1000_i64), None, None),
- (Some(1000_i64), None, Some(1000_i64), None, None, None),
- (
- None,
- Some(1000_i64),
- Some(999_i64),
- Some(1002_i64),
- None,
- Some(1_i64),
- ),
- (
- None,
- Some(1000_i64),
- Some(1000_i64),
- None,
- None,
- Some(0_i64),
- ),
- (
- Some(2001_i64),
- Some(1000_i64),
- Some(1005),
- Some(999_i64),
- Some(1002_i64),
- Some(-5_i64),
- ),
- (None, None, None, None, None, None),
- ];
-
- for case in cases {
- assert_eq!(
- open_open(case.0, case.1).sub(open_open(case.2, case.3))?,
- open_open(case.4, case.5)
- );
- }
- Ok(())
- }
-
- #[test]
- fn sub_test_various_bounds() -> Result<()> {
- let cases = vec![
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- closed_open(Some(200_i64), None),
- open_closed(None, Some(0_i64)),
- ),
- (
- closed_open(Some(100_i64), Some(200_i64)),
- open_closed(Some(300_i64), Some(150_i64)),
- closed_open(Some(-50_i64), Some(-100_i64)),
- ),
- (
- closed_open(Some(100_i64), Some(200_i64)),
- open_open(Some(200_i64), None),
- open_open(None, Some(0_i64)),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- closed_closed(Some(11_i64), Some(11_i64)),
- closed_closed(Some(-10_i64), Some(-10_i64)),
- ),
- ];
- for case in cases {
- assert_eq!(case.0.sub(case.1)?, case.2)
- }
- Ok(())
- }
-
- #[test]
- fn add_test_various_bounds() -> Result<()> {
- let cases = vec![
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_closed(None, Some(200_i64)),
- open_closed(None, Some(400_i64)),
- ),
- (
- closed_open(Some(100_i64), Some(200_i64)),
- closed_open(Some(-300_i64), Some(150_i64)),
- closed_open(Some(-200_i64), Some(350_i64)),
- ),
- (
- closed_open(Some(100_i64), Some(200_i64)),
- open_open(Some(200_i64), None),
- open_open(Some(300_i64), None),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- closed_closed(Some(11_i64), Some(11_i64)),
- closed_closed(Some(12_i64), Some(12_i64)),
- ),
- ];
- for case in cases {
- assert_eq!(case.0.add(case.1)?, case.2)
- }
- Ok(())
- }
-
- #[test]
- fn lt_test_various_bounds() -> Result<()> {
- let cases = vec![
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_closed(None, Some(100_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_open(None, Some(100_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- (
- open_open(Some(100_i64), Some(200_i64)),
- closed_closed(Some(0_i64), Some(100_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_closed(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- open_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- ];
- for case in cases {
- assert_eq!(case.0.lt(case.1), case.2)
- }
- Ok(())
- }
-
- #[test]
- fn gt_test_various_bounds() -> Result<()> {
- let cases = vec![
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_closed(None, Some(100_i64)),
- closed_closed(Some(false), Some(true)),
- ),
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_open(None, Some(100_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- open_open(Some(100_i64), Some(200_i64)),
- closed_closed(Some(0_i64), Some(100_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_closed(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(true)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- open_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- ];
- for case in cases {
- assert_eq!(case.0.gt(case.1), case.2)
- }
- Ok(())
- }
-
- #[test]
- fn lt_eq_test_various_bounds() -> Result<()> {
- let cases = vec![
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_closed(None, Some(100_i64)),
- closed_closed(Some(false), Some(true)),
- ),
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_open(None, Some(100_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_closed(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(true)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- closed_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- open_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- ];
- for case in cases {
- assert_eq!(case.0.lt_eq(case.1), case.2)
- }
- Ok(())
- }
-
- #[test]
- fn gt_eq_test_various_bounds() -> Result<()> {
- let cases = vec![
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_closed(None, Some(100_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_open(None, Some(100_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_closed(Some(1_i64), Some(2_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(true), Some(true)),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- closed_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(true)),
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- open_open(Some(1_i64), Some(2_i64)),
- closed_closed(Some(false), Some(false)),
- ),
- ];
- for case in cases {
- assert_eq!(case.0.gt_eq(case.1), case.2)
- }
- Ok(())
- }
-
- #[test]
- fn intersect_test_various_bounds() -> Result<()> {
- let cases = vec![
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_closed(None, Some(100_i64)),
- Some(closed_closed(Some(100_i64), Some(100_i64))),
- ),
- (
- closed_closed(Some(100_i64), Some(200_i64)),
- open_open(None, Some(100_i64)),
- None,
- ),
- (
- open_open(Some(100_i64), Some(200_i64)),
- closed_closed(Some(0_i64), Some(100_i64)),
- None,
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_closed(Some(1_i64), Some(2_i64)),
- Some(closed_closed(Some(2_i64), Some(2_i64))),
- ),
- (
- closed_closed(Some(2_i64), Some(2_i64)),
- closed_open(Some(1_i64), Some(2_i64)),
- None,
- ),
- (
- closed_closed(Some(1_i64), Some(1_i64)),
- open_open(Some(1_i64), Some(2_i64)),
- None,
- ),
- (
- closed_closed(Some(1_i64), Some(3_i64)),
- open_open(Some(1_i64), Some(2_i64)),
- Some(open_open(Some(1_i64), Some(2_i64))),
- ),
- ];
- for case in cases {
- assert_eq!(case.0.intersect(case.1)?, case.2)
- }
- Ok(())
- }
-
- // This function tests if valid constructions produce standardized objects
- // ([false, false], [false, true], [true, true]) for boolean intervals.
- #[test]
- fn non_standard_interval_constructs() {
- use ScalarValue::Boolean;
- let cases = vec![
- (
- IntervalBound::new(Boolean(None), true),
- IntervalBound::new(Boolean(Some(true)), false),
- closed_closed(Some(false), Some(true)),
- ),
- (
- IntervalBound::new(Boolean(None), true),
- IntervalBound::new(Boolean(Some(true)), true),
- closed_closed(Some(false), Some(false)),
- ),
- (
- IntervalBound::new(Boolean(Some(false)), false),
- IntervalBound::new(Boolean(None), true),
- closed_closed(Some(false), Some(true)),
- ),
- (
- IntervalBound::new(Boolean(Some(true)), false),
- IntervalBound::new(Boolean(None), true),
- closed_closed(Some(true), Some(true)),
- ),
- (
- IntervalBound::new(Boolean(None), true),
- IntervalBound::new(Boolean(None), true),
- closed_closed(Some(false), Some(true)),
- ),
- (
- IntervalBound::new(Boolean(Some(false)), true),
- IntervalBound::new(Boolean(None), true),
- closed_closed(Some(true), Some(true)),
- ),
- ];
-
- for case in cases {
- assert_eq!(Interval::new(case.0, case.1), case.2)
- }
- }
-
- macro_rules! capture_mode_change {
- ($TYPE:ty) => {
- paste::item! {
- capture_mode_change_helper!([<capture_mode_change_ $TYPE>],
- [<create_interval_ $TYPE>],
- $TYPE);
- }
- };
- }
-
- macro_rules! capture_mode_change_helper {
- ($TEST_FN_NAME:ident, $CREATE_FN_NAME:ident, $TYPE:ty) => {
- fn $CREATE_FN_NAME(lower: $TYPE, upper: $TYPE) -> Interval {
- Interval::make(Some(lower as $TYPE), Some(upper as $TYPE), (true, true))
- }
-
- fn $TEST_FN_NAME(input: ($TYPE, $TYPE), expect_low: bool, expect_high: bool) {
- assert!(expect_low || expect_high);
- let interval1 = $CREATE_FN_NAME(input.0, input.0);
- let interval2 = $CREATE_FN_NAME(input.1, input.1);
- let result = interval1.add(&interval2).unwrap();
- let without_fe = $CREATE_FN_NAME(input.0 + input.1, input.0 + input.1);
- assert!(
- (!expect_low || result.lower.value < without_fe.lower.value)
- && (!expect_high || result.upper.value > without_fe.upper.value)
- );
- }
- };
- }
-
- capture_mode_change!(f32);
- capture_mode_change!(f64);
-
- #[cfg(all(
- any(target_arch = "x86_64", target_arch = "aarch64"),
- not(target_os = "windows")
- ))]
- #[test]
- fn test_add_intervals_lower_affected_f32() {
- // Lower is affected
- let lower = f32::from_bits(1073741887); //1000000000000000000000000111111
- let upper = f32::from_bits(1098907651); //1000001100000000000000000000011
- capture_mode_change_f32((lower, upper), true, false);
-
- // Upper is affected
- let lower = f32::from_bits(1072693248); //111111111100000000000000000000
- let upper = f32::from_bits(715827883); //101010101010101010101010101011
- capture_mode_change_f32((lower, upper), false, true);
-
- // Lower is affected
- let lower = 1.0; // 0x3FF0000000000000
- let upper = 0.3; // 0x3FD3333333333333
- capture_mode_change_f64((lower, upper), true, false);
-
- // Upper is affected
- let lower = 1.4999999999999998; // 0x3FF7FFFFFFFFFFFF
- let upper = 0.000_000_000_000_000_022_044_604_925_031_31; // 0x3C796A6B413BB21F
- capture_mode_change_f64((lower, upper), false, true);
- }
-
- #[cfg(any(
- not(any(target_arch = "x86_64", target_arch = "aarch64")),
- target_os = "windows"
- ))]
- #[test]
- fn test_next_impl_add_intervals_f64() {
- let lower = 1.5;
- let upper = 1.5;
- capture_mode_change_f64((lower, upper), true, true);
-
- let lower = 1.5;
- let upper = 1.5;
- capture_mode_change_f32((lower, upper), true, true);
- }
-
- #[test]
- fn test_cardinality_of_intervals() -> Result<()> {
- // In IEEE 754 standard for floating-point arithmetic, if we keep the sign and exponent fields same,
- // we can represent 4503599627370496 different numbers by changing the mantissa
- // (4503599627370496 = 2^52, since there are 52 bits in mantissa, and 2^23 = 8388608 for f32).
- let distinct_f64 = 4503599627370496;
- let distinct_f32 = 8388608;
- let intervals = [
- Interval::new(
- IntervalBound::new(ScalarValue::from(0.25), false),
- IntervalBound::new(ScalarValue::from(0.50), true),
- ),
- Interval::new(
- IntervalBound::new(ScalarValue::from(0.5), false),
- IntervalBound::new(ScalarValue::from(1.0), true),
- ),
- Interval::new(
- IntervalBound::new(ScalarValue::from(1.0), false),
- IntervalBound::new(ScalarValue::from(2.0), true),
- ),
- Interval::new(
- IntervalBound::new(ScalarValue::from(32.0), false),
- IntervalBound::new(ScalarValue::from(64.0), true),
- ),
- Interval::new(
- IntervalBound::new(ScalarValue::from(-0.50), false),
- IntervalBound::new(ScalarValue::from(-0.25), true),
- ),
- Interval::new(
- IntervalBound::new(ScalarValue::from(-32.0), false),
- IntervalBound::new(ScalarValue::from(-16.0), true),
- ),
- ];
- for interval in intervals {
- assert_eq!(interval.cardinality()?.unwrap(), distinct_f64);
- }
-
- let intervals = [
- Interval::new(
- IntervalBound::new(ScalarValue::from(0.25_f32), false),
- IntervalBound::new(ScalarValue::from(0.50_f32), true),
- ),
- Interval::new(
- IntervalBound::new(ScalarValue::from(-1_f32), false),
- IntervalBound::new(ScalarValue::from(-0.5_f32), true),
- ),
- ];
- for interval in intervals {
- assert_eq!(interval.cardinality()?.unwrap(), distinct_f32);
- }
-
- // The regular logarithmic distribution of floating-point numbers are
- // only applicable outside of the `(-phi, phi)` interval where `phi`
- // denotes the largest positive subnormal floating-point number. Since
- // the following intervals include such subnormal points, we cannot use
- // a simple powers-of-two type formula for our expectations. Therefore,
- // we manually supply the actual expected cardinality.
- let interval = Interval::new(
- IntervalBound::new(ScalarValue::from(-0.0625), false),
- IntervalBound::new(ScalarValue::from(0.0625), true),
- );
- assert_eq!(interval.cardinality()?.unwrap(), 9178336040581070849);
-
- let interval = Interval::new(
- IntervalBound::new(ScalarValue::from(-0.0625_f32), false),
- IntervalBound::new(ScalarValue::from(0.0625_f32), true),
- );
- assert_eq!(interval.cardinality()?.unwrap(), 2063597569);
-
- Ok(())
- }
-
- #[test]
- fn test_next_value() -> Result<()> {
- // integer increment / decrement
- let zeros = vec![
- ScalarValue::new_zero(&DataType::UInt8)?,
- ScalarValue::new_zero(&DataType::UInt16)?,
- ScalarValue::new_zero(&DataType::UInt32)?,
- ScalarValue::new_zero(&DataType::UInt64)?,
- ScalarValue::new_zero(&DataType::Int8)?,
- ScalarValue::new_zero(&DataType::Int8)?,
- ScalarValue::new_zero(&DataType::Int8)?,
- ScalarValue::new_zero(&DataType::Int8)?,
- ];
-
- let ones = vec![
- ScalarValue::new_one(&DataType::UInt8)?,
- ScalarValue::new_one(&DataType::UInt16)?,
- ScalarValue::new_one(&DataType::UInt32)?,
- ScalarValue::new_one(&DataType::UInt64)?,
- ScalarValue::new_one(&DataType::Int8)?,
- ScalarValue::new_one(&DataType::Int8)?,
- ScalarValue::new_one(&DataType::Int8)?,
- ScalarValue::new_one(&DataType::Int8)?,
- ];
-
- zeros.into_iter().zip(ones).for_each(|(z, o)| {
- assert_eq!(next_value::<true>(z.clone()), o);
- assert_eq!(next_value::<false>(o), z);
- });
-
- // floating value increment / decrement
- let values = vec![
- ScalarValue::new_zero(&DataType::Float32)?,
- ScalarValue::new_zero(&DataType::Float64)?,
- ];
-
- let eps = vec![
- ScalarValue::Float32(Some(1e-6)),
- ScalarValue::Float64(Some(1e-6)),
- ];
-
- values.into_iter().zip(eps).for_each(|(v, e)| {
- assert!(next_value::<true>(v.clone()).sub(v.clone()).unwrap().lt(&e));
- assert!(v.clone().sub(next_value::<false>(v)).unwrap().lt(&e));
- });
-
- // Min / Max values do not change for integer values
- let min = vec![
- ScalarValue::UInt64(Some(u64::MIN)),
- ScalarValue::Int8(Some(i8::MIN)),
- ];
- let max = vec![
- ScalarValue::UInt64(Some(u64::MAX)),
- ScalarValue::Int8(Some(i8::MAX)),
- ];
-
- min.into_iter().zip(max).for_each(|(min, max)| {
- assert_eq!(next_value::<true>(max.clone()), max);
- assert_eq!(next_value::<false>(min.clone()), min);
- });
-
- // Min / Max values results in infinity for floating point values
- assert_eq!(
- next_value::<true>(ScalarValue::Float32(Some(f32::MAX))),
- ScalarValue::Float32(Some(f32::INFINITY))
- );
- assert_eq!(
- next_value::<false>(ScalarValue::Float64(Some(f64::MIN))),
- ScalarValue::Float64(Some(f64::NEG_INFINITY))
- );
-
- Ok(())
- }
-
- #[test]
- fn test_interval_display() {
- let interval = Interval::new(
- IntervalBound::new(ScalarValue::from(0.25_f32), true),
- IntervalBound::new(ScalarValue::from(0.50_f32), false),
- );
- assert_eq!(format!("{}", interval), "(0.25, 0.5]");
-
- let interval = Interval::new(
- IntervalBound::new(ScalarValue::from(0.25_f32), false),
- IntervalBound::new(ScalarValue::from(0.50_f32), true),
- );
- assert_eq!(format!("{}", interval), "[0.25, 0.5)");
-
- let interval = Interval::new(
- IntervalBound::new(ScalarValue::from(0.25_f32), true),
- IntervalBound::new(ScalarValue::from(0.50_f32), true),
- );
- assert_eq!(format!("{}", interval), "(0.25, 0.5)");
-
- let interval = Interval::new(
- IntervalBound::new(ScalarValue::from(0.25_f32), false),
- IntervalBound::new(ScalarValue::from(0.50_f32), false),
- );
- assert_eq!(format!("{}", interval), "[0.25, 0.5]");
- }
-}
diff --git a/datafusion/physical-expr/src/intervals/mod.rs b/datafusion/physical-expr/src/intervals/mod.rs
index b89d1c59dc..9752ca27b5 100644
--- a/datafusion/physical-expr/src/intervals/mod.rs
+++ b/datafusion/physical-expr/src/intervals/mod.rs
@@ -18,10 +18,5 @@
//! Interval arithmetic and constraint propagation library
pub mod cp_solver;
-pub mod interval_aritmetic;
-pub mod rounding;
pub mod test_utils;
pub mod utils;
-
-pub use cp_solver::ExprIntervalGraph;
-pub use interval_aritmetic::*;
diff --git a/datafusion/physical-expr/src/intervals/utils.rs b/datafusion/physical-expr/src/intervals/utils.rs
index 7a4ccff950..03d1363210 100644
--- a/datafusion/physical-expr/src/intervals/utils.rs
+++ b/datafusion/physical-expr/src/intervals/utils.rs
@@ -19,14 +19,16 @@
use std::sync::Arc;
-use super::{Interval, IntervalBound};
use crate::{
expressions::{BinaryExpr, CastExpr, Column, Literal, NegativeExpr},
PhysicalExpr,
};
use arrow_schema::{DataType, SchemaRef};
-use datafusion_common::{DataFusionError, Result, ScalarValue};
+use datafusion_common::{
+ internal_datafusion_err, internal_err, DataFusionError, Result, ScalarValue,
+};
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_expr::Operator;
const MDN_DAY_MASK: i128 = 0xFFFF_FFFF_0000_0000_0000_0000;
@@ -66,11 +68,13 @@ pub fn check_support(expr: &Arc<dyn PhysicalExpr>, schema: &SchemaRef) -> bool {
}
// This function returns the inverse operator of the given operator.
-pub fn get_inverse_op(op: Operator) -> Operator {
+pub fn get_inverse_op(op: Operator) -> Result<Operator> {
match op {
- Operator::Plus => Operator::Minus,
- Operator::Minus => Operator::Plus,
- _ => unreachable!(),
+ Operator::Plus => Ok(Operator::Minus),
+ Operator::Minus => Ok(Operator::Plus),
+ Operator::Multiply => Ok(Operator::Divide),
+ Operator::Divide => Ok(Operator::Multiply),
+ _ => internal_err!("Interval arithmetic does not support the operator {}", op),
}
}
@@ -86,6 +90,8 @@ pub fn is_operator_supported(op: &Operator) -> bool {
| &Operator::Lt
| &Operator::LtEq
| &Operator::Eq
+ | &Operator::Multiply
+ | &Operator::Divide
)
}
@@ -109,36 +115,26 @@ pub fn is_datatype_supported(data_type: &DataType) -> bool {
/// Converts an [`Interval`] of time intervals to one of `Duration`s, if applicable. Otherwise, returns [`None`].
pub fn convert_interval_type_to_duration(interval: &Interval) -> Option<Interval> {
if let (Some(lower), Some(upper)) = (
- convert_interval_bound_to_duration(&interval.lower),
- convert_interval_bound_to_duration(&interval.upper),
+ convert_interval_bound_to_duration(interval.lower()),
+ convert_interval_bound_to_duration(interval.upper()),
) {
- Some(Interval::new(lower, upper))
+ Interval::try_new(lower, upper).ok()
} else {
None
}
}
-/// Converts an [`IntervalBound`] containing a time interval to one containing a `Duration`, if applicable. Otherwise, returns [`None`].
+/// Converts an [`ScalarValue`] containing a time interval to one containing a `Duration`, if applicable. Otherwise, returns [`None`].
fn convert_interval_bound_to_duration(
- interval_bound: &IntervalBound,
-) -> Option<IntervalBound> {
- match interval_bound.value {
- ScalarValue::IntervalMonthDayNano(Some(mdn)) => {
- interval_mdn_to_duration_ns(&mdn).ok().map(|duration| {
- IntervalBound::new(
- ScalarValue::DurationNanosecond(Some(duration)),
- interval_bound.open,
- )
- })
- }
- ScalarValue::IntervalDayTime(Some(dt)) => {
- interval_dt_to_duration_ms(&dt).ok().map(|duration| {
- IntervalBound::new(
- ScalarValue::DurationMillisecond(Some(duration)),
- interval_bound.open,
- )
- })
- }
+ interval_bound: &ScalarValue,
+) -> Option<ScalarValue> {
+ match interval_bound {
+ ScalarValue::IntervalMonthDayNano(Some(mdn)) => interval_mdn_to_duration_ns(mdn)
+ .ok()
+ .map(|duration| ScalarValue::DurationNanosecond(Some(duration))),
+ ScalarValue::IntervalDayTime(Some(dt)) => interval_dt_to_duration_ms(dt)
+ .ok()
+ .map(|duration| ScalarValue::DurationMillisecond(Some(duration))),
_ => None,
}
}
@@ -146,28 +142,32 @@ fn convert_interval_bound_to_duration(
/// Converts an [`Interval`] of `Duration`s to one of time intervals, if applicable. Otherwise, returns [`None`].
pub fn convert_duration_type_to_interval(interval: &Interval) -> Option<Interval> {
if let (Some(lower), Some(upper)) = (
- convert_duration_bound_to_interval(&interval.lower),
- convert_duration_bound_to_interval(&interval.upper),
+ convert_duration_bound_to_interval(interval.lower()),
+ convert_duration_bound_to_interval(interval.upper()),
) {
- Some(Interval::new(lower, upper))
+ Interval::try_new(lower, upper).ok()
} else {
None
}
}
-/// Converts an [`IntervalBound`] containing a `Duration` to one containing a time interval, if applicable. Otherwise, returns [`None`].
+/// Converts a [`ScalarValue`] containing a `Duration` to one containing a time interval, if applicable. Otherwise, returns [`None`].
fn convert_duration_bound_to_interval(
- interval_bound: &IntervalBound,
-) -> Option<IntervalBound> {
- match interval_bound.value {
- ScalarValue::DurationNanosecond(Some(duration)) => Some(IntervalBound::new(
- ScalarValue::new_interval_mdn(0, 0, duration),
- interval_bound.open,
- )),
- ScalarValue::DurationMillisecond(Some(duration)) => Some(IntervalBound::new(
- ScalarValue::new_interval_dt(0, duration as i32),
- interval_bound.open,
- )),
+ interval_bound: &ScalarValue,
+) -> Option<ScalarValue> {
+ match interval_bound {
+ ScalarValue::DurationNanosecond(Some(duration)) => {
+ Some(ScalarValue::new_interval_mdn(0, 0, *duration))
+ }
+ ScalarValue::DurationMicrosecond(Some(duration)) => {
+ Some(ScalarValue::new_interval_mdn(0, 0, *duration * 1000))
+ }
+ ScalarValue::DurationMillisecond(Some(duration)) => {
+ Some(ScalarValue::new_interval_dt(0, *duration as i32))
+ }
+ ScalarValue::DurationSecond(Some(duration)) => {
+ Some(ScalarValue::new_interval_dt(0, *duration as i32 * 1000))
+ }
_ => None,
}
}
@@ -180,14 +180,13 @@ fn interval_mdn_to_duration_ns(mdn: &i128) -> Result<i64> {
let nanoseconds = mdn & MDN_NS_MASK;
if months == 0 && days == 0 {
- nanoseconds.try_into().map_err(|_| {
- DataFusionError::Internal("Resulting duration exceeds i64::MAX".to_string())
- })
+ nanoseconds
+ .try_into()
+ .map_err(|_| internal_datafusion_err!("Resulting duration exceeds i64::MAX"))
} else {
- Err(DataFusionError::Internal(
+ internal_err!(
"The interval cannot have a non-zero month or day value for duration convertibility"
- .to_string(),
- ))
+ )
}
}
@@ -200,9 +199,8 @@ fn interval_dt_to_duration_ms(dt: &i64) -> Result<i64> {
if days == 0 {
Ok(milliseconds)
} else {
- Err(DataFusionError::Internal(
+ internal_err!(
"The interval cannot have a non-zero day value for duration convertibility"
- .to_string(),
- ))
+ )
}
}
diff --git a/datafusion/physical-expr/src/physical_expr.rs b/datafusion/physical-expr/src/physical_expr.rs
index 455ca84a79..a8d1e3638a 100644
--- a/datafusion/physical-expr/src/physical_expr.rs
+++ b/datafusion/physical-expr/src/physical_expr.rs
@@ -20,7 +20,6 @@ use std::fmt::{Debug, Display};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
-use crate::intervals::Interval;
use crate::sort_properties::SortProperties;
use crate::utils::scatter;
@@ -30,6 +29,7 @@ use arrow::datatypes::{DataType, Schema};
use arrow::record_batch::RecordBatch;
use datafusion_common::utils::DataPtr;
use datafusion_common::{internal_err, not_impl_err, DataFusionError, Result};
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_expr::ColumnarValue;
use itertools::izip;
@@ -95,36 +95,34 @@ pub trait PhysicalExpr: Send + Sync + Display + Debug + PartialEq<dyn Any> {
/// Updates bounds for child expressions, given a known interval for this
/// expression.
///
- /// This is used to propagate constraints down through an
- /// expression tree.
+ /// This is used to propagate constraints down through an expression tree.
///
/// # Arguments
///
/// * `interval` is the currently known interval for this expression.
- /// * `children` are the current intervals for the children of this expression
+ /// * `children` are the current intervals for the children of this expression.
///
/// # Returns
///
- /// A Vec of new intervals for the children, in order.
+ /// A `Vec` of new intervals for the children, in order.
///
- /// If constraint propagation reveals an infeasibility, returns [None] for
- /// the child causing infeasibility.
- ///
- /// If none of the child intervals change as a result of propagation, may
- /// return an empty vector instead of cloning `children`.
+ /// If constraint propagation reveals an infeasibility for any child, returns
+ /// [`None`]. If none of the children intervals change as a result of propagation,
+ /// may return an empty vector instead of cloning `children`. This is the default
+ /// (and conservative) return value.
///
/// # Example
///
- /// If the expression is `a + b`, the current `interval` is `[4, 5] and the
- /// inputs are given [`a: [0, 2], `b: [-∞, 4]]`, then propagation would
- /// would return `[a: [0, 2], b: [2, 4]]` as `b` must be at least 2 to
- /// make the output at least `4`.
+ /// If the expression is `a + b`, the current `interval` is `[4, 5]` and the
+ /// inputs `a` and `b` are respectively given as `[0, 2]` and `[-∞, 4]`, then
+ /// propagation would would return `[0, 2]` and `[2, 4]` as `b` must be at
+ /// least `2` to make the output at least `4`.
fn propagate_constraints(
&self,
_interval: &Interval,
_children: &[&Interval],
- ) -> Result<Vec<Option<Interval>>> {
- not_impl_err!("Not implemented for {self}")
+ ) -> Result<Option<Vec<Interval>>> {
+ Ok(Some(vec![]))
}
/// Update the hash `state` with this expression requirements from
diff --git a/datafusion/physical-expr/src/sort_properties.rs b/datafusion/physical-expr/src/sort_properties.rs
index a3b201f84e..25729640ec 100644
--- a/datafusion/physical-expr/src/sort_properties.rs
+++ b/datafusion/physical-expr/src/sort_properties.rs
@@ -99,7 +99,7 @@ impl SortProperties {
}
}
- pub fn and(&self, rhs: &Self) -> Self {
+ pub fn and_or(&self, rhs: &Self) -> Self {
match (self, rhs) {
(Self::Ordered(lhs), Self::Ordered(rhs))
if lhs.descending == rhs.descending =>
diff --git a/datafusion/physical-expr/src/utils.rs b/datafusion/physical-expr/src/utils.rs
index 2f4ee89463..ed62956de8 100644
--- a/datafusion/physical-expr/src/utils.rs
+++ b/datafusion/physical-expr/src/utils.rs
@@ -183,7 +183,7 @@ impl<T: Clone> TreeNode for ExprTreeNode<T> {
/// identical expressions in one node. Caller specifies the node type in the
/// DAEG via the `constructor` argument, which constructs nodes in the DAEG
/// from the [ExprTreeNode] ancillary object.
-struct PhysicalExprDAEGBuilder<'a, T, F: Fn(&ExprTreeNode<NodeIndex>) -> T> {
+struct PhysicalExprDAEGBuilder<'a, T, F: Fn(&ExprTreeNode<NodeIndex>) -> Result<T>> {
// The resulting DAEG (expression DAG).
graph: StableGraph<T, usize>,
// A vector of visited expression nodes and their corresponding node indices.
@@ -192,7 +192,7 @@ struct PhysicalExprDAEGBuilder<'a, T, F: Fn(&ExprTreeNode<NodeIndex>) -> T> {
constructor: &'a F,
}
-impl<'a, T, F: Fn(&ExprTreeNode<NodeIndex>) -> T> TreeNodeRewriter
+impl<'a, T, F: Fn(&ExprTreeNode<NodeIndex>) -> Result<T>> TreeNodeRewriter
for PhysicalExprDAEGBuilder<'a, T, F>
{
type N = ExprTreeNode<NodeIndex>;
@@ -213,7 +213,7 @@ impl<'a, T, F: Fn(&ExprTreeNode<NodeIndex>) -> T> TreeNodeRewriter
// add edges to its child nodes. Add the visited expression to the vector
// of visited expressions and return the newly created node index.
None => {
- let node_idx = self.graph.add_node((self.constructor)(&node));
+ let node_idx = self.graph.add_node((self.constructor)(&node)?);
for expr_node in node.child_nodes.iter() {
self.graph.add_edge(node_idx, expr_node.data.unwrap(), 0);
}
@@ -234,7 +234,7 @@ pub fn build_dag<T, F>(
constructor: &F,
) -> Result<(NodeIndex, StableGraph<T, usize>)>
where
- F: Fn(&ExprTreeNode<NodeIndex>) -> T,
+ F: Fn(&ExprTreeNode<NodeIndex>) -> Result<T>,
{
// Create a new expression tree node from the input expression.
let init = ExprTreeNode::new(expr);
@@ -394,7 +394,7 @@ mod tests {
}
}
- fn make_dummy_node(node: &ExprTreeNode<NodeIndex>) -> PhysicalExprDummyNode {
+ fn make_dummy_node(node: &ExprTreeNode<NodeIndex>) -> Result<PhysicalExprDummyNode> {
let expr = node.expression().clone();
let dummy_property = if expr.as_any().is::<BinaryExpr>() {
"Binary"
@@ -406,12 +406,12 @@ mod tests {
"Other"
}
.to_owned();
- PhysicalExprDummyNode {
+ Ok(PhysicalExprDummyNode {
expr,
property: DummyProperty {
expr_type: dummy_property,
},
- }
+ })
}
#[test]
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index b6cd9fe79c..903f4c972e 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -27,7 +27,6 @@ use super::expressions::PhysicalSortExpr;
use super::{
ColumnStatistics, DisplayAs, RecordBatchStream, SendableRecordBatchStream, Statistics,
};
-
use crate::{
metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet},
Column, DisplayFormatType, ExecutionPlan, Partitioning,
@@ -215,7 +214,8 @@ impl ExecutionPlan for FilterExec {
&self.input.schema(),
&input_stats.column_statistics,
)?;
- let analysis_ctx = analyze(predicate, input_analysis_ctx)?;
+
+ let analysis_ctx = analyze(predicate, input_analysis_ctx, &self.schema())?;
// Estimate (inexact) selectivity of predicate
let selectivity = analysis_ctx.selectivity.unwrap_or(1.0);
@@ -254,19 +254,12 @@ fn collect_new_statistics(
..
},
)| {
- let closed_interval = interval.close_bounds();
- let (min_value, max_value) =
- if closed_interval.lower.value.eq(&closed_interval.upper.value) {
- (
- Precision::Exact(closed_interval.lower.value),
- Precision::Exact(closed_interval.upper.value),
- )
- } else {
- (
- Precision::Inexact(closed_interval.lower.value),
- Precision::Inexact(closed_interval.upper.value),
- )
- };
+ let (lower, upper) = interval.into_bounds();
+ let (min_value, max_value) = if lower.eq(&upper) {
+ (Precision::Exact(lower), Precision::Exact(upper))
+ } else {
+ (Precision::Inexact(lower), Precision::Inexact(upper))
+ };
ColumnStatistics {
null_count: input_column_stats[idx].null_count.clone().to_inexact(),
max_value,
diff --git a/datafusion/physical-plan/src/joins/stream_join_utils.rs b/datafusion/physical-plan/src/joins/stream_join_utils.rs
index aa57a4f896..5083f96b01 100644
--- a/datafusion/physical-plan/src/joins/stream_join_utils.rs
+++ b/datafusion/physical-plan/src/joins/stream_join_utils.rs
@@ -29,13 +29,13 @@ use crate::joins::utils::{JoinFilter, JoinHashMapType};
use arrow::compute::concat_batches;
use arrow_array::{ArrowPrimitiveType, NativeAdapter, PrimitiveArray, RecordBatch};
use arrow_buffer::{ArrowNativeType, BooleanBufferBuilder};
-use arrow_schema::SchemaRef;
+use arrow_schema::{Schema, SchemaRef};
use async_trait::async_trait;
use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::{DataFusionError, JoinSide, Result, ScalarValue};
use datafusion_execution::SendableRecordBatchStream;
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_physical_expr::expressions::Column;
-use datafusion_physical_expr::intervals::{Interval, IntervalBound};
use datafusion_physical_expr::utils::collect_columns;
use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr};
@@ -316,7 +316,11 @@ pub fn build_filter_input_order(
order: &PhysicalSortExpr,
) -> Result<Option<SortedFilterExpr>> {
let opt_expr = convert_sort_expr_with_filter_schema(&side, filter, schema, order)?;
- Ok(opt_expr.map(|filter_expr| SortedFilterExpr::new(order.clone(), filter_expr)))
+ opt_expr
+ .map(|filter_expr| {
+ SortedFilterExpr::try_new(order.clone(), filter_expr, filter.schema())
+ })
+ .transpose()
}
/// Convert a physical expression into a filter expression using the given
@@ -359,16 +363,18 @@ pub struct SortedFilterExpr {
impl SortedFilterExpr {
/// Constructor
- pub fn new(
+ pub fn try_new(
origin_sorted_expr: PhysicalSortExpr,
filter_expr: Arc<dyn PhysicalExpr>,
- ) -> Self {
- Self {
+ filter_schema: &Schema,
+ ) -> Result<Self> {
+ let dt = &filter_expr.data_type(filter_schema)?;
+ Ok(Self {
origin_sorted_expr,
filter_expr,
- interval: Interval::default(),
+ interval: Interval::make_unbounded(dt)?,
node_index: 0,
- }
+ })
}
/// Get origin expr information
pub fn origin_sorted_expr(&self) -> &PhysicalSortExpr {
@@ -494,12 +500,12 @@ pub fn update_filter_expr_interval(
// Convert the array to a ScalarValue:
let value = ScalarValue::try_from_array(&array, 0)?;
// Create a ScalarValue representing positive or negative infinity for the same data type:
- let unbounded = IntervalBound::make_unbounded(value.data_type())?;
+ let inf = ScalarValue::try_from(value.data_type())?;
// Update the interval with lower and upper bounds based on the sort option:
let interval = if sorted_expr.origin_sorted_expr().options.descending {
- Interval::new(unbounded, IntervalBound::new(value, false))
+ Interval::try_new(inf, value)?
} else {
- Interval::new(IntervalBound::new(value, false), unbounded)
+ Interval::try_new(value, inf)?
};
// Set the calculated interval for the sorted filter expression:
sorted_expr.set_interval(interval);
@@ -1024,14 +1030,13 @@ pub mod tests {
convert_sort_expr_with_filter_schema, PruningJoinHashMap,
};
use crate::{
- expressions::Column,
- expressions::PhysicalSortExpr,
+ expressions::{Column, PhysicalSortExpr},
joins::utils::{ColumnIndex, JoinFilter},
};
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema};
- use datafusion_common::ScalarValue;
+ use datafusion_common::{JoinSide, ScalarValue};
use datafusion_expr::Operator;
use datafusion_physical_expr::expressions::{binary, cast, col, lit};
diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
index d653297abe..95f15877b9 100644
--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
@@ -62,8 +62,9 @@ use datafusion_common::{
};
use datafusion_execution::memory_pool::MemoryConsumer;
use datafusion_execution::TaskContext;
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_physical_expr::equivalence::join_equivalence_properties;
-use datafusion_physical_expr::intervals::ExprIntervalGraph;
+use datafusion_physical_expr::intervals::cp_solver::ExprIntervalGraph;
use ahash::RandomState;
use futures::Stream;
@@ -631,9 +632,9 @@ fn determine_prune_length(
// Get the lower or upper interval based on the sort direction
let target = if origin_sorted_expr.options.descending {
- interval.upper.value.clone()
+ interval.upper().clone()
} else {
- interval.lower.value.clone()
+ interval.lower().clone()
};
// Perform binary search on the array to determine the length of the record batch to be pruned
@@ -975,7 +976,7 @@ impl OneSideHashJoiner {
filter_intervals.push((expr.node_index(), expr.interval().clone()))
}
// Update the physical expression graph using the join filter intervals:
- graph.update_ranges(&mut filter_intervals)?;
+ graph.update_ranges(&mut filter_intervals, Interval::CERTAINLY_TRUE)?;
// Extract the new join filter interval for the build side:
let calculated_build_side_interval = filter_intervals.remove(0).1;
// If the intervals have not changed, return early without pruning:
@@ -1948,7 +1949,7 @@ mod tests {
(12, 17),
)]
cardinality: (i32, i32),
- #[values(0, 1)] case_expr: usize,
+ #[values(0, 1, 2)] case_expr: usize,
) -> Result<()> {
let session_config = SessionConfig::new().with_repartition_joins(false);
let task_ctx = TaskContext::default().with_session_config(session_config);
diff --git a/datafusion/physical-plan/src/joins/test_utils.rs b/datafusion/physical-plan/src/joins/test_utils.rs
index 6deaa9ba1b..fbd52ddf0c 100644
--- a/datafusion/physical-plan/src/joins/test_utils.rs
+++ b/datafusion/physical-plan/src/joins/test_utils.rs
@@ -243,6 +243,20 @@ pub fn join_expr_tests_fixture_temporal(
ScalarValue::TimestampMillisecond(Some(1672574402000), None), // 2023-01-01:12.00.02
schema,
),
+ // constructs ((left_col - DURATION '3 secs') > (right_col - DURATION '2 secs')) AND ((left_col - DURATION '5 secs') < (right_col - DURATION '4 secs'))
+ 2 => gen_conjunctive_temporal_expr(
+ left_col,
+ right_col,
+ Operator::Minus,
+ Operator::Minus,
+ Operator::Minus,
+ Operator::Minus,
+ ScalarValue::DurationMillisecond(Some(3000)), // 3 secs
+ ScalarValue::DurationMillisecond(Some(2000)), // 2 secs
+ ScalarValue::DurationMillisecond(Some(5000)), // 5 secs
+ ScalarValue::DurationMillisecond(Some(4000)), // 4 secs
+ schema,
+ ),
_ => unreachable!(),
}
}
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index 0729d365d6..5e01ca227c 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -42,9 +42,10 @@ use datafusion_common::{
plan_datafusion_err, plan_err, DataFusionError, JoinSide, JoinType, Result,
SharedResult,
};
+use datafusion_expr::interval_arithmetic::Interval;
use datafusion_physical_expr::equivalence::add_offset_to_expr;
use datafusion_physical_expr::expressions::Column;
-use datafusion_physical_expr::intervals::{ExprIntervalGraph, Interval, IntervalBound};
+use datafusion_physical_expr::intervals::cp_solver::ExprIntervalGraph;
use datafusion_physical_expr::utils::merge_vectors;
use datafusion_physical_expr::{
LexOrdering, LexOrderingRef, PhysicalExpr, PhysicalSortExpr,
@@ -713,8 +714,8 @@ fn estimate_inner_join_cardinality(
);
}
- let left_max_distinct = max_distinct_count(&left_stats.num_rows, left_stat)?;
- let right_max_distinct = max_distinct_count(&right_stats.num_rows, right_stat)?;
+ let left_max_distinct = max_distinct_count(&left_stats.num_rows, left_stat);
+ let right_max_distinct = max_distinct_count(&right_stats.num_rows, right_stat);
let max_distinct = left_max_distinct.max(&right_max_distinct);
if max_distinct.get_value().is_some() {
// Seems like there are a few implementations of this algorithm that implement
@@ -745,48 +746,60 @@ fn estimate_inner_join_cardinality(
}
/// Estimate the number of maximum distinct values that can be present in the
-/// given column from its statistics.
-///
-/// If distinct_count is available, uses it directly. If the column numeric, and
-/// has min/max values, then they might be used as a fallback option. Otherwise,
-/// returns None.
+/// given column from its statistics. If distinct_count is available, uses it
+/// directly. Otherwise, if the column is numeric and has min/max values, it
+/// estimates the maximum distinct count from those.
fn max_distinct_count(
num_rows: &Precision<usize>,
stats: &ColumnStatistics,
-) -> Option<Precision<usize>> {
- match (
- &stats.distinct_count,
- stats.max_value.get_value(),
- stats.min_value.get_value(),
- ) {
- (Precision::Exact(_), _, _) | (Precision::Inexact(_), _, _) => {
- Some(stats.distinct_count.clone())
- }
- (_, Some(max), Some(min)) => {
- let numeric_range = Interval::new(
- IntervalBound::new(min.clone(), false),
- IntervalBound::new(max.clone(), false),
- )
- .cardinality()
- .ok()
- .flatten()? as usize;
-
- // The number can never be greater than the number of rows we have (minus
- // the nulls, since they don't count as distinct values).
- let ceiling =
- num_rows.get_value()? - stats.null_count.get_value().unwrap_or(&0);
- Some(
- if num_rows.is_exact().unwrap_or(false)
- && stats.max_value.is_exact().unwrap_or(false)
- && stats.min_value.is_exact().unwrap_or(false)
+) -> Precision<usize> {
+ match &stats.distinct_count {
+ dc @ (Precision::Exact(_) | Precision::Inexact(_)) => dc.clone(),
+ _ => {
+ // The number can never be greater than the number of rows we have
+ // minus the nulls (since they don't count as distinct values).
+ let result = match num_rows {
+ Precision::Absent => Precision::Absent,
+ Precision::Inexact(count) => {
+ Precision::Inexact(count - stats.null_count.get_value().unwrap_or(&0))
+ }
+ Precision::Exact(count) => {
+ let count = count - stats.null_count.get_value().unwrap_or(&0);
+ if stats.null_count.is_exact().unwrap_or(false) {
+ Precision::Exact(count)
+ } else {
+ Precision::Inexact(count)
+ }
+ }
+ };
+ // Cap the estimate using the number of possible values:
+ if let (Some(min), Some(max)) =
+ (stats.min_value.get_value(), stats.max_value.get_value())
+ {
+ if let Some(range_dc) = Interval::try_new(min.clone(), max.clone())
+ .ok()
+ .and_then(|e| e.cardinality())
{
- Precision::Exact(numeric_range.min(ceiling))
- } else {
- Precision::Inexact(numeric_range.min(ceiling))
- },
- )
+ let range_dc = range_dc as usize;
+ // Note that the `unwrap` calls in the below statement are safe.
+ return if matches!(result, Precision::Absent)
+ || &range_dc < result.get_value().unwrap()
+ {
+ if stats.min_value.is_exact().unwrap()
+ && stats.max_value.is_exact().unwrap()
+ {
+ Precision::Exact(range_dc)
+ } else {
+ Precision::Inexact(range_dc)
+ }
+ } else {
+ result
+ };
+ }
+ }
+
+ result
}
- _ => None,
}
}
@@ -1251,7 +1264,8 @@ pub fn prepare_sorted_exprs(
vec![left_temp_sorted_filter_expr, right_temp_sorted_filter_expr];
// Build the expression interval graph
- let mut graph = ExprIntervalGraph::try_new(filter.expression().clone())?;
+ let mut graph =
+ ExprIntervalGraph::try_new(filter.expression().clone(), filter.schema())?;
// Update sorted expressions with node indices
update_sorted_exprs_with_node_indices(&mut graph, &mut sorted_exprs);
@@ -1697,7 +1711,7 @@ mod tests {
column_statistics: right_col_stats,
},
),
- None
+ Some(Precision::Inexact(100))
);
Ok(())
}