You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/04/29 05:03:54 UTC
(doris) branch master updated: [fix](be) Fix ANN query vector extraction to handle all constant expression types (#62637)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 430781b1296 [fix](be) Fix ANN query vector extraction to handle all constant expression types (#62637)
430781b1296 is described below
commit 430781b1296dec4add1fef5d9271d84164dd7022
Author: zhiqiang <se...@163.com>
AuthorDate: Wed Apr 29 13:03:46 2026 +0800
[fix](be) Fix ANN query vector extraction to handle all constant expression types (#62637)
## Proposed changes
Fix `extract_query_vector` in `ann_topn_runtime.cpp` to handle all
constant expression types instead of only `VArrayLiteral` and
`VCastExpr`.
### What problem does this PR solve?
Issue Number: close #xxx
Problem Summary:
The `extract_query_vector` function previously checked whether the query
vector argument was specifically a `VArrayLiteral` or `VCastExpr` via
`dynamic_pointer_cast`. When FE produced other constant expression forms
(e.g., after expression rewrites), ANN index queries would fail with
`InvalidArgument: Constant must be ArrayLiteral or CAST to array`.
The fix removes the rigid type check and instead:
1. Calls `get_const_col()` to materialize the constant expression
2. Unwraps `ColumnConst` via `check_and_get_column<ColumnConst>` to get
the underlying data column
3. Validates the resulting column shape (Nullable → Array → Float32)
This approach works for any constant expression that materializes to an
array column, regardless of the expression node type.
### Release note
Fix ANN vector index query failure when the query vector expression is
not a direct array literal or CAST expression.
### Check List (For Author)
- Test: Regression test
- Behavior changed: No
- Does this need documentation: No
---
be/src/storage/index/ann/ann_topn_runtime.cpp | 25 +--
be/src/storage/index/ann/ann_topn_runtime.h | 2 -
.../index/ann/extract_query_vector_test.cpp | 248 +++++++++++++++++++++
.../data/ann_index_p0/ann_const_expr_vector.out | 21 ++
.../ann_index_p0/ann_const_expr_vector.groovy | 117 ++++++++++
.../ann_index_p0/cast_string_as_array.groovy | 2 +-
6 files changed, 394 insertions(+), 21 deletions(-)
diff --git a/be/src/storage/index/ann/ann_topn_runtime.cpp b/be/src/storage/index/ann/ann_topn_runtime.cpp
index a4de1f3f7de..63586121fa8 100644
--- a/be/src/storage/index/ann/ann_topn_runtime.cpp
+++ b/be/src/storage/index/ann/ann_topn_runtime.cpp
@@ -27,11 +27,10 @@
#include "common/status.h"
#include "core/column/column.h"
#include "core/column/column_array.h"
+#include "core/column/column_const.h"
#include "core/column/column_nullable.h"
#include "core/data_type/primitive_type.h"
#include "exprs/function/array/function_array_distance.h"
-#include "exprs/varray_literal.h"
-#include "exprs/vcast_expr.h"
#include "exprs/vexpr_context.h"
#include "exprs/vexpr_fwd.h"
#include "exprs/virtual_slot_ref.h"
@@ -49,19 +48,6 @@ Result<IColumn::Ptr> extract_query_vector(std::shared_ptr<VExpr> arg_expr) {
arg_expr->debug_string()));
}
- // Accept either ArrayLiteral([..]) or CAST('..' AS Nullable(Array(Nullable(Float32))))
- // First, check the expr node type for clarity.
-
- bool is_array_literal = std::dynamic_pointer_cast<VArrayLiteral>(arg_expr) != nullptr;
- bool is_cast_expr = std::dynamic_pointer_cast<VCastExpr>(arg_expr) != nullptr;
- if (!is_array_literal && !is_cast_expr) {
- return ResultError(
- Status::InvalidArgument("Constant must be ArrayLiteral or CAST to array, got\n{}",
- arg_expr->debug_string()));
- }
-
- // We'll validate shape by inspecting the materialized constant column below.
-
std::shared_ptr<ColumnPtrWrapper> column_wrapper;
auto st = arg_expr->get_const_col(nullptr, &column_wrapper);
if (!st.ok()) {
@@ -69,8 +55,11 @@ Result<IColumn::Ptr> extract_query_vector(std::shared_ptr<VExpr> arg_expr) {
st.to_string()));
}
- // Execute the constant array literal and extract its float elements into _query_array
- IColumn::Ptr col_ptr = column_wrapper->column_ptr->convert_to_full_column_if_const();
+ // Unwrap ColumnConst without copy to get the underlying single-row column
+ IColumn::Ptr col_ptr = column_wrapper->column_ptr;
+ if (const auto* const_col = check_and_get_column<ColumnConst>(*col_ptr)) {
+ col_ptr = const_col->get_data_column_ptr();
+ }
// The expected runtime column layout for the literal is:
// Nullable(ColumnArray(Nullable(ColumnFloat32))) with exactly 1 row (one array literal)
@@ -126,7 +115,7 @@ Status AnnTopNRuntime::prepare(RuntimeState* state, const RowDescriptor& row_des
|----------------
| |
| |
- SlotRef CAST(String as Nullable<ArrayFloat>) OR ArrayLiteral
+ SlotRef Constant Array Expression
*/
std::shared_ptr<VirtualSlotRef> vir_slot_ref =
std::dynamic_pointer_cast<VirtualSlotRef>(_order_by_expr_ctx->root());
diff --git a/be/src/storage/index/ann/ann_topn_runtime.h b/be/src/storage/index/ann/ann_topn_runtime.h
index 5beca98f599..95b13b9462f 100644
--- a/be/src/storage/index/ann/ann_topn_runtime.h
+++ b/be/src/storage/index/ann/ann_topn_runtime.h
@@ -37,8 +37,6 @@
#include "core/column/column.h"
#include "core/data_type/primitive_type.h"
-#include "exprs/varray_literal.h"
-#include "exprs/vcast_expr.h"
#include "exprs/vectorized_fn_call.h"
#include "exprs/vexpr.h"
#include "exprs/vexpr_context.h"
diff --git a/be/test/storage/index/ann/extract_query_vector_test.cpp b/be/test/storage/index/ann/extract_query_vector_test.cpp
new file mode 100644
index 00000000000..13e059825b3
--- /dev/null
+++ b/be/test/storage/index/ann/extract_query_vector_test.cpp
@@ -0,0 +1,248 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "core/column/column_array.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "exprs/vexpr.h"
+#include "storage/index/ann/ann_topn_runtime.h"
+
+namespace doris::segment_v2 {
+
+// A minimal mock VExpr that returns a pre-set constant column.
+class MockConstVExpr : public VExpr {
+public:
+ static TExprNode make_tnode() {
+ TExprNode node;
+ node.node_type = TExprNodeType::FLOAT_LITERAL;
+ node.type = TTypeDesc();
+ TTypeNode type_node;
+ type_node.type = TTypeNodeType::SCALAR;
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::FLOAT);
+ type_node.__set_scalar_type(scalar_type);
+ node.type.types.push_back(type_node);
+ return node;
+ }
+
+ MockConstVExpr() : VExpr(make_tnode()) {}
+
+ Status get_const_col(VExprContext* /*context*/,
+ std::shared_ptr<ColumnPtrWrapper>* output) override {
+ *output = std::make_shared<ColumnPtrWrapper>(_col);
+ return Status::OK();
+ }
+
+ bool is_constant() const override { return _is_constant; }
+
+ Status execute_column(VExprContext* context, const Block* block, const Selector* selector,
+ size_t count, ColumnPtr& result_column) const {
+ return Status::OK();
+ }
+
+ Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector,
+ size_t count, ColumnPtr& result_column) const override {
+ return Status::OK();
+ }
+
+ Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override {
+ return Status::OK();
+ }
+
+ Status open(RuntimeState* state, VExprContext* context,
+ FunctionContext::FunctionStateScope scope) override {
+ return Status::OK();
+ }
+
+ void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override {}
+
+ const std::string& expr_name() const override {
+ static std::string name = "MockConstVExpr";
+ return name;
+ }
+
+ std::string debug_string() const override { return "MockConstVExpr"; }
+
+ void set_column(IColumn::Ptr col) { _col = std::move(col); }
+ void set_is_constant(bool v) { _is_constant = v; }
+
+private:
+ IColumn::Ptr _col;
+ bool _is_constant = true;
+};
+
+// Helper: build Nullable(ColumnArray(Nullable(ColumnFloat32))) with 1 row of given floats
+static IColumn::Ptr make_nullable_array_column(const std::vector<float>& values) {
+ auto float_col = ColumnFloat32::create();
+ for (float v : values) {
+ float_col->insert_value(v);
+ }
+ auto null_map_inner = ColumnUInt8::create(values.size(), 0);
+ auto nullable_inner = ColumnNullable::create(std::move(float_col), std::move(null_map_inner));
+
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->insert_value(static_cast<IColumn::Offset>(values.size()));
+ auto array_col = ColumnArray::create(std::move(nullable_inner), std::move(offsets));
+
+ auto null_map_outer = ColumnUInt8::create(1, 0);
+ return ColumnNullable::create(std::move(array_col), std::move(null_map_outer));
+}
+
+class ExtractQueryVectorTest : public ::testing::Test {};
+
+// ColumnConst wrapping a Nullable(Array(Nullable(Float32))) — the array_repeat case
+TEST_F(ExtractQueryVectorTest, ColumnConstWrappedArray) {
+ auto inner = make_nullable_array_column({1.0f, 2.0f, 3.0f});
+ auto const_col = ColumnConst::create(std::move(inner), 1);
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(const_col));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_TRUE(result.has_value()) << result.error().to_string();
+ EXPECT_EQ(result.value()->size(), 3u);
+}
+
+// Direct Nullable(Array(Nullable(Float32))) without ColumnConst wrapper
+TEST_F(ExtractQueryVectorTest, DirectNullableArray) {
+ auto col = make_nullable_array_column({4.0f, 5.0f});
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(col));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_TRUE(result.has_value()) << result.error().to_string();
+ EXPECT_EQ(result.value()->size(), 2u);
+}
+
+// Non-nullable ColumnArray(Nullable(Float32)) directly
+TEST_F(ExtractQueryVectorTest, NonNullableArray) {
+ auto float_col = ColumnFloat32::create();
+ float_col->insert_value(1.0f);
+ float_col->insert_value(2.0f);
+ auto null_map = ColumnUInt8::create(2, 0);
+ auto nullable_inner = ColumnNullable::create(std::move(float_col), std::move(null_map));
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->insert_value(2);
+ auto array_col = ColumnArray::create(std::move(nullable_inner), std::move(offsets));
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(array_col));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_TRUE(result.has_value()) << result.error().to_string();
+ EXPECT_EQ(result.value()->size(), 2u);
+}
+
+// ColumnConst wrapping non-nullable array (another possible shape)
+TEST_F(ExtractQueryVectorTest, ColumnConstNonNullableArray) {
+ auto float_col = ColumnFloat32::create();
+ float_col->insert_value(7.0f);
+ auto null_map = ColumnUInt8::create(1, 0);
+ auto nullable_inner = ColumnNullable::create(std::move(float_col), std::move(null_map));
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->insert_value(1);
+ auto array_col = ColumnArray::create(std::move(nullable_inner), std::move(offsets));
+ auto const_col = ColumnConst::create(std::move(array_col), 1);
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(const_col));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_TRUE(result.has_value()) << result.error().to_string();
+ EXPECT_EQ(result.value()->size(), 1u);
+}
+
+// Verify extracted float values match input
+TEST_F(ExtractQueryVectorTest, ValuesMatchInput) {
+ std::vector<float> input = {1.5f, 2.5f, 3.5f, 4.5f};
+ auto col = make_nullable_array_column(input);
+ auto const_col = ColumnConst::create(std::move(col), 1);
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(const_col));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_TRUE(result.has_value());
+ auto* float_col = assert_cast<const ColumnFloat32*>(result.value().get());
+ ASSERT_EQ(float_col->size(), 4u);
+ for (size_t i = 0; i < input.size(); ++i) {
+ EXPECT_FLOAT_EQ(float_col->get_data()[i], input[i]);
+ }
+}
+
+// Error: non-constant expression
+TEST_F(ExtractQueryVectorTest, NonConstantExprFails) {
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_is_constant(false);
+
+ auto result = extract_query_vector(mock);
+ ASSERT_FALSE(result.has_value());
+ EXPECT_TRUE(result.error().to_string().find("must be constant") != std::string::npos);
+}
+
+// Error: NULL array
+TEST_F(ExtractQueryVectorTest, NullArrayFails) {
+ auto float_col = ColumnFloat32::create();
+ auto null_map_inner = ColumnUInt8::create(0, 0);
+ auto nullable_inner = ColumnNullable::create(std::move(float_col), std::move(null_map_inner));
+ auto offsets = ColumnArray::ColumnOffsets::create();
+ offsets->insert_value(0);
+ auto array_col = ColumnArray::create(std::move(nullable_inner), std::move(offsets));
+ // Outer nullable with null flag set to 1
+ auto null_map_outer = ColumnUInt8::create(1, 1);
+ auto nullable_outer = ColumnNullable::create(std::move(array_col), std::move(null_map_outer));
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(nullable_outer));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_FALSE(result.has_value());
+ EXPECT_TRUE(result.error().to_string().find("cannot be NULL") != std::string::npos);
+}
+
+// Error: empty array
+TEST_F(ExtractQueryVectorTest, EmptyArrayFails) {
+ auto col = make_nullable_array_column({});
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(col));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_FALSE(result.has_value());
+ EXPECT_TRUE(result.error().to_string().find("cannot be empty") != std::string::npos);
+}
+
+// Error: not an array column at all
+TEST_F(ExtractQueryVectorTest, NonArrayColumnFails) {
+ auto float_col = ColumnFloat32::create();
+ float_col->insert_value(1.0f);
+
+ auto mock = std::make_shared<MockConstVExpr>();
+ mock->set_column(std::move(float_col));
+
+ auto result = extract_query_vector(mock);
+ ASSERT_FALSE(result.has_value());
+ EXPECT_TRUE(result.error().to_string().find("Array literal") != std::string::npos);
+}
+
+} // namespace doris::segment_v2
diff --git a/regression-test/data/ann_index_p0/ann_const_expr_vector.out b/regression-test/data/ann_index_p0/ann_const_expr_vector.out
new file mode 100644
index 00000000000..3aec3b406d2
--- /dev/null
+++ b/regression-test/data/ann_index_p0/ann_const_expr_vector.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !array_repeat --
+1 0.0
+2 1.98
+3 9.98
+
+-- !array_with_constant --
+1 0.0
+2 1.98
+3 9.98
+
+-- !direct_literal --
+1 0.0
+2 1.98
+3 9.98
+
+-- !cast_string --
+1 0.0
+2 1.98
+3 9.98
+
diff --git a/regression-test/suites/ann_index_p0/ann_const_expr_vector.groovy b/regression-test/suites/ann_index_p0/ann_const_expr_vector.groovy
new file mode 100644
index 00000000000..380ac4cc40e
--- /dev/null
+++ b/regression-test/suites/ann_index_p0/ann_const_expr_vector.groovy
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test that ANN queries work with various constant expression forms for the
+// query vector, not just direct array literals or CAST expressions.
+// Covers: array_repeat, array_with_constant, nested function calls, etc.
+suite("ann_const_expr_vector") {
+ sql "unset variable all;"
+ sql "set enable_common_expr_pushdown=true;"
+
+ def tableName = "ann_const_expr_tbl"
+
+ sql "drop table if exists ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ id INT NOT NULL,
+ embedding ARRAY<FLOAT> NOT NULL,
+ INDEX idx_emb (`embedding`) USING ANN PROPERTIES(
+ "index_type"="hnsw",
+ "metric_type"="l2_distance",
+ "dim"="4"
+ )
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS AUTO
+ PROPERTIES ("replication_num" = "1");
+ """
+
+ sql """
+ INSERT INTO ${tableName} VALUES
+ (1, [0.01, 0.01, 0.01, 0.01]),
+ (2, [1.0, 1.0, 1.0, 1.0]),
+ (3, [5.0, 5.0, 5.0, 5.0]),
+ (4, [10.0, 10.0, 10.0, 10.0]);
+ """
+
+
+ // =========================================================================
+ // Test 1: array_repeat - generates a constant array via function call
+ // This is the key test case: array_repeat produces a ColumnConst wrapping
+ // an array, which the old code rejected.
+ // =========================================================================
+ qt_array_repeat """
+ SELECT id, l2_distance_approximate(embedding, array_repeat(CAST(0.01 AS FLOAT), 4)) AS score
+ FROM ${tableName}
+ ORDER BY score ASC
+ LIMIT 3;
+ """
+
+ // =========================================================================
+ // Test 2: array_with_constant - another way to produce constant arrays
+ // =========================================================================
+ qt_array_with_constant """
+ SELECT id, l2_distance_approximate(embedding, array_with_constant(4, CAST(0.01 AS FLOAT))) AS score
+ FROM ${tableName}
+ ORDER BY score ASC
+ LIMIT 3;
+ """
+
+ // =========================================================================
+ // Test 3: Direct array literal (baseline, should always work)
+ // =========================================================================
+ qt_direct_literal """
+ SELECT id, l2_distance_approximate(embedding, [0.01, 0.01, 0.01, 0.01]) AS score
+ FROM ${tableName}
+ ORDER BY score ASC
+ LIMIT 3;
+ """
+
+ // =========================================================================
+ // Test 4: CAST string to array (existing functionality)
+ // =========================================================================
+ qt_cast_string """
+ SELECT id, l2_distance_approximate(embedding, cast('[0.01,0.01,0.01,0.01]' as array<float>)) AS score
+ FROM ${tableName}
+ ORDER BY score ASC
+ LIMIT 3;
+ """
+
+ // =========================================================================
+ // Test 5: Error case - array_repeat with wrong dimension
+ // =========================================================================
+ test {
+ sql """
+ SELECT id FROM ${tableName}
+ ORDER BY l2_distance_approximate(embedding, array_repeat(CAST(0.01 AS FLOAT), 3))
+ LIMIT 1;
+ """
+ exception "[INVALID_ARGUMENT]"
+ }
+
+ // =========================================================================
+ // Test 6: Error case - empty array via array_repeat
+ // =========================================================================
+ test {
+ sql """
+ SELECT id FROM ${tableName}
+ ORDER BY l2_distance_approximate(embedding, array_repeat(CAST(0.01 AS FLOAT), 0))
+ LIMIT 1;
+ """
+ exception "Ann topn query vector cannot be empty"
+ }
+}
diff --git a/regression-test/suites/ann_index_p0/cast_string_as_array.groovy b/regression-test/suites/ann_index_p0/cast_string_as_array.groovy
index 9d0ea331ef2..01a1c4e5f07 100644
--- a/regression-test/suites/ann_index_p0/cast_string_as_array.groovy
+++ b/regression-test/suites/ann_index_p0/cast_string_as_array.groovy
@@ -67,7 +67,7 @@ suite("cast_string_as_array") {
// runtime of ANN topn. So here we will get null directly...
test {
sql "select id from ann_cast_rhs_l2 order by l2_distance_approximate(embedding, cast(NULL as array<float>)) limit 1;"
- exception "Constant must be ArrayLiteral or CAST to array"
+ exception "Ann query vector cannot be NULL"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org