You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by "Copilot (via GitHub)" <gi...@apache.org> on 2026/05/06 04:47:28 UTC
Re: [PR] Add levenshtein and hamming_distance functions [doris]
Copilot commented on code in PR #60412:
URL: https://github.com/apache/doris/pull/60412#discussion_r3193059453
##########
regression-test/suites/query_p0/sql_functions/string_functions/test_string_all.groovy:
##########
@@ -753,6 +753,195 @@ suite("string_functions_all") {
testFoldConst("SELECT soundex('R@b-e123rt'), soundex('Robert');")
// SOUNDEX tests with non-ASCII characters - Skipped (not supported)
+ // LEVENSHTEIN tests
+ qt_levenshtein_331 "SELECT levenshtein('', ''), levenshtein('kitten', 'sitting'), levenshtein('flaw', 'lawn'), levenshtein('你好', '你们'), levenshtein('数据库', '数据');"
+ testFoldConst("SELECT levenshtein('', ''), levenshtein('kitten', 'sitting'), levenshtein('flaw', 'lawn'), levenshtein('你好', '你们'), levenshtein('数据库', '数据');")
+ qt_levenshtein_332 "SELECT levenshtein('abc', 'abc'), levenshtein('abc', ''), levenshtein('', 'abc'), levenshtein(NULL, 'abc'), levenshtein('abc', NULL);"
+ testFoldConst("SELECT levenshtein('abc', 'abc'), levenshtein('abc', ''), levenshtein('', 'abc'), levenshtein(NULL, 'abc'), levenshtein('abc', NULL);")
+ qt_levenshtein_333 "SELECT levenshtein('abcd', 'abdc'), levenshtein('你好呀', '你好'), levenshtein('a你b', 'a们b');"
+ testFoldConst("SELECT levenshtein('abcd', 'abdc'), levenshtein('你好呀', '你好'), levenshtein('a你b', 'a们b');")
+ qt_levenshtein_334 "SELECT levenshtein(NULL, NULL), levenshtein('', '你好'), levenshtein('你好世界', '你好世间');"
+ testFoldConst("SELECT levenshtein(NULL, NULL), levenshtein('', '你好'), levenshtein('你好世界', '你好世间');")
+ sql """DROP TABLE IF EXISTS string_distance_lv_test"""
+ sql """
+ CREATE TABLE IF NOT EXISTS string_distance_lv_test (
+ id int,
+ s1 VARCHAR,
+ s2 VARCHAR
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num"="1")
+ """
+ sql """
+ insert into string_distance_lv_test values
+ (1, 'kitten', 'sitting'),
+ (2, 'abc', 'abc'),
+ (3, '数据库', '数据'),
+ (4, null, 'abc'),
+ (5, '你好呀', '你好'),
+ (6, 'abcd', 'abdc'),
+ (7, '', '数据库'),
+ (8, '你好', ''),
+ (9, '数据', '数据库')
+ """
+ qt_levenshtein_tbl "SELECT id, levenshtein(s1, s2) FROM string_distance_lv_test ORDER BY id"
+
+ sql """DROP TABLE IF EXISTS string_distance_nn_test"""
+ sql """
+ CREATE TABLE IF NOT EXISTS string_distance_nn_test (
+ id int,
+ s1 VARCHAR(20) NOT NULL,
+ s2 VARCHAR(20) NOT NULL
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num"="1")
+ """
+ sql """
+ insert into string_distance_nn_test values
+ (1, 'abc', 'abd'),
+ (2, 'abcd', 'wxyz'),
+ (3, '你好', '你们'),
+ (4, '数据库', '数库据')
+ """
+ qt_levenshtein_nn_vector_vector "SELECT id, levenshtein(s1, s2) FROM string_distance_nn_test ORDER BY id"
+ qt_levenshtein_nn_vector_scalar_ascii "SELECT id, levenshtein(s1, 'abc') FROM string_distance_nn_test WHERE id = 1 ORDER BY id"
+ qt_levenshtein_nn_scalar_vector_ascii "SELECT id, levenshtein('abc', s1) FROM string_distance_nn_test WHERE id = 1 ORDER BY id"
+ qt_levenshtein_nn_vector_scalar_utf8 "SELECT id, levenshtein(s1, '你们') FROM string_distance_nn_test WHERE id = 3 ORDER BY id"
+ qt_levenshtein_nn_scalar_vector_utf8 "SELECT id, levenshtein('你们', s1) FROM string_distance_nn_test WHERE id = 3 ORDER BY id"
+ qt_levenshtein_vector_scalar_nullable "SELECT id, levenshtein(s1, 'abc') FROM string_distance_lv_test WHERE id IN (2, 4) ORDER BY id"
+ qt_levenshtein_scalar_vector_nullable "SELECT id, levenshtein('abc', s1) FROM string_distance_lv_test WHERE id IN (2, 4) ORDER BY id"
+ qt_levenshtein_vector_scalar_utf8 "SELECT id, levenshtein(s1, '你好') FROM string_distance_lv_test WHERE id IN (5, 7) ORDER BY id"
+ qt_levenshtein_scalar_vector_utf8 "SELECT id, levenshtein('你好', s1) FROM string_distance_lv_test WHERE id IN (5, 7) ORDER BY id"
+ qt_levenshtein_vector_scalar_empty_utf8 "SELECT id, levenshtein(s1, '') FROM string_distance_lv_test WHERE id IN (5, 8) ORDER BY id"
+ qt_levenshtein_scalar_vector_empty_utf8 "SELECT id, levenshtein('', s1) FROM string_distance_lv_test WHERE id IN (5, 8) ORDER BY id"
+
+ sql """DROP TABLE IF EXISTS string_distance_lv_nn_test"""
+ sql """
+ CREATE TABLE IF NOT EXISTS string_distance_lv_nn_test (
+ id int,
+ s1 VARCHAR(20) NOT NULL,
+ s2 VARCHAR(20) NOT NULL
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num"="1")
+ """
+ sql """
+ insert into string_distance_lv_nn_test values
+ (1, 'abc', 'abcdef'),
+ (2, 'abcdef', 'abc'),
+ (3, '', 'abc'),
+ (4, 'abc', ''),
+ (5, '数据', '数据库'),
+ (6, '数据库', '数据'),
+ (7, '', '数据库'),
+ (8, '数据库', '')
+ """
+ qt_levenshtein_lv_nn_vector_vector "SELECT id, levenshtein(s1, s2) FROM string_distance_lv_nn_test ORDER BY id"
+ qt_levenshtein_lv_nn_vector_scalar_empty "SELECT id, levenshtein(s1, '') FROM string_distance_lv_nn_test ORDER BY id"
+ qt_levenshtein_lv_nn_scalar_vector_empty "SELECT id, levenshtein('', s1) FROM string_distance_lv_nn_test ORDER BY id"
+ qt_levenshtein_lv_nn_vector_scalar_ascii "SELECT id, levenshtein(s1, 'abcdef') FROM string_distance_lv_nn_test WHERE id IN (1, 2, 3, 4) ORDER BY id"
+ qt_levenshtein_lv_nn_scalar_vector_ascii "SELECT id, levenshtein('abcdef', s1) FROM string_distance_lv_nn_test WHERE id IN (1, 2, 3, 4) ORDER BY id"
+ qt_levenshtein_lv_nn_vector_scalar_utf8 "SELECT id, levenshtein(s1, '数据库') FROM string_distance_lv_nn_test WHERE id IN (5, 6, 7, 8) ORDER BY id"
+ qt_levenshtein_lv_nn_scalar_vector_utf8 "SELECT id, levenshtein('数据库', s1) FROM string_distance_lv_nn_test WHERE id IN (5, 6, 7, 8) ORDER BY id"
+
+ // HAMMING_DISTANCE tests
+ qt_hamming_distance_333 "SELECT hamming_distance('', ''), hamming_distance('abc', 'abc'), hamming_distance('abc', 'abd'), hamming_distance('你好', '你们');"
+ testFoldConst("SELECT hamming_distance('', ''), hamming_distance('abc', 'abc'), hamming_distance('abc', 'abd'), hamming_distance('你好', '你们');")
+ qt_hamming_distance_334 "SELECT hamming_distance('abc', 'abc'), hamming_distance(NULL, 'abc'), hamming_distance('abc', NULL);"
+ testFoldConst("SELECT hamming_distance('abc', 'abc'), hamming_distance(NULL, 'abc'), hamming_distance('abc', NULL);")
+ qt_hamming_distance_335 "SELECT hamming_distance('abcd', 'wxyz'), hamming_distance('你好吗', '你们吗'), hamming_distance('数据库', '数库据');"
+ testFoldConst("SELECT hamming_distance('abcd', 'wxyz'), hamming_distance('你好吗', '你们吗'), hamming_distance('数据库', '数库据');")
+ qt_hamming_distance_336 "SELECT hamming_distance(NULL, NULL), hamming_distance(NULL, 'addd'), hamming_distance('addd', NULL);"
+ testFoldConst("SELECT hamming_distance(NULL, NULL), hamming_distance(NULL, 'addd'), hamming_distance('addd', NULL);")
+ sql """ set debug_skip_fold_constant = false; """
+ test {
+ sql "SELECT hamming_distance('abc', 'ab');"
+ exception "hamming_distance requires strings of the same length"
+ }
+ test {
+ sql "SELECT hamming_distance('你好', '你');"
+ exception "hamming_distance requires strings of the same length"
+ }
+ sql """ set debug_skip_fold_constant = true; """
+ test {
+ sql "SELECT hamming_distance('abc', 'ab');"
+ exception "hamming_distance requires strings of the same length"
+ }
+ test {
+ sql "SELECT hamming_distance('你好', '你');"
Review Comment:
These tests change `debug_skip_fold_constant` globally but don’t restore the original session value afterwards, which can affect later `qt_*` queries in this suite depending on the default. Consider saving `@@debug_skip_fold_constant` before toggling and restoring it after, or use per-query `/*+SET_VAR(debug_skip_fold_constant=...)*/` hints to avoid side effects.
##########
be/src/exprs/function/function_levenshtein.cpp:
##########
@@ -0,0 +1,263 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <algorithm>
+#include <vector>
+
+#include "common/status.h"
+#include "core/data_type/data_type_number.h"
+#include "core/string_ref.h"
+#include "exprs/function/function_totype.h"
+#include "exprs/function/simple_function_factory.h"
+#include "util/simd/vstring_function.h"
+
+namespace doris {
+
+struct NameLevenshtein {
+ static constexpr auto name = "levenshtein";
+};
+
+template <typename LeftDataType, typename RightDataType>
+struct LevenshteinImpl {
+ using ResultDataType = DataTypeInt32;
+ using ResultPaddedPODArray = PaddedPODArray<Int32>;
+
+ static Status vector_vector(const ColumnString::Chars& ldata,
+ const ColumnString::Offsets& loffsets,
+ const ColumnString::Chars& rdata,
+ const ColumnString::Offsets& roffsets, ResultPaddedPODArray& res) {
+ DCHECK_EQ(loffsets.size(), roffsets.size());
+
+ const size_t size = loffsets.size();
+ res.resize(size);
+ std::vector<size_t> left_offsets;
+ std::vector<size_t> right_offsets;
+ for (size_t i = 0; i < size; ++i) {
+ res[i] = levenshtein_distance(string_ref_at(ldata, loffsets, i),
+ string_ref_at(rdata, roffsets, i), left_offsets,
+ right_offsets);
+ }
+ return Status::OK();
+ }
+
+ static Status vector_scalar(const ColumnString::Chars& ldata,
+ const ColumnString::Offsets& loffsets, const StringRef& rdata,
+ ResultPaddedPODArray& res) {
+ const size_t size = loffsets.size();
+ res.resize(size);
+ const auto right = rdata.trim_tail_padding_zero();
+ const bool right_ascii = simd::VStringFunctions::is_ascii(right);
+ std::vector<size_t> right_offsets;
+ simd::VStringFunctions::get_utf8_char_offsets(right, right_offsets);
+ std::vector<size_t> left_offsets;
+ for (size_t i = 0; i < size; ++i) {
+ res[i] = levenshtein_distance_with_right_offsets(string_ref_at(ldata, loffsets, i),
+ left_offsets, right, right_offsets,
+ right_ascii);
+ }
+ return Status::OK();
+ }
+
+ static Status scalar_vector(const StringRef& ldata, const ColumnString::Chars& rdata,
+ const ColumnString::Offsets& roffsets, ResultPaddedPODArray& res) {
+ const size_t size = roffsets.size();
+ res.resize(size);
+ const auto left = ldata.trim_tail_padding_zero();
+ const bool left_ascii = simd::VStringFunctions::is_ascii(left);
+ std::vector<size_t> left_offsets;
+ simd::VStringFunctions::get_utf8_char_offsets(left, left_offsets);
+ std::vector<size_t> right_offsets;
+ for (size_t i = 0; i < size; ++i) {
+ res[i] = levenshtein_distance_with_left_offsets(left, left_offsets, left_ascii,
+ string_ref_at(rdata, roffsets, i),
+ right_offsets);
+ }
+ return Status::OK();
+ }
+
+private:
+ static StringRef string_ref_at(const ColumnString::Chars& data,
+ const ColumnString::Offsets& offsets, size_t i) {
+ DCHECK_LT(i, offsets.size());
+ const auto idx = static_cast<ssize_t>(i);
+ return StringRef(data.data() + offsets[idx - 1], offsets[idx] - offsets[idx - 1])
+ .trim_tail_padding_zero();
Review Comment:
`string_ref_at()` computes the start offset as `offsets[idx - 1]`, which reads `offsets[-1]` when `i == 0`. This is out-of-bounds and will produce incorrect StringRef (or crash) for the first row in every batch. Compute `start = (i == 0 ? 0 : offsets[i - 1])` and `len = offsets[i] - start` (no signed index needed).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org