You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ai...@apache.org on 2026/02/25 10:02:18 UTC
(doris) branch master updated: [fix](search) Fix implicit conjunction incorrectly modifying preceding term in lucene mode (#60814)
This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1cde086c55d [fix](search) Fix implicit conjunction incorrectly modifying preceding term in lucene mode (#60814)
1cde086c55d is described below
commit 1cde086c55df903c8f7a083074dad67cf353dd37
Author: Jack <ji...@selectdb.com>
AuthorDate: Wed Feb 25 18:02:11 2026 +0800
[fix](search) Fix implicit conjunction incorrectly modifying preceding term in lucene mode (#60814)
### What problem does this PR solve?
Issue Number: close #DORIS-24545
Problem Summary:
In `search()` function's lucene mode, queries with mixed explicit and
implicit operators produce different results from Elasticsearch. For
example:
- Query: `"Sumer" OR Ptolemaic\ dynasty Limonene` with
`default_operator=AND`
- ES result: 1 row
- Doris result: 0 rows (before fix)
**Root cause:** In Lucene's `QueryParserBase.addClause()`, only explicit
`CONJ_AND`/`CONJ_OR` modify the preceding term's occur. Implicit
conjunction (`CONJ_NONE`, i.e., space-separated terms without an
explicit operator) only affects the **current** term via
`default_operator`, without modifying the preceding term.
The FE `SearchDslParser.hasExplicitAndBefore()` incorrectly returned
`true` (based on `default_operator`) when no explicit AND token was
found. This caused implicit conjunction to be treated identically to
explicit AND, making it modify the preceding term's occur — diverging
from Lucene/ES semantics.
**Example of the bug:**
For `a OR b c` with `default_operator=AND`:
- Before fix: `SHOULD(a) MUST(b) MUST(c)` — wrong, implicit space before
`c` incorrectly upgraded `b` from SHOULD to MUST
- After fix: `SHOULD(a) SHOULD(b) MUST(c)` — correct, matches ES
behavior. Only `c` gets MUST (from default_operator), `b` retains SHOULD
(from the preceding OR)
**Fix:** `hasExplicitAndBefore()` now returns `false` when no explicit
AND token is found, regardless of `default_operator`. Only explicit AND
tokens trigger the "introduced by AND" logic that modifies preceding
terms.
---
.../functions/scalar/SearchDslParser.java | 30 +++---
.../functions/scalar/SearchDslParserTest.java | 108 +++++++++++++++++++++
2 files changed, 124 insertions(+), 14 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java
index bf0bc8f6168..2dc5c52fb1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java
@@ -2143,16 +2143,13 @@ public class SearchDslParser {
QsOccur defaultOccur, boolean introducedByOr) {
List<SearchParser.NotClauseContext> notClauses = ctx.notClause();
- // Determine how to handle implicit operators
- String defaultOperator = options.getDefaultOperator();
- boolean useAndForImplicit = "AND".equalsIgnoreCase(defaultOperator);
-
for (int i = 0; i < notClauses.size(); i++) {
boolean introducedByAnd;
if (i > 0) {
- // Check if there's an explicit AND before this notClause
- // by walking ctx.children and finding the token immediately before this notClause
- introducedByAnd = hasExplicitAndBefore(ctx, notClauses.get(i), useAndForImplicit);
+ // Check if there's an explicit AND token before this notClause.
+ // Implicit conjunction (no AND token) returns false - only explicit AND
+ // should trigger the "introduced by AND" logic that modifies preceding terms.
+ introducedByAnd = hasExplicitAndBefore(ctx, notClauses.get(i));
} else {
introducedByAnd = false;
}
@@ -2166,13 +2163,18 @@ public class SearchDslParser {
/**
* Check if there's an explicit AND token before the target notClause.
* Walks ctx.children to find the position of target and checks the preceding token.
+ *
+ * IMPORTANT: Returns false for implicit conjunction (no explicit AND token).
+ * In Lucene's QueryParserBase.addClause(), only explicit CONJ_AND modifies the
+ * preceding term. CONJ_NONE (implicit conjunction) only affects the current term's
+ * occur via the default_operator, without modifying the preceding term.
+ *
* @param ctx The AndClauseContext containing the children
* @param target The target NotClauseContext to check
- * @param implicitDefault Value to return if no explicit AND (use default_operator)
- * @return true if explicit AND before target, implicitDefault if no explicit AND
+ * @return true only if there's an explicit AND token before target
*/
private boolean hasExplicitAndBefore(SearchParser.AndClauseContext ctx,
- SearchParser.NotClauseContext target, boolean implicitDefault) {
+ SearchParser.NotClauseContext target) {
for (int j = 0; j < ctx.getChildCount(); j++) {
if (ctx.getChild(j) == target) {
// Found the target - check if the preceding sibling is an AND token
@@ -2181,12 +2183,12 @@ public class SearchDslParser {
(org.antlr.v4.runtime.tree.TerminalNode) ctx.getChild(j - 1);
return terminal.getSymbol().getType() == SearchParser.AND;
}
- // No explicit AND before this term - use default
- return implicitDefault;
+ // No explicit AND before this term
+ return false;
}
}
- // Target not found (should not happen) - use default
- return implicitDefault;
+ // Target not found (should not happen)
+ return false;
}
private void collectTermsFromNotClause(SearchParser.NotClauseContext ctx, List<TermWithOccur> terms,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java
index 01bbcf8d925..8e562401e14 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java
@@ -839,6 +839,114 @@ public class SearchDslParserTest {
Assertions.assertEquals(QsClauseType.AND, plan.getRoot().getType());
}
+ // ============ Tests for Implicit Conjunction (CONJ_NONE) ============
+
+ @Test
+ public void testLuceneModeImplicitConjunctionAndOperator() {
+ // Test: "a OR b c" with default_operator=AND
+ // In Lucene, implicit conjunction (CONJ_NONE) does NOT modify the preceding term.
+ // Only explicit AND/OR conjunctions modify the preceding term.
+ // a(CONJ_NONE)→MUST, b(CONJ_OR)→prev(a) SHOULD, b SHOULD,
+ // c(CONJ_NONE)→MUST (no modification to prev b)
+ // Result: [SHOULD(a), SHOULD(b), MUST(c)]
+ // This matches ES query_string: "a OR b c" with default_operator=AND
+ String dsl = "field:a OR field:b field:c";
+ String options = "{\"mode\":\"lucene\",\"default_operator\":\"AND\",\"minimum_should_match\":0}";
+ QsPlan plan = SearchDslParser.parseDsl(dsl, options);
+
+ Assertions.assertNotNull(plan);
+ Assertions.assertEquals(QsClauseType.OCCUR_BOOLEAN, plan.getRoot().getType());
+ Assertions.assertEquals(3, plan.getRoot().getChildren().size());
+
+ QsNode nodeA = plan.getRoot().getChildren().get(0);
+ Assertions.assertEquals("a", nodeA.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.SHOULD, nodeA.getOccur());
+
+ QsNode nodeB = plan.getRoot().getChildren().get(1);
+ Assertions.assertEquals("b", nodeB.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.SHOULD, nodeB.getOccur());
+
+ QsNode nodeC = plan.getRoot().getChildren().get(2);
+ Assertions.assertEquals("c", nodeC.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.MUST, nodeC.getOccur());
+ }
+
+ @Test
+ public void testLuceneModeImplicitConjunctionNotAndOperator() {
+ // Test: "a OR b NOT c" with default_operator=AND
+ // In Lucene, implicit NOT conjunction (CONJ_NONE + MOD_NOT) does NOT modify preceding term.
+ // a(CONJ_NONE)→MUST, b(CONJ_OR)→prev(a) SHOULD, b SHOULD,
+ // NOT c(CONJ_NONE, MOD_NOT)→MUST_NOT (no modification to prev b)
+ // Result: [SHOULD(a), SHOULD(b), MUST_NOT(c)]
+ // This matches ES query_string: "a OR b NOT c" with default_operator=AND
+ String dsl = "field:a OR field:b NOT field:c";
+ String options = "{\"mode\":\"lucene\",\"default_operator\":\"AND\",\"minimum_should_match\":0}";
+ QsPlan plan = SearchDslParser.parseDsl(dsl, options);
+
+ Assertions.assertNotNull(plan);
+ Assertions.assertEquals(QsClauseType.OCCUR_BOOLEAN, plan.getRoot().getType());
+ Assertions.assertEquals(3, plan.getRoot().getChildren().size());
+
+ QsNode nodeA = plan.getRoot().getChildren().get(0);
+ Assertions.assertEquals("a", nodeA.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.SHOULD, nodeA.getOccur());
+
+ QsNode nodeB = plan.getRoot().getChildren().get(1);
+ Assertions.assertEquals("b", nodeB.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.SHOULD, nodeB.getOccur());
+
+ QsNode nodeC = plan.getRoot().getChildren().get(2);
+ Assertions.assertEquals("c", nodeC.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.MUST_NOT, nodeC.getOccur());
+ }
+
+ @Test
+ public void testLuceneModeImplicitConjunctionOrOperator() {
+ // Test: "a OR b c" with default_operator=OR
+ // With OR_OPERATOR, implicit conjunction gives SHOULD to current term.
+ // a(CONJ_NONE)→SHOULD, b(CONJ_OR)→SHOULD, c(CONJ_NONE)→SHOULD
+ // Result: [SHOULD(a), SHOULD(b), SHOULD(c)]
+ String dsl = "field:a OR field:b field:c";
+ String options = "{\"mode\":\"lucene\",\"minimum_should_match\":0}";
+ QsPlan plan = SearchDslParser.parseDsl(dsl, options);
+
+ Assertions.assertNotNull(plan);
+ Assertions.assertEquals(QsClauseType.OCCUR_BOOLEAN, plan.getRoot().getType());
+ Assertions.assertEquals(3, plan.getRoot().getChildren().size());
+
+ for (QsNode child : plan.getRoot().getChildren()) {
+ Assertions.assertEquals(SearchDslParser.QsOccur.SHOULD, child.getOccur());
+ }
+ }
+
+ @Test
+ public void testLuceneModeExplicitAndStillModifiesPrev() {
+ // Test: "a OR b AND c" with default_operator=AND
+ // Explicit AND SHOULD modify the preceding term, unlike implicit conjunction.
+ // a(CONJ_NONE)→MUST, b(CONJ_OR)→prev(a) SHOULD, b SHOULD,
+ // c(CONJ_AND)→prev(b) MUST, c MUST
+ // Result: [SHOULD(a), MUST(b), MUST(c)]
+ String dsl = "field:a OR field:b AND field:c";
+ String options = "{\"mode\":\"lucene\",\"default_operator\":\"AND\",\"minimum_should_match\":0}";
+ QsPlan plan = SearchDslParser.parseDsl(dsl, options);
+
+ Assertions.assertNotNull(plan);
+ Assertions.assertEquals(QsClauseType.OCCUR_BOOLEAN, plan.getRoot().getType());
+ Assertions.assertEquals(3, plan.getRoot().getChildren().size());
+
+ QsNode nodeA = plan.getRoot().getChildren().get(0);
+ Assertions.assertEquals("a", nodeA.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.SHOULD, nodeA.getOccur());
+
+ QsNode nodeB = plan.getRoot().getChildren().get(1);
+ Assertions.assertEquals("b", nodeB.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.MUST, nodeB.getOccur());
+
+ QsNode nodeC = plan.getRoot().getChildren().get(2);
+ Assertions.assertEquals("c", nodeC.getValue());
+ Assertions.assertEquals(SearchDslParser.QsOccur.MUST, nodeC.getOccur());
+ }
+
// ============ Tests for Escape Handling ============
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org