You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2026/04/30 02:01:22 UTC
(pinot) branch master updated: Add SEMANTIC_MATCH filter predicate for hidden semantic search layer (#18191)
This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 653b9b1cf6c Add SEMANTIC_MATCH filter predicate for hidden semantic search layer (#18191)
653b9b1cf6c is described below
commit 653b9b1cf6c945dd1e501ae400a9cabb0d5eb54e
Author: Xiang Fu <xi...@apache.org>
AuthorDate: Wed Apr 29 19:01:16 2026 -0700
Add SEMANTIC_MATCH filter predicate for hidden semantic search layer (#18191)
Registers SEMANTIC_MATCH(column, queryText, topK) as a Calcite SQL function
and FilterKind so it can be parsed and passed through the query rewriter chain.
The SemanticSearchQueryRewriter (in startree-pinot) translates it to
VECTOR_SIMILARITY at query time after computing the embedding vector.
Co-authored-by: Claude Opus 4.6 <no...@anthropic.com>
---
.../main/java/org/apache/pinot/sql/FilterKind.java | 3 ++-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 19 +++++++++++++++++++
.../parsers/rewriter/PredicateComparisonRewriter.java | 6 ++++++
.../pinot/core/function/FunctionRegistryTest.java | 1 +
.../pinot/calcite/sql/fun/PinotOperatorTable.java | 3 +++
5 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java b/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java
index 30a217fd828..5b68cbb00e9 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java
@@ -39,7 +39,8 @@ public enum FilterKind {
IS_NULL,
IS_NOT_NULL,
VECTOR_SIMILARITY,
- VECTOR_SIMILARITY_RADIUS;
+ VECTOR_SIMILARITY_RADIUS,
+ SEMANTIC_MATCH;
/**
* Helper method that returns true if the enum maps to a Range.
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
index e95ef249d9b..05ec83d6fc5 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
@@ -271,6 +271,25 @@ public class CalciteSqlParser {
for (Expression filter : filterExpression.getFunctionCall().getOperands()) {
validateFilter(filter);
}
+ } else if (operator.equals(FilterKind.SEMANTIC_MATCH.name())) {
+ // SEMANTIC_MATCH(column, 'query text', topK) — validated here, rewritten by QueryRewriter
+ List<Expression> smOperands = filterExpression.getFunctionCall().getOperands();
+ if (smOperands.size() < 2 || smOperands.size() > 3) {
+ throw new IllegalStateException(
+ "SEMANTIC_MATCH requires 2 or 3 arguments: SEMANTIC_MATCH(column, 'query text'[, topK])");
+ }
+ if (!smOperands.get(0).isSetIdentifier()) {
+ throw new IllegalStateException(
+ "The first argument of SEMANTIC_MATCH must be a column identifier");
+ }
+ if (!smOperands.get(1).isSetLiteral() || smOperands.get(1).getLiteral().isSetNullValue()) {
+ throw new IllegalStateException(
+ "The second argument of SEMANTIC_MATCH must be a non-null string literal (query text)");
+ }
+ if (smOperands.size() == 3 && !smOperands.get(2).isSetLiteral()) {
+ throw new IllegalStateException(
+ "The third argument of SEMANTIC_MATCH must be an integer literal (topK)");
+ }
} else if (operator.equals(FilterKind.VECTOR_SIMILARITY.name())) {
Expression vectorIdentifier = filterExpression.getFunctionCall().getOperands().get(0);
if (!vectorIdentifier.isSetIdentifier()) {
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
index 58937d412d8..5b9fff5659f 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
@@ -123,6 +123,12 @@ public class PredicateComparisonRewriter implements QueryRewriter {
break;
}
break;
+ case SEMANTIC_MATCH: {
+ // SEMANTIC_MATCH(column, 'query text', topK) — pass through, rewritten by semantic rewriter
+ Preconditions.checkArgument(operands.size() >= 2 && operands.size() <= 3,
+ "For %s predicate, the number of operands must be 2 or 3, got: %s", filterKind, expression);
+ break;
+ }
case VECTOR_SIMILARITY: {
Preconditions.checkArgument(operands.size() >= 2 && operands.size() <= 3,
"For %s predicate, the number of operands must be at either 2 or 3, got: %s", filterKind, expression);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java b/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java
index 52be2353a5a..abc996d2447 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java
@@ -57,6 +57,7 @@ public class FunctionRegistryTest {
private static final EnumSet<FilterKind> IGNORED_FILTER_KINDS = EnumSet.of(
// Special filter functions without implementation
FilterKind.TEXT_MATCH, FilterKind.JSON_MATCH, FilterKind.VECTOR_SIMILARITY, FilterKind.VECTOR_SIMILARITY_RADIUS,
+ FilterKind.SEMANTIC_MATCH,
// TODO: Support these functions
FilterKind.RANGE, FilterKind.IN, FilterKind.NOT_IN);
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java
index 283b644c841..57c68d5d2c8 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java
@@ -296,6 +296,9 @@ public class PinotOperatorTable implements SqlOperatorTable {
OperandTypes.family(List.of(SqlTypeFamily.ARRAY, SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER), i -> i == 2)),
new PinotSqlFunction("VECTOR_SIMILARITY_RADIUS", ReturnTypes.BOOLEAN,
OperandTypes.family(List.of(SqlTypeFamily.ARRAY, SqlTypeFamily.ARRAY, SqlTypeFamily.NUMERIC), i -> i == 2)),
+ new PinotSqlFunction("SEMANTIC_MATCH", ReturnTypes.BOOLEAN,
+ OperandTypes.family(
+ List.of(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER), i -> i == 2)),
// Placeholder for special functions to handle MV
// NOTE:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org