You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2026/04/30 02:01:22 UTC

(pinot) branch master updated: Add SEMANTIC_MATCH filter predicate for hidden semantic search layer (#18191)

This is an automated email from the ASF dual-hosted git repository.

xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 653b9b1cf6c Add SEMANTIC_MATCH filter predicate for hidden semantic search layer (#18191)
653b9b1cf6c is described below

commit 653b9b1cf6c945dd1e501ae400a9cabb0d5eb54e
Author: Xiang Fu <xi...@apache.org>
AuthorDate: Wed Apr 29 19:01:16 2026 -0700

    Add SEMANTIC_MATCH filter predicate for hidden semantic search layer (#18191)
    
    Registers SEMANTIC_MATCH(column, queryText, topK) as a Calcite SQL function
    and FilterKind so it can be parsed and passed through the query rewriter chain.
    The SemanticSearchQueryRewriter (in startree-pinot) translates it to
    VECTOR_SIMILARITY at query time after computing the embedding vector.
    
    Co-authored-by: Claude Opus 4.6 <no...@anthropic.com>
---
 .../main/java/org/apache/pinot/sql/FilterKind.java    |  3 ++-
 .../apache/pinot/sql/parsers/CalciteSqlParser.java    | 19 +++++++++++++++++++
 .../parsers/rewriter/PredicateComparisonRewriter.java |  6 ++++++
 .../pinot/core/function/FunctionRegistryTest.java     |  1 +
 .../pinot/calcite/sql/fun/PinotOperatorTable.java     |  3 +++
 5 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java b/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java
index 30a217fd828..5b68cbb00e9 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/FilterKind.java
@@ -39,7 +39,8 @@ public enum FilterKind {
   IS_NULL,
   IS_NOT_NULL,
   VECTOR_SIMILARITY,
-  VECTOR_SIMILARITY_RADIUS;
+  VECTOR_SIMILARITY_RADIUS,
+  SEMANTIC_MATCH;
 
   /**
    * Helper method that returns true if the enum maps to a Range.
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
index e95ef249d9b..05ec83d6fc5 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
@@ -271,6 +271,25 @@ public class CalciteSqlParser {
       for (Expression filter : filterExpression.getFunctionCall().getOperands()) {
         validateFilter(filter);
       }
+    } else if (operator.equals(FilterKind.SEMANTIC_MATCH.name())) {
+      // SEMANTIC_MATCH(column, 'query text', topK) — validated here, rewritten by QueryRewriter
+      List<Expression> smOperands = filterExpression.getFunctionCall().getOperands();
+      if (smOperands.size() < 2 || smOperands.size() > 3) {
+        throw new IllegalStateException(
+            "SEMANTIC_MATCH requires 2 or 3 arguments: SEMANTIC_MATCH(column, 'query text'[, topK])");
+      }
+      if (!smOperands.get(0).isSetIdentifier()) {
+        throw new IllegalStateException(
+            "The first argument of SEMANTIC_MATCH must be a column identifier");
+      }
+      if (!smOperands.get(1).isSetLiteral() || smOperands.get(1).getLiteral().isSetNullValue()) {
+        throw new IllegalStateException(
+            "The second argument of SEMANTIC_MATCH must be a non-null string literal (query text)");
+      }
+      if (smOperands.size() == 3 && !smOperands.get(2).isSetLiteral()) {
+        throw new IllegalStateException(
+            "The third argument of SEMANTIC_MATCH must be an integer literal (topK)");
+      }
     } else if (operator.equals(FilterKind.VECTOR_SIMILARITY.name())) {
       Expression vectorIdentifier = filterExpression.getFunctionCall().getOperands().get(0);
       if (!vectorIdentifier.isSetIdentifier()) {
diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
index 58937d412d8..5b9fff5659f 100644
--- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
+++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java
@@ -123,6 +123,12 @@ public class PredicateComparisonRewriter implements QueryRewriter {
             break;
           }
           break;
+        case SEMANTIC_MATCH: {
+          // SEMANTIC_MATCH(column, 'query text', topK) — pass through, rewritten by semantic rewriter
+          Preconditions.checkArgument(operands.size() >= 2 && operands.size() <= 3,
+              "For %s predicate, the number of operands must be 2 or 3, got: %s", filterKind, expression);
+          break;
+        }
         case VECTOR_SIMILARITY: {
           Preconditions.checkArgument(operands.size() >= 2 && operands.size() <= 3,
               "For %s predicate, the number of operands must be at either 2 or 3, got: %s", filterKind, expression);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java b/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java
index 52be2353a5a..abc996d2447 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/function/FunctionRegistryTest.java
@@ -57,6 +57,7 @@ public class FunctionRegistryTest {
   private static final EnumSet<FilterKind> IGNORED_FILTER_KINDS = EnumSet.of(
       // Special filter functions without implementation
       FilterKind.TEXT_MATCH, FilterKind.JSON_MATCH, FilterKind.VECTOR_SIMILARITY, FilterKind.VECTOR_SIMILARITY_RADIUS,
+      FilterKind.SEMANTIC_MATCH,
       // TODO: Support these functions
       FilterKind.RANGE, FilterKind.IN, FilterKind.NOT_IN);
 
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java
index 283b644c841..57c68d5d2c8 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java
@@ -296,6 +296,9 @@ public class PinotOperatorTable implements SqlOperatorTable {
           OperandTypes.family(List.of(SqlTypeFamily.ARRAY, SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER), i -> i == 2)),
       new PinotSqlFunction("VECTOR_SIMILARITY_RADIUS", ReturnTypes.BOOLEAN,
           OperandTypes.family(List.of(SqlTypeFamily.ARRAY, SqlTypeFamily.ARRAY, SqlTypeFamily.NUMERIC), i -> i == 2)),
+      new PinotSqlFunction("SEMANTIC_MATCH", ReturnTypes.BOOLEAN,
+          OperandTypes.family(
+              List.of(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER), i -> i == 2)),
 
       // Placeholder for special functions to handle MV
       // NOTE:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org