You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by sh...@apache.org on 2026/09/30 17:54:49 UTC
(asterixdb) branch master updated: [ASTERIXDB-3853][COMP] Inline UDFs called inside a CLUSTER BY clause
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 68f930f6db [ASTERIXDB-3853][COMP] Inline UDFs called inside a CLUSTER BY clause
68f930f6db is described below
commit 68f930f6dbf982786be86f27d20ac80288fc6ed1
Author: Shahrzad Shirazi <Sh...@couchbase.com>
AuthorDate: Tue Sep 29 11:13:58 2026 -0700
[ASTERIXDB-3853][COMP] Inline UDFs called inside a CLUSTER BY clause
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
A SQL++ UDF called in the clustering expression, e.g.
CLUSTER BY shift(p.vec), failed to compile: the function was
never found. SqlppGatherFunctionCallsVisitor and
SqlppInlineUdfsVisitor walk a SELECT block clause by clause,
and their visit(SelectBlock) skipped the CLUSTER BY clause,
so the call was neither loaded nor inlined. Both visitors
already had a visit(ClusterbyClause); it was just never
reached.
visit(SelectBlock) now visits the clause right after GROUP BY
in both visitors. Because the gather visitor also records
view and function dependencies, a UDF used only inside
CLUSTER BY is now recorded as a dependency too.
Adds cluster-by/kmeans-udf-in-clause: clusters on
shift(p.vec) and checks the centroids are the shifted means.
Ext-ref: MB-73589
Generated-by: Claude Opus 5.5 <no...@anthropic.com>
Change-Id: I5642be49b3920a37c0f3b4f25988bb3b4c1f255f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21929
Reviewed-by: Hongyu Shi <le...@gmail.com>
Tested-by: Murtadha Hubail <mh...@apache.org>
---
.../queries_sqlpp/cluster-by/ClusterByQueries.xml | 5 ++++
.../kmeans-udf-in-clause.1.ddl.sqlpp | 30 ++++++++++++++++++++++
.../kmeans-udf-in-clause.2.update.sqlpp | 26 +++++++++++++++++++
.../kmeans-udf-in-clause.3.query.sqlpp | 26 +++++++++++++++++++
.../kmeans-udf-in-clause.3.adm | 2 ++
.../visitor/SqlppGatherFunctionCallsVisitor.java | 3 +++
.../rewrites/visitor/SqlppInlineUdfsVisitor.java | 3 +++
7 files changed, 95 insertions(+)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/ClusterByQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/ClusterByQueries.xml
index 0a405cf544..d66aaf6e32 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/ClusterByQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/ClusterByQueries.xml
@@ -102,6 +102,11 @@
<output-dir compare="Text">kmeans-udf-param</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="cluster-by">
+ <compilation-unit name="kmeans-udf-in-clause">
+ <output-dir compare="Text">kmeans-udf-in-clause</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="cluster-by/negative">
<compilation-unit name="udf-correlated-clusterby">
<output-dir compare="Text">udf-correlated-clusterby</output-dir>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.1.ddl.sqlpp
new file mode 100644
index 0000000000..cc702619df
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.1.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- A UDF called inside the CLUSTER BY clause must be loaded and inlined like a UDF anywhere else.
+drop dataverse test if exists;
+create dataverse test;
+use test;
+create type PointType as closed {
+ id : integer,
+ vec : [double]
+};
+create dataset Points(PointType) primary key id;
+
+create function shift(v) { [v[0] + 1.0, v[1] + 1.0] };
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.2.update.sqlpp
new file mode 100644
index 0000000000..606f3d770c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.2.update.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+insert into Points ([
+ {"id": 1, "vec": [0.0, 0.0]},
+ {"id": 2, "vec": [1.0, 0.0]},
+ {"id": 3, "vec": [10.0, 10.0]},
+ {"id": 4, "vec": [11.0, 10.0]}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.3.query.sqlpp
new file mode 100644
index 0000000000..2d4485c359
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.3.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- UDF directly in the clustering expression: centroids are the shifted means.
+use test;
+from Points as p
+cluster by shift(p.vec) as sc
+with {"clustering_algorithm": "K-Means", "dimension": [2], "num_clusters": 2, "seed": 10 }
+select value sc
+order by sc.cluster_id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.3.adm
new file mode 100644
index 0000000000..91ba085dd5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cluster-by/kmeans-udf-in-clause/kmeans-udf-in-clause.3.adm
@@ -0,0 +1,2 @@
+{ "cluster_id": 0, "centroid": [ 1.5, 1.0 ] }
+{ "cluster_id": 1, "centroid": [ 11.5, 11.0 ] }
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGatherFunctionCallsVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGatherFunctionCallsVisitor.java
index d9733aeb1b..d87da4a8fc 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGatherFunctionCallsVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGatherFunctionCallsVisitor.java
@@ -121,6 +121,9 @@ public final class SqlppGatherFunctionCallsVisitor extends GatherFunctionCallsVi
if (selectBlock.hasGroupbyClause()) {
selectBlock.getGroupbyClause().accept(this, arg);
}
+ if (selectBlock.hasClusterbyClause()) {
+ selectBlock.getClusterbyClause().accept(this, arg);
+ }
if (selectBlock.hasLetHavingClausesAfterGroupby()) {
for (AbstractClause letHavingClause : selectBlock.getLetHavingListAfterGroupby()) {
letHavingClause.accept(this, arg);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java
index 774d1ba251..586dce02ae 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java
@@ -167,6 +167,9 @@ public class SqlppInlineUdfsVisitor extends AbstractInlineUdfsVisitor implements
if (selectBlock.hasGroupbyClause()) {
changed |= selectBlock.getGroupbyClause().accept(this, arg);
}
+ if (selectBlock.hasClusterbyClause()) {
+ changed |= selectBlock.getClusterbyClause().accept(this, arg);
+ }
if (selectBlock.hasLetHavingClausesAfterGroupby()) {
for (AbstractClause letHavingClause : selectBlock.getLetHavingListAfterGroupby()) {
changed |= letHavingClause.accept(this, arg);