You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2026/03/04 23:23:41 UTC
(asterixdb) branch master updated (b744be6dc2 -> 0fba729d23)
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
from b744be6dc2 [NO ISSUE][EXT]: avoid closing resources on non-initialized tasks
new 7f7af27cfd [ASTERIXDB-3706][COMP] Support IF NOT EXISTS Before and After identifier in DDL Statements
new 466bb8b8b5 [ASTERIXDB-3634][EXT]: Validate snapshot on creation and query
new 1933dfca4b [ASTERIXDB-3677][COMP] Fix nested star operators failure
new bee318fb0a Null merge branch 'gerrit/morpheus' into 'master'
new 8f3b0f6495 [ASTERIXDB-3677][COMP] Fix nested star operators failure
new bfdc44e92a [NO ISSUE][HYR][NET] Upgrade gRPC to address Netty CVE
new 627a10668c [NO ISSUE][MISC] Update Log4j to 2.25.3 for CVE
new 5510be7dc6 Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
new f42ec662c3 [NO ISSUE][*DB][EXT] Update Apache Hadoop -> 3.4.2 for CVE
new 1ed3c060ba Merge branch 'gerrit/neo' into 'gerrit/trinity'
new a2e1e197ca Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
new 27562debb3 Merge branch 'gerrit/morpheus' into 'master'
new 996a015003 MB-70678: re-resolve cluster address on initial connect retry
new b4f3098763 Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
new 287a9a6416 Merge branch 'gerrit/morpheus' into 'master'
new 72a01fe09f [ASTERIXDB-3705][GEO] Add new geospatial functions
new deca321998 [ASTERIXDB-3636][API] Migrate UDF API to domain sockets
new 0fba729d23 [NO ISSUE]Update README
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 26 +-
.../apache/asterix/optimizer/rules/cbo/Stats.java | 2 +-
.../apache/asterix/translator/Receptionist.java | 8 +-
asterixdb/asterix-app/pom.xml | 30 ++
.../api/http/server/AbstractNCUdfServlet.java | 276 ++++++++++++++++++-
.../asterix/api/http/server/NCUdfApiServlet.java | 302 ++-------------------
.../asterix/api/http/server/NCUdfDSApiServlet.java | 116 ++++++++
.../api/http/server/NCUdfRecoveryServlet.java | 18 +-
.../asterix/app/translator/QueryTranslator.java | 26 +-
...er.java => IcebergCatalogStatementHandler.java} | 4 +-
.../helpers/IcebergStatementValidationHelper.java | 74 +++++
.../asterix/hyracks/bootstrap/NCApplication.java | 48 +++-
.../api/common/CloudStorageIntegrationUtil.java | 10 +-
.../asterix/app/external/CloudUDFLibrarian.java | 112 ++++++++
.../asterix/app/external/ExternalUDFLibrarian.java | 13 +-
.../app/external/IExternalUDFLibrarian.java | 12 +-
.../test/cloud_storage/CloudPythonTest.java | 7 +-
.../apache/asterix/test/common/TestExecutor.java | 13 +-
.../src/test/resources/cc-cloud-storage-main.conf | 4 +-
.../single-method/single-method.16.query.sqlpp} | 19 +-
.../single-method/single-method.17.query.sqlpp} | 17 +-
.../single-method/single-method.18.query.sqlpp} | 17 +-
.../single-method/single-method.19.query.sqlpp} | 18 +-
.../single-method/single-method.20.query.sqlpp} | 19 +-
.../single-method/single-method.21.query.sqlpp} | 17 +-
.../two-geometries/two-geometries.06.query.sqlpp} | 16 +-
.../results/geojson/datatype/result.08.adm | 4 +-
.../results/geojson/single-method/result.16.adm | 1 +
.../results/geojson/single-method/result.17.adm | 1 +
.../results/geojson/single-method/result.18.adm | 1 +
.../results/geojson/single-method/result.19.adm | 1 +
.../results/geojson/single-method/result.20.adm | 1 +
.../results/geojson/single-method/result.21.adm | 1 +
.../results/geojson/two-geometries/result.06.adm | 10 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 2 +-
.../src/main/resources/asx_errormsg/en.properties | 2 +-
.../src/main/user-defined_function/udf.md | 4 +-
.../asterix-doc/src/site/markdown/geo/functions.md | 102 +++++++
.../aws/iceberg/IcebergFileRecordReader.java | 2 +-
.../iceberg/IcebergParquetRecordReaderFactory.java | 26 +-
.../util/iceberg/IcebergSnapshotUtils.java | 115 ++++++++
.../external/util/iceberg/IcebergUtils.java | 28 +-
.../geo/evaluators/GeoFunctionRegistrant.java | 16 ++
.../functions/AbstractSTGeometryNDescriptor.java | 1 +
...tryNDescriptor.java => STBufferDescriptor.java} | 77 +++---
.../evaluators/functions/STCentroidDescriptor.java | 42 +++
.../functions/STConvexHullDescriptor.java | 42 +++
.../evaluators/functions/STDWithinDescriptor.java | 139 ++++++++++
.../functions/STDistanceSphereDescriptor.java | 59 ++++
.../functions/STFlipCoordinatesDescriptor.java | 69 +++++
.../evaluators/functions/STIsValidDescriptor.java | 42 +++
.../evaluators/functions/STReverseDescriptor.java | 42 +++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 245 ++++++++++++++---
.../metadata/declared/MetadataProvider.java | 2 +-
.../asterix/om/functions/BuiltinFunctions.java | 19 ++
.../asterix/test/podman/PodmanUDFLibrarian.java | 14 +-
asterixdb/pom.xml | 11 +-
.../control/common/controllers/NCConfig.java | 8 +
.../apache/hyracks/http/api/IServletRequest.java | 4 +
.../apache/hyracks/http/server/BaseRequest.java | 19 ++
.../org/apache/hyracks/http/server/CLFLogger.java | 8 +-
.../org/apache/hyracks/http/server/HttpServer.java | 95 ++++++-
.../hyracks/http/server/HttpServerInitializer.java | 6 +-
.../hyracks/ipc/impl/IPCConnectionManager.java | 1 +
hyracks-fullstack/pom.xml | 4 +-
65 files changed, 1865 insertions(+), 625 deletions(-)
create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfDSApiServlet.java
rename asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/{CatalogStatementHandler.java => IcebergCatalogStatementHandler.java} (98%)
create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java
create mode 100644 asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/CloudUDFLibrarian.java
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/app/external/IExternalUDFLibrarian.java => resources/runtimets/queries_sqlpp/geojson/single-method/single-method.16.query.sqlpp} (63%)
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/app/external/IExternalUDFLibrarian.java => resources/runtimets/queries_sqlpp/geojson/single-method/single-method.17.query.sqlpp} (63%)
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/app/external/IExternalUDFLibrarian.java => resources/runtimets/queries_sqlpp/geojson/single-method/single-method.18.query.sqlpp} (63%)
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/app/external/IExternalUDFLibrarian.java => resources/runtimets/queries_sqlpp/geojson/single-method/single-method.19.query.sqlpp} (63%)
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/app/external/IExternalUDFLibrarian.java => resources/runtimets/queries_sqlpp/geojson/single-method/single-method.20.query.sqlpp} (63%)
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/app/external/IExternalUDFLibrarian.java => resources/runtimets/queries_sqlpp/geojson/single-method/single-method.21.query.sqlpp} (63%)
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/app/external/IExternalUDFLibrarian.java => resources/runtimets/queries_sqlpp/geojson/two-geometries/two-geometries.06.query.sqlpp} (63%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.16.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.17.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.18.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.19.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.20.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.21.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/two-geometries/result.06.adm
create mode 100644 asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergSnapshotUtils.java
copy asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/{AbstractSTGeometryNDescriptor.java => STBufferDescriptor.java} (61%)
create mode 100644 asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STCentroidDescriptor.java
create mode 100644 asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STConvexHullDescriptor.java
create mode 100644 asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDWithinDescriptor.java
create mode 100644 asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDistanceSphereDescriptor.java
create mode 100644 asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STFlipCoordinatesDescriptor.java
create mode 100644 asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STIsValidDescriptor.java
create mode 100644 asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STReverseDescriptor.java
(asterixdb) 06/18: [NO ISSUE][MISC] Update Log4j to 2.25.3 for CVE
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 627a10668c5bfef86d27a6a6d5ea7341555daf0a
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Feb 21 12:21:36 2026 -0500
[NO ISSUE][MISC] Update Log4j to 2.25.3 for CVE
Ext-ref: MB-70643
Change-Id: I7446e4e53684bc54bd41ce96ef1d866ad4db67f8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20925
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
asterixdb/pom.xml | 2 +-
hyracks-fullstack/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 96a831e070..a661f161b9 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -87,7 +87,7 @@
<hyracks.version>0.3.8.3-SNAPSHOT</hyracks.version>
<hadoop.version>3.4.1</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.22.1</log4j.version>
+ <log4j.version>2.25.3</log4j.version>
<awsjavasdk.version>2.29.27</awsjavasdk.version>
<parquet.version>1.15.2</parquet.version> <!-- NOTICE: please update transitives from parquet below on any change -->
<hadoop-awsjavasdk.version>1.12.779</hadoop-awsjavasdk.version>
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index a8ea92cdb4..459fc90b59 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -71,7 +71,7 @@
<!-- Versions under dependencymanagement or used in many projects via properties -->
<hadoop.version>3.4.1</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.22.1</log4j.version>
+ <log4j.version>2.25.3</log4j.version>
<snappy.version>1.1.10.5</snappy.version>
<jackson.version>2.19.2</jackson.version>
<jackson-databind.version>${jackson.version}</jackson-databind.version>
(asterixdb) 07/18: Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5510be7dc62018250046499f42a3f5dec58cac77
Merge: 8f3b0f6495 627a10668c
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Feb 22 00:06:41 2026 -0500
Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
* [ASTERIXDB-3677][COMP] Fix nested star operators failure (MB-70076)
* [NO ISSUE][HYR][NET] Upgrade gRPC to address Netty CVE (MB-70641)
* [NO ISSUE][MISC] Update Log4j to 2.25.3 for CVE (MB-70643)
Ext-ref: MB-70641,MB-70642,MB-70643
Change-Id: Idcaefe1920e9018080e1a14cac961b1ae6dd67d3
asterixdb/pom.xml | 4 ++--
hyracks-fullstack/pom.xml | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
(asterixdb) 13/18: MB-70678: re-resolve cluster address on initial connect retry
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 996a0150031d89cb1379bc337a00b76e54ac9123
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Tue Feb 24 16:04:32 2026 -0500
MB-70678: re-resolve cluster address on initial connect retry
Change-Id: I9b8cebcfb6cba48cba039478013a21b4aa7b5ec2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20934
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Tested-by: Michael Blow <mb...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java | 1 +
hyracks-fullstack/pom.xml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java
index 5834f26d66..cdb91c3c71 100644
--- a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java
@@ -140,6 +140,7 @@ public class IPCConnectionManager {
}
Thread.sleep(delay);
delay = Math.min(MAX_RETRY_DELAY_MILLIS, (int) (delay * 1.5));
+ remoteAddress = NetworkUtil.refresh(remoteAddress);
} else {
throw new IOException("Connection failed to " + remoteAddress);
}
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 459fc90b59..41daad0878 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -69,7 +69,7 @@
<test.includes>${global.test.includes}</test.includes>
<test.excludes>${global.test.excludes}</test.excludes>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <hadoop.version>3.4.1</hadoop.version>
+ <hadoop.version>3.4.2</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.25.3</log4j.version>
<snappy.version>1.1.10.5</snappy.version>
(asterixdb) 09/18: Merge branch 'gerrit/neo' into 'gerrit/trinity'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1ed3c060bacf0603c5e8ee307b44e35f1bd78dc7
Merge: 627a10668c f42ec662c3
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Feb 22 18:06:21 2026 -0500
Merge branch 'gerrit/neo' into 'gerrit/trinity'
* [NO ISSUE][*DB][EXT] Update Apache Hadoop -> 3.4.2 for CVE (MB-70644)
Ext-ref: MB-70644
Change-Id: I0e6cfb7cf9893572087038c15a83488ee41817a9
asterixdb/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --cc asterixdb/pom.xml
index a661f161b9,8c503f54ba..4f4e82590b
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@@ -83,11 -82,11 +83,11 @@@
<skip-npm-touch.stage>none</skip-npm-touch.stage>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <algebricks.version>0.3.8.2-SNAPSHOT</algebricks.version>
- <hyracks.version>0.3.8.2-SNAPSHOT</hyracks.version>
+ <algebricks.version>0.3.8.3-SNAPSHOT</algebricks.version>
+ <hyracks.version>0.3.8.3-SNAPSHOT</hyracks.version>
- <hadoop.version>3.4.1</hadoop.version>
+ <hadoop.version>3.4.2</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.19.0</log4j.version>
+ <log4j.version>2.25.3</log4j.version>
<awsjavasdk.version>2.29.27</awsjavasdk.version>
<parquet.version>1.15.2</parquet.version> <!-- NOTICE: please update transitives from parquet below on any change -->
<hadoop-awsjavasdk.version>1.12.779</hadoop-awsjavasdk.version>
(asterixdb) 18/18: [NO ISSUE]Update README
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0fba729d23d58801087f834c198ea1649edf427b
Author: Ian Maxon <im...@apache.org>
AuthorDate: Thu Feb 26 09:03:51 2026 -0800
[NO ISSUE]Update README
Change-Id: I434142a9b9cd2d1fc941b1e1f250ef7403a893ed
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20526
Reviewed-by: Ian Maxon <im...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Suryaa Charan Shivakumar <ss...@ucr.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
README.md | 26 ++++++++++----------------
1 file changed, 10 insertions(+), 16 deletions(-)
diff --git a/README.md b/README.md
index 3b52c765c7..203491452b 100644
--- a/README.md
+++ b/README.md
@@ -20,20 +20,20 @@
## What is AsterixDB?
-AsterixDB is a BDMS (Big Data Management System) with a rich feature set that sets it apart from other Big Data platforms. Its feature set makes it well-suited to modern needs such as web data warehousing and social data storage and analysis. AsterixDB has:
+AsterixDB is a BDMS (Big Data Management System) with a rich feature set that sets it apart from other platforms. Its feature set makes it well-suited to modern needs such as web data warehousing and social data storage and analysis. AsterixDB has:
- __Data model__<br/>
-A semistructured NoSQL style data model ([ADM](https://ci.apache.org/projects/asterixdb/datamodel.html)) resulting from
+A semistructured NoSQL style data model ([ADM](https://nightlies.apache.org/asterixdb/datamodel.html)) resulting from
extending JSON with object database ideas
- __Query languages__<br/>
-An expressive and declarative query language ([SQL++](http://asterixdb.apache.org/docs/0.9.7/sqlpp/manual.html) that supports a broad range of queries and analysis over semistructured data
+An expressive and declarative query language ([SQL++](http://asterixdb.apache.org/docs/0.9.9/sqlpp/manual.html) that supports a broad range of queries and analysis over semistructured data
- __Scalability__<br/>
A parallel runtime query execution engine, Apache Hyracks, that has been scale-tested on up to 1000+ cores and 500+ disks
- __Native storage__<br/>
-Partitioned LSM-based data storage and indexing to support efficient ingestion and management of semistructured data
+Partitioned, flexible LSM-based row and column storage, with indexing to support efficient ingestion and management of semistructured data
- __External storage__<br/>
Support for query access to externally stored data (e.g., data in HDFS) as well as to data stored natively by AsterixDB
@@ -57,8 +57,8 @@ To build AsterixDB from source, you should have a platform with the following:
* A Unix-ish environment (Linux, OS X, will all do).
* git
* Maven 3.3.9 or newer.
-* JDK 11 or newer.
-* Python 3.6+ with pip and venv
+* JDK 17 or newer.
+* Python 3.11+ with pip and venv
Instructions for building the master:
@@ -84,7 +84,7 @@ Here are steps to get AsterixDB running on your local machine:
http://localhost:19006
-* Read more [documentation](https://ci.apache.org/projects/asterixdb/index.html) to learn the data model, query language, and how to create a cluster instance.
+* Read more [documentation](https://nightlies.apache.org/asterixdb/index.html) to learn the data model, query language, and how to create a cluster instance.
## Documentation
@@ -92,16 +92,10 @@ To generate the documentation, run asterix-doc with the generate.rr profile in m
Be sure to run `mvn package` beforehand or run `mvn site` in asterix-lang-sqlpp to generate some resources that
are used in the documentation that are generated directly from the grammar.
-* [master](https://ci.apache.org/projects/asterixdb/index.html) |
+* [master](https://nightlies.apache.org/asterixdb/index.html) |
+ [0.9.9](http://asterixdb.apache.org/docs/0.9.9/index.html) |
+ [0.9.8](http://asterixdb.apache.org/docs/0.9.8/index.html) |
[0.9.7](http://asterixdb.apache.org/docs/0.9.7/index.html) |
- [0.9.6](http://asterixdb.apache.org/docs/0.9.6/index.html) |
- [0.9.5](http://asterixdb.apache.org/docs/0.9.5/index.html) |
- [0.9.4.1](http://asterixdb.apache.org/docs/0.9.4.1/index.html) |
- [0.9.4](http://asterixdb.apache.org/docs/0.9.4/index.html) |
- [0.9.3](http://asterixdb.apache.org/docs/0.9.3/index.html) |
- [0.9.2](http://asterixdb.apache.org/docs/0.9.2/index.html) |
- [0.9.1](http://asterixdb.apache.org/docs/0.9.1/index.html) |
- [0.9.0](http://asterixdb.apache.org/docs/0.9.0/index.html)
## Community support
(asterixdb) 12/18: Merge branch 'gerrit/morpheus' into 'master'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 27562debb36efee63891a55ccc44c4b8d519709c
Merge: bee318fb0a a2e1e197ca
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Mon Feb 23 17:44:13 2026 -0500
Merge branch 'gerrit/morpheus' into 'master'
* [ASTERIXDB-3677][COMP] Fix nested star operators failure (MB-70076)
* [NO ISSUE][HYR][NET] Upgrade gRPC to address Netty CVE (MB-70641)
* [NO ISSUE][MISC] Update Log4j to 2.25.3 for CVE (MB-70643)
* [NO ISSUE][*DB][EXT] Update Apache Hadoop -> 3.4.2 for CVE (MB-70644)
Change-Id: Ic7feef98087205543e831d44edd1d9acea5045cb
.../java/org/apache/asterix/optimizer/rules/cbo/Stats.java | 2 +-
asterixdb/pom.xml | 11 +++--------
hyracks-fullstack/pom.xml | 2 +-
3 files changed, 5 insertions(+), 10 deletions(-)
diff --cc asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
index 476ef27f1f,38d817b454..bdcbe3cd1c
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
@@@ -19,36 -19,15 +19,36 @@@
package org.apache.asterix.optimizer.rules.cbo;
+import static org.apache.asterix.om.functions.BuiltinFunctions.getBuiltinFunctionInfo;
+
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.compiler.provider.IRuleSetFactory;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.DataSource;
-import org.apache.asterix.metadata.declared.DataSourceId;
-import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.declared.SampleDataSource;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.base.ADouble;
+import org.apache.asterix.om.base.AInt64;
+import org.apache.asterix.om.base.ARecord;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.exceptions.ExceptionUtil;
+import org.apache.asterix.om.functions.BuiltinFunctionInfo;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.optimizer.base.AnalysisUtil;
+import org.apache.asterix.optimizer.rules.am.array.AbstractOperatorFromSubplanRewrite;
+import org.apache.asterix.translator.ConstantHelper;
- import org.apache.commons.collections.CollectionUtils;
++import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
diff --cc asterixdb/pom.xml
index 8d9a5c818e,a91b3e07af..aecb8b5b07
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@@ -81,29 -81,16 +81,29 @@@
<shim.stage>none</shim.stage>
<pytestlib.stage>none</pytestlib.stage>
<skip-npm-touch.stage>none</skip-npm-touch.stage>
+ <azurite.npm.install.stage>none</azurite.npm.install.stage>
+ <azurite.install.stage>none</azurite.install.stage>
+ <azurite.stage>none</azurite.stage>
+ <gcs.download.stage>none</gcs.download.stage>
+ <gcs.install.stage>none</gcs.install.stage>
+ <gcs.stage>none</gcs.stage>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <algebricks.version>0.3.8.3-SNAPSHOT</algebricks.version>
- <hyracks.version>0.3.8.3-SNAPSHOT</hyracks.version>
+ <algebricks.version>0.3.10-SNAPSHOT</algebricks.version>
+ <hyracks.version>0.3.10-SNAPSHOT</hyracks.version>
- <hadoop.version>3.4.1</hadoop.version>
+ <hadoop.version>3.4.2</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
- <log4j.version>2.22.1</log4j.version>
+ <log4j.version>2.25.3</log4j.version>
+ <!-- IMPORTANT: please keep the aws-crt version in sync with that defined in the AWS SDK BOM -->
+ <!-- you can get this by inspecting the aws-sdk-java-pom for the SDK version. e.g.
+ $ curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/aws-sdk-java-pom/2.31.57/aws-sdk-java-pom-2.31.57.pom | grep awscrt.version
+ <awscrt.version>0.38.1</awscrt.version>
+ -->
<awsjavasdk.version>2.29.27</awsjavasdk.version>
- <parquet.version>1.15.2</parquet.version> <!-- NOTICE: please update transitives from parquet below on any change -->
- <hadoop-awsjavasdk.version>1.12.779</hadoop-awsjavasdk.version>
+ <awsjavasdk.crt.version>0.33.3</awsjavasdk.crt.version>
+
+ <parquet.version>1.16.0</parquet.version>
+ <hadoop-awsjavasdk.version>1.12.788</hadoop-awsjavasdk.version>
<azureblobjavasdk.version>12.31.1</azureblobjavasdk.version>
<azurecommonjavasdk.version>12.30.1</azurecommonjavasdk.version>
(asterixdb) 17/18: [ASTERIXDB-3636][API] Migrate UDF API to domain sockets
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit deca3219983b6f906f0d6b20f5c87277de028862
Author: Ian Maxon <im...@apache.org>
AuthorDate: Wed Oct 29 11:26:23 2025 -0700
[ASTERIXDB-3636][API] Migrate UDF API to domain sockets
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
Migrate the UDF API to use a domain socket, instead of
being restricted to localhost on the normal NC servlet
Change-Id: I5f8ac2170fd6b2beef14d99c38b9141af0f12ba3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20492
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
---
.../apache/asterix/translator/Receptionist.java | 8 +-
asterixdb/asterix-app/pom.xml | 30 ++
.../api/http/server/AbstractNCUdfServlet.java | 276 ++++++++++++++++++-
.../asterix/api/http/server/NCUdfApiServlet.java | 302 ++-------------------
.../asterix/api/http/server/NCUdfDSApiServlet.java | 116 ++++++++
.../api/http/server/NCUdfRecoveryServlet.java | 18 +-
.../asterix/hyracks/bootstrap/NCApplication.java | 48 +++-
.../api/common/CloudStorageIntegrationUtil.java | 10 +-
.../asterix/app/external/CloudUDFLibrarian.java | 112 ++++++++
.../asterix/app/external/ExternalUDFLibrarian.java | 13 +-
.../app/external/IExternalUDFLibrarian.java | 12 +-
.../test/cloud_storage/CloudPythonTest.java | 7 +-
.../apache/asterix/test/common/TestExecutor.java | 13 +-
.../src/test/resources/cc-cloud-storage-main.conf | 4 +-
.../src/main/user-defined_function/udf.md | 4 +-
.../asterix/test/podman/PodmanUDFLibrarian.java | 14 +-
.../control/common/controllers/NCConfig.java | 8 +
.../apache/hyracks/http/api/IServletRequest.java | 4 +
.../apache/hyracks/http/server/BaseRequest.java | 19 ++
.../org/apache/hyracks/http/server/CLFLogger.java | 8 +-
.../org/apache/hyracks/http/server/HttpServer.java | 95 ++++++-
.../hyracks/http/server/HttpServerInitializer.java | 6 +-
22 files changed, 762 insertions(+), 365 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
index 524af430e6..12619a96fd 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.translator;
-import java.net.InetSocketAddress;
import java.util.UUID;
import org.apache.asterix.common.api.IClientRequest;
@@ -31,18 +30,15 @@ import org.apache.http.HttpHeaders;
import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
-import org.apache.hyracks.util.NetworkUtil;
public class Receptionist implements IReceptionist {
@Override
public IRequestReference welcome(IServletRequest request) {
final String uuid = UUID.randomUUID().toString();
- final InetSocketAddress localAddress = request.getLocalAddress();
- final RequestReference ref =
- RequestReference.of(uuid, NetworkUtil.toHostPort(localAddress), System.currentTimeMillis());
+ final RequestReference ref = RequestReference.of(uuid, request.getHostPort(), System.currentTimeMillis());
ref.setUserAgent(request.getHeader(HttpHeaders.USER_AGENT));
- ref.setRemoteAddr(NetworkUtil.toHostPort(request.getRemoteAddress()));
+ ref.setRemoteAddr(request.getRemotePort());
return ref;
}
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 3b27957944..c9486c6149 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -842,6 +842,36 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.client5</groupId>
+ <artifactId>httpclient5</artifactId>
+ <version>5.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.projectreactor.netty</groupId>
+ <artifactId>reactor-netty</artifactId>
+ <version>1.2.10</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-classes-epoll</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-kqueue</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-classes-kqueue</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-lang-common</artifactId>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
index 8c5a93b213..3f22014547 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractNCUdfServlet.java
@@ -18,38 +18,71 @@
*/
package org.apache.asterix.api.http.server;
+import static org.apache.asterix.api.http.server.ServletConstants.SYS_AUTH_HEADER;
import static org.apache.asterix.common.functions.ExternalFunctionLanguage.JAVA;
import static org.apache.asterix.common.functions.ExternalFunctionLanguage.PYTHON;
+import static org.apache.asterix.common.library.LibraryDescriptor.FIELD_HASH;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PrintWriter;
+import java.net.InetAddress;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import org.apache.asterix.app.message.CreateLibraryRequestMessage;
+import org.apache.asterix.app.message.DropLibraryRequestMessage;
+import org.apache.asterix.app.message.InternalRequestResponse;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.api.IReceptionist;
+import org.apache.asterix.common.api.IRequestReference;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
+import org.apache.asterix.common.messaging.api.INCMessageBroker;
+import org.apache.asterix.common.messaging.api.MessageFuture;
+import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.Namespace;
+import org.apache.asterix.external.util.ExternalLibraryUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.exceptions.IFormattedException;
-import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.common.work.SynchronizableWork;
import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.AbstractServlet;
import org.apache.hyracks.http.server.utils.HttpUtil;
+import org.apache.hyracks.util.JSONUtil;
+import org.apache.hyracks.util.file.FileUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http.HttpScheme;
import io.netty.handler.codec.http.multipart.FileUpload;
import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder;
import io.netty.handler.codec.http.multipart.InterfaceHttpData;
@@ -61,8 +94,13 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet {
INCServiceContext srvCtx;
protected final IApplicationContext plainAppCtx;
- private final HttpScheme httpServerProtocol;
- private final int httpServerPort;
+ protected final IReceptionist receptionist;
+ protected final int timeout;
+ protected ILibraryManager libraryManager;
+ protected Path workingDir;
+ protected String sysAuthHeader;
+
+ private static final Logger LOGGER = LogManager.getLogger();
public static final String GET_UDF_DIST_ENDPOINT = "/dist";
public static final String TYPE_PARAMETER = "type";
@@ -90,15 +128,148 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet {
}
- public AbstractNCUdfServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx,
- HttpScheme httpServerProtocol, int httpServerPort) {
+ public AbstractNCUdfServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx) {
super(ctx, paths);
this.plainAppCtx = appCtx;
- this.httpServerProtocol = httpServerProtocol;
- this.httpServerPort = httpServerPort;
+ this.timeout = appCtx.getExternalProperties().getLibraryDeployTimeout();
+ this.receptionist = appCtx.getReceptionist();
+ }
+
+ public void init() throws IOException {
+ appCtx = (INcApplicationContext) plainAppCtx;
+ this.libraryManager = appCtx.getLibraryManager();
+ srvCtx = this.appCtx.getServiceContext();
+ workingDir = Paths.get(appCtx.getLibraryManager().getDistributionDir().getAbsolutePath()).normalize();
+ initAuth();
+ initStorage();
+ }
+
+ protected void initAuth() {
+ sysAuthHeader = (String) ctx.get(SYS_AUTH_HEADER);
+ }
+
+ protected void initStorage() throws IOException {
+ // prepare working directory
+ if (Files.isDirectory(workingDir)) {
+ try {
+ FileUtils.cleanDirectory(workingDir.toFile());
+ } catch (IOException e) {
+ LOGGER.warn("Could not clean directory: " + workingDir, e);
+ }
+ } else {
+ Files.deleteIfExists(workingDir);
+ FileUtil.forceMkdirs(workingDir.toFile());
+ }
+ }
+
+ protected enum LibraryOperation {
+ UPSERT,
+ DELETE
+ }
+
+ protected Map<String, String> additionalHttpHeadersFromRequest(IServletRequest request) {
+ return Collections.emptyMap();
}
- void readFromFile(Path filePath, IServletResponse response, String contentType, OpenOption opt) throws Exception {
+ protected void doCreate(Namespace libNamespace, String libraryName, ExternalFunctionLanguage language, String hash,
+ URI downloadURI, boolean replaceIfExists, String sysAuthHeader, IRequestReference requestReference,
+ IServletRequest request) throws Exception {
+ INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker();
+ MessageFuture responseFuture = ncMb.registerMessageFuture();
+ CreateLibraryRequestMessage req = new CreateLibraryRequestMessage(srvCtx.getNodeId(),
+ responseFuture.getFutureId(), libNamespace, libraryName, language, hash, downloadURI, replaceIfExists,
+ sysAuthHeader, requestReference, additionalHttpHeadersFromRequest(request));
+ sendMessage(req, responseFuture);
+ }
+
+ protected void doDrop(Namespace namespace, String libraryName, boolean replaceIfExists,
+ IRequestReference requestReference, IServletRequest request) throws Exception {
+ INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker();
+ MessageFuture responseFuture = ncMb.registerMessageFuture();
+ DropLibraryRequestMessage req = new DropLibraryRequestMessage(srvCtx.getNodeId(), responseFuture.getFutureId(),
+ namespace, libraryName, replaceIfExists, requestReference, additionalHttpHeadersFromRequest(request));
+ sendMessage(req, responseFuture);
+ }
+
+ private void sendMessage(ICcAddressedMessage requestMessage, MessageFuture responseFuture) throws Exception {
+ // Running on NC -> send 'execute' message to CC
+ INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker();
+ InternalRequestResponse responseMsg;
+ try {
+ ncMb.sendMessageToPrimaryCC(requestMessage);
+ responseMsg = (InternalRequestResponse) responseFuture.get(timeout, TimeUnit.SECONDS);
+ } finally {
+ ncMb.deregisterMessageFuture(responseFuture.getFutureId());
+ }
+
+ Throwable err = responseMsg.getError();
+ if (err != null) {
+ if (err instanceof Error) {
+ throw (Error) err;
+ } else if (err instanceof Exception) {
+ throw (Exception) err;
+ } else {
+ throw new Exception(err.toString(), err);
+ }
+ }
+ }
+
+ protected void handleModification(IServletRequest request, IServletResponse response, LibraryOperation op) {
+ HttpRequest httpRequest = request.getHttpRequest();
+ Path libraryTempFile = null;
+ FileOutputStream libTmpOut = null;
+ HttpPostRequestDecoder requestDecoder = null;
+ String localPath = localPath(request);
+ try {
+ Pair<Namespace, String> namespaceAndName = decodeDvAndLibFromLocalPath(localPath);
+ String libName = namespaceAndName.second;
+ Namespace libNamespace = namespaceAndName.first;
+ DataverseName libDv = libNamespace.getDataverseName();
+ IRequestReference requestReference = receptionist.welcome(request);
+ if (op == LibraryOperation.UPSERT) {
+ requestDecoder = new HttpPostRequestDecoder(httpRequest);
+ LibraryUploadData uploadData = decodeMultiPartLibraryOptions(requestDecoder);
+ ExternalFunctionLanguage language = uploadData.type;
+ String fileExt = FilenameUtils.getExtension(uploadData.fileUpload.getFilename());
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ distributeLibrary(uploadData, libDv, libName, fileExt, language, digest, libNamespace, requestReference,
+ request);
+ } else if (op == LibraryOperation.DELETE) {
+ //DELETE semantics imply ifExists
+ doDrop(libNamespace, libName, false, requestReference, request);
+ }
+ response.setStatus(HttpResponseStatus.OK);
+ PrintWriter responseWriter = response.writer();
+ String emptyJson = "{}";
+ responseWriter.write(emptyJson);
+ responseWriter.flush();
+ } catch (Exception e) {
+ writeException(e, response);
+ LOGGER.info("Error modifying library", e);
+ } finally {
+ if (requestDecoder != null) {
+ requestDecoder.destroy();
+ }
+ try {
+ if (libraryTempFile != null) {
+ if (libTmpOut != null) {
+ libTmpOut.close();
+ }
+ Files.deleteIfExists(libraryTempFile);
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Could not delete temporary file " + libraryTempFile, e);
+ }
+ }
+ }
+
+ protected void distributeLibrary(LibraryUploadData uploadData, DataverseName libDv, String libName, String fileExt,
+ ExternalFunctionLanguage language, MessageDigest digest, Namespace namespace,
+ IRequestReference requestReference, IServletRequest request) throws Exception {
+ }
+
+ protected void readFromFile(Path filePath, IServletResponse response, String contentType, OpenOption opt)
+ throws Exception {
class InputStreamGetter extends SynchronizableWork {
private InputStream is;
@@ -132,12 +303,6 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet {
return DATAVERSE_KEY;
}
- URI createDownloadURI(Path file) throws Exception {
- String host = appCtx.getServiceContext().getAppConfig().getString(NCConfig.Option.PUBLIC_ADDRESS);
- String path = paths[0].substring(0, servletPathLengths[0]) + GET_UDF_DIST_ENDPOINT + '/' + file.getFileName();
- return new URI(httpServerProtocol.toString(), null, host, httpServerPort, path, null, null);
- }
-
private boolean isNotAttribute(InterfaceHttpData field) {
return field == null || !field.getHttpDataType().equals(InterfaceHttpData.HttpDataType.Attribute);
}
@@ -207,4 +372,85 @@ public abstract class AbstractNCUdfServlet extends AbstractServlet {
return HttpResponseStatus.INTERNAL_SERVER_ERROR;
}
+ @Override
+ protected void get(IServletRequest request, IServletResponse response) throws Exception {
+ String localPath = localPath(request);
+ try {
+ if (localPath.equals("/") || localPath.equals("")) {
+ //TODO: nicer way to get this into display form?
+ Map<Namespace, Map<String, String>> dvToLibHashes =
+ ExternalLibraryUtils.produceLibraryListing(libraryManager);
+ List<Map<String, Object>> libraryList = new ArrayList<>();
+ for (Map.Entry<Namespace, Map<String, String>> dvAndLibs : dvToLibHashes.entrySet()) {
+ for (Map.Entry<String, String> libsInDv : dvAndLibs.getValue().entrySet()) {
+ Map<String, Object> libraryEntry = new HashMap<>();
+ libraryEntry.put(getDataverseKey(), libraryManager.getNsOrDv(dvAndLibs.getKey()));
+ libraryEntry.put(NAME_KEY, libsInDv.getKey());
+ libraryEntry.put(FIELD_HASH, libsInDv.getValue());
+ libraryList.add(libraryEntry);
+ }
+ }
+ JsonNode libraryListing = OBJECT_MAPPER.valueToTree(libraryList);
+ response.setStatus(HttpResponseStatus.OK);
+ HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
+ PrintWriter responseWriter = response.writer();
+ JSONUtil.writeNode(responseWriter, libraryListing);
+ responseWriter.flush();
+ } else if (localPath(request).startsWith(GET_UDF_DIST_ENDPOINT)) {
+ localPath = localPath(request).substring(GET_UDF_DIST_ENDPOINT.length());
+ while (localPath.startsWith("/")) {
+ localPath = localPath.substring(1);
+ }
+ if (localPath.isEmpty()) {
+ response.setStatus(HttpResponseStatus.BAD_REQUEST);
+ return;
+ }
+ Path filePath = workingDir.resolve(localPath).normalize();
+ if (!filePath.startsWith(workingDir)) {
+ response.setStatus(HttpResponseStatus.BAD_REQUEST);
+ return;
+ }
+ readFromFile(filePath, response, HttpUtil.ContentType.APPLICATION_OCTET_STREAM, null);
+ } else {
+ response.setStatus(HttpResponseStatus.NOT_FOUND);
+ }
+ } catch (Exception e) {
+ writeException(e, response);
+ LOGGER.error("Error reading library", e);
+ }
+ }
+
+ protected void writeException(Exception e, IServletResponse response) {
+ response.setStatus(toHttpErrorStatus(e));
+ PrintWriter responseWriter = response.writer();
+ Map<String, String> error = Collections.singletonMap("error", e.getMessage());
+ String errorJson = "";
+ try {
+ errorJson = OBJECT_MAPPER.writeValueAsString(error);
+ } catch (JsonProcessingException ex) {
+ responseWriter.write("{ \"error\": \"Unable to process error message!\" }");
+ }
+ responseWriter.write(errorJson);
+ responseWriter.flush();
+ }
+
+ abstract protected boolean isRequestPermitted(IServletRequest request, IServletResponse response)
+ throws IOException;
+
+ protected boolean isRequestOnLoopback(IServletRequest request) {
+ if (request.getLocalAddress() != null && request.getRemoteAddress() != null) {
+ InetAddress local = request.getLocalAddress().getAddress();
+ InetAddress remote = request.getRemoteAddress().getAddress();
+ return remote.isLoopbackAddress() && local.isLoopbackAddress();
+ } else {
+ return false;
+ }
+ }
+
+ protected void rejectForbidden(IServletResponse response) throws IOException {
+ // TODO: why this JSON format, do we use this anywhere else?
+ sendError(response, HttpUtil.ContentType.APPLICATION_JSON, HttpResponseStatus.FORBIDDEN,
+ "{ \"error\": \"Forbidden\" }");
+ }
+
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index 02a99f657e..7194d5d86b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -18,245 +18,60 @@
*/
package org.apache.asterix.api.http.server;
-import static org.apache.asterix.api.http.server.ServletConstants.SYS_AUTH_HEADER;
-import static org.apache.asterix.common.library.LibraryDescriptor.FIELD_HASH;
-import static org.apache.asterix.external.library.ExternalLibraryManager.DESCRIPTOR_FILE_NAME;
-import static org.apache.asterix.external.library.ExternalLibraryManager.writeDescriptor;
-
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.net.InetAddress;
import java.net.URI;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import org.apache.asterix.app.message.CreateLibraryRequestMessage;
-import org.apache.asterix.app.message.DropLibraryRequestMessage;
-import org.apache.asterix.app.message.InternalRequestResponse;
import org.apache.asterix.common.api.IApplicationContext;
-import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.common.api.IReceptionist;
import org.apache.asterix.common.api.IRequestReference;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.library.LibraryDescriptor;
-import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
-import org.apache.asterix.common.messaging.api.INCMessageBroker;
-import org.apache.asterix.common.messaging.api.MessageFuture;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.util.ExternalLibraryUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IFileHandle;
-import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
-import org.apache.hyracks.http.server.utils.HttpUtil;
-import org.apache.hyracks.util.JSONUtil;
-import org.apache.hyracks.util.file.FileUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-
import io.netty.buffer.ByteBufInputStream;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.HttpScheme;
-import io.netty.handler.codec.http.multipart.HttpPostRequestDecoder;
public class NCUdfApiServlet extends AbstractNCUdfServlet {
- protected final IReceptionist receptionist;
-
- protected Path workingDir;
- private String sysAuthHeader;
- private ILibraryManager libraryManager;
- private int timeout;
+ private final HttpScheme httpServerProtocol;
+ private final int httpServerPort;
private static final Logger LOGGER = LogManager.getLogger();
public NCUdfApiServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx,
HttpScheme httpServerProtocol, int httpServerPort) {
- super(ctx, paths, appCtx, httpServerProtocol, httpServerPort);
- this.receptionist = appCtx.getReceptionist();
- this.timeout = appCtx.getExternalProperties().getLibraryDeployTimeout();
+ super(ctx, paths, appCtx);
+ this.httpServerProtocol = httpServerProtocol;
+ this.httpServerPort = httpServerPort;
}
- private enum LibraryOperation {
- UPSERT,
- DELETE
+ URI createDownloadURI(Path file) throws Exception {
+ String host = appCtx.getServiceContext().getAppConfig().getString(NCConfig.Option.PUBLIC_ADDRESS);
+ String path = paths[0].substring(0, servletPathLengths[0]) + GET_UDF_DIST_ENDPOINT + '/' + file.getFileName();
+ return new URI(httpServerProtocol.toString(), null, host, httpServerPort, path, null, null);
}
@Override
- public void init() throws IOException {
- appCtx = (INcApplicationContext) plainAppCtx;
- this.libraryManager = appCtx.getLibraryManager();
- srvCtx = this.appCtx.getServiceContext();
- workingDir = Paths.get(appCtx.getLibraryManager().getDistributionDir().getAbsolutePath()).normalize();
- initAuth();
- initStorage();
- }
-
- protected void initAuth() {
- sysAuthHeader = (String) ctx.get(SYS_AUTH_HEADER);
- }
-
- protected void initStorage() throws IOException {
- // prepare working directory
- if (Files.isDirectory(workingDir)) {
- try {
- FileUtils.cleanDirectory(workingDir.toFile());
- } catch (IOException e) {
- LOGGER.warn("Could not clean directory: " + workingDir, e);
- }
- } else {
- Files.deleteIfExists(workingDir);
- FileUtil.forceMkdirs(workingDir.toFile());
- }
- }
-
- protected Map<String, String> additionalHttpHeadersFromRequest(IServletRequest request) {
- return Collections.emptyMap();
- }
-
- private void doCreate(Namespace libNamespace, String libraryName, ExternalFunctionLanguage language, String hash,
- URI downloadURI, boolean replaceIfExists, String sysAuthHeader, IRequestReference requestReference,
- IServletRequest request) throws Exception {
- INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker();
- MessageFuture responseFuture = ncMb.registerMessageFuture();
- CreateLibraryRequestMessage req = new CreateLibraryRequestMessage(srvCtx.getNodeId(),
- responseFuture.getFutureId(), libNamespace, libraryName, language, hash, downloadURI, replaceIfExists,
- sysAuthHeader, requestReference, additionalHttpHeadersFromRequest(request));
- sendMessage(req, responseFuture);
- }
-
- private void doDrop(Namespace namespace, String libraryName, boolean replaceIfExists,
+ protected void distributeLibrary(LibraryUploadData uploadData, DataverseName libDv, String libName, String fileExt,
+ ExternalFunctionLanguage language, MessageDigest digest, Namespace libNamespace,
IRequestReference requestReference, IServletRequest request) throws Exception {
- INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker();
- MessageFuture responseFuture = ncMb.registerMessageFuture();
- DropLibraryRequestMessage req = new DropLibraryRequestMessage(srvCtx.getNodeId(), responseFuture.getFutureId(),
- namespace, libraryName, replaceIfExists, requestReference, additionalHttpHeadersFromRequest(request));
- sendMessage(req, responseFuture);
- }
-
- private void sendMessage(ICcAddressedMessage requestMessage, MessageFuture responseFuture) throws Exception {
- // Running on NC -> send 'execute' message to CC
- INCMessageBroker ncMb = (INCMessageBroker) srvCtx.getMessageBroker();
- InternalRequestResponse responseMsg;
- try {
- ncMb.sendMessageToPrimaryCC(requestMessage);
- responseMsg = (InternalRequestResponse) responseFuture.get(timeout, TimeUnit.SECONDS);
- } finally {
- ncMb.deregisterMessageFuture(responseFuture.getFutureId());
- }
-
- Throwable err = responseMsg.getError();
- if (err != null) {
- if (err instanceof Error) {
- throw (Error) err;
- } else if (err instanceof Exception) {
- throw (Exception) err;
- } else {
- throw new Exception(err.toString(), err);
- }
- }
- }
-
- @Override
- protected void get(IServletRequest request, IServletResponse response) throws Exception {
- String localPath = localPath(request);
- try {
- if (localPath.equals("/") || localPath.equals("")) {
- //TODO: nicer way to get this into display form?
- Map<Namespace, Map<String, String>> dvToLibHashes =
- ExternalLibraryUtils.produceLibraryListing(libraryManager);
- List<Map<String, Object>> libraryList = new ArrayList<>();
- for (Map.Entry<Namespace, Map<String, String>> dvAndLibs : dvToLibHashes.entrySet()) {
- for (Map.Entry<String, String> libsInDv : dvAndLibs.getValue().entrySet()) {
- Map<String, Object> libraryEntry = new HashMap<>();
- libraryEntry.put(getDataverseKey(), libraryManager.getNsOrDv(dvAndLibs.getKey()));
- libraryEntry.put(NAME_KEY, libsInDv.getKey());
- libraryEntry.put(FIELD_HASH, libsInDv.getValue());
- libraryList.add(libraryEntry);
- }
- }
- JsonNode libraryListing = OBJECT_MAPPER.valueToTree(libraryList);
- response.setStatus(HttpResponseStatus.OK);
- HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
- PrintWriter responseWriter = response.writer();
- JSONUtil.writeNode(responseWriter, libraryListing);
- responseWriter.flush();
- } else if (localPath(request).startsWith(GET_UDF_DIST_ENDPOINT)) {
- localPath = localPath(request).substring(GET_UDF_DIST_ENDPOINT.length());
- while (localPath.startsWith("/")) {
- localPath = localPath.substring(1);
- }
- if (localPath.isEmpty()) {
- response.setStatus(HttpResponseStatus.BAD_REQUEST);
- return;
- }
- Path filePath = workingDir.resolve(localPath).normalize();
- if (!filePath.startsWith(workingDir)) {
- response.setStatus(HttpResponseStatus.BAD_REQUEST);
- return;
- }
- readFromFile(filePath, response, HttpUtil.ContentType.APPLICATION_OCTET_STREAM, null);
- } else {
- response.setStatus(HttpResponseStatus.NOT_FOUND);
- }
- } catch (Exception e) {
- writeException(e, response);
- LOGGER.error("Error reading library", e);
- }
- }
-
- private void writeLibToCloud(LibraryUploadData uploadData, Namespace libNamespace, String libName,
- MessageDigest digest, ExternalFunctionLanguage language) throws IOException {
- FileReference libDir = libraryManager.getLibraryDir(libNamespace, libName);
- IIOManager cloudIoMgr = libraryManager.getCloudIOManager();
- FileReference lib = libDir.getChild(ILibraryManager.LIBRARY_ARCHIVE_NAME);
- if (!libDir.getFile().exists()) {
- Files.createDirectories(lib.getFile().toPath().getParent());
- }
- if (!lib.getFile().exists()) {
- Files.createFile(lib.getFile().toPath());
- }
- IFileHandle fh = cloudIoMgr.open(lib, IIOManager.FileReadWriteMode.READ_WRITE,
- IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
- WritableByteChannel outChannel = cloudIoMgr.newWritableChannel(fh);
- byte[] writeBuf = new byte[4096];
- FileReference targetDescFile = libDir.getChild(DESCRIPTOR_FILE_NAME);
- try (OutputStream outputStream = new DigestOutputStream(Channels.newOutputStream(outChannel), digest);
- InputStream ui = new ByteBufInputStream((uploadData.fileUpload).getByteBuf())) {
- IOUtils.copyLarge(ui, outputStream, writeBuf);
- outputStream.flush();
- cloudIoMgr.sync(fh, true);
- writeDescriptor(libraryManager, targetDescFile,
- new LibraryDescriptor(language, ExternalLibraryUtils.digestToHexString(digest)), true, writeBuf);
- } finally {
- cloudIoMgr.close(fh);
- }
+ URI downloadURI = cacheLibAndDistribute(uploadData, libDv, libName, fileExt, digest);
+ doCreate(libNamespace, libName, language, ExternalLibraryUtils.digestToHexString(digest), downloadURI, true,
+ getSysAuthHeader(), requestReference, request);
}
private URI cacheLibAndDistribute(LibraryUploadData uploadData, DataverseName libDv, String libName, String fileExt,
@@ -274,80 +89,11 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet {
return createDownloadURI(libraryTempFile);
}
- private void handleModification(IServletRequest request, IServletResponse response, LibraryOperation op) {
- HttpRequest httpRequest = request.getHttpRequest();
- Path libraryTempFile = null;
- FileOutputStream libTmpOut = null;
- HttpPostRequestDecoder requestDecoder = null;
- String localPath = localPath(request);
- try {
- Pair<Namespace, String> namespaceAndName = decodeDvAndLibFromLocalPath(localPath);
- String libName = namespaceAndName.second;
- Namespace libNamespace = namespaceAndName.first;
- DataverseName libDv = libNamespace.getDataverseName();
- IRequestReference requestReference = receptionist.welcome(request);
- if (op == LibraryOperation.UPSERT) {
- requestDecoder = new HttpPostRequestDecoder(httpRequest);
- LibraryUploadData uploadData = decodeMultiPartLibraryOptions(requestDecoder);
- ExternalFunctionLanguage language = uploadData.type;
- String fileExt = FilenameUtils.getExtension(uploadData.fileUpload.getFilename());
- MessageDigest digest = MessageDigest.getInstance("MD5");
- if (appCtx.isCloudDeployment()) {
- writeLibToCloud(uploadData, libNamespace, libName, digest, language);
- doCreate(libNamespace, libName, language, ExternalLibraryUtils.digestToHexString(digest), null,
- true, getSysAuthHeader(), requestReference, request);
- } else {
- URI downloadURI = cacheLibAndDistribute(uploadData, libDv, libName, fileExt, digest);
- doCreate(libNamespace, libName, language, ExternalLibraryUtils.digestToHexString(digest),
- downloadURI, true, getSysAuthHeader(), requestReference, request);
- }
- } else if (op == LibraryOperation.DELETE) {
- //DELETE semantics imply ifExists
- doDrop(libNamespace, libName, false, requestReference, request);
- }
- response.setStatus(HttpResponseStatus.OK);
- PrintWriter responseWriter = response.writer();
- String emptyJson = "{}";
- responseWriter.write(emptyJson);
- responseWriter.flush();
- } catch (Exception e) {
- writeException(e, response);
- LOGGER.info("Error modifying library", e);
- } finally {
- if (requestDecoder != null) {
- requestDecoder.destroy();
- }
- try {
- if (libraryTempFile != null) {
- if (libTmpOut != null) {
- libTmpOut.close();
- }
- Files.deleteIfExists(libraryTempFile);
- }
- } catch (IOException e) {
- LOGGER.warn("Could not delete temporary file " + libraryTempFile, e);
- }
- }
- }
-
protected String getSysAuthHeader() {
return sysAuthHeader;
}
- private void writeException(Exception e, IServletResponse response) {
- response.setStatus(toHttpErrorStatus(e));
- PrintWriter responseWriter = response.writer();
- Map<String, String> error = Collections.singletonMap("error", e.getMessage());
- String errorJson = "";
- try {
- errorJson = OBJECT_MAPPER.writeValueAsString(error);
- } catch (JsonProcessingException ex) {
- responseWriter.write("{ \"error\": \"Unable to process error message!\" }");
- }
- responseWriter.write(errorJson);
- responseWriter.flush();
- }
-
+ @Override
protected boolean isRequestPermitted(IServletRequest request, IServletResponse response) throws IOException {
if (!isRequestOnLoopback(request)) {
rejectForbidden(response);
@@ -356,22 +102,6 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet {
return true;
}
- protected boolean isRequestOnLoopback(IServletRequest request) {
- if (request.getLocalAddress() != null && request.getRemoteAddress() != null) {
- InetAddress local = request.getLocalAddress().getAddress();
- InetAddress remote = request.getRemoteAddress().getAddress();
- return remote.isLoopbackAddress() && local.isLoopbackAddress();
- } else {
- return false;
- }
- }
-
- protected void rejectForbidden(IServletResponse response) throws IOException {
- // TODO: why this JSON format, do we use this anywhere else?
- sendError(response, HttpUtil.ContentType.APPLICATION_JSON, HttpResponseStatus.FORBIDDEN,
- "{ \"error\": \"Forbidden\" }");
- }
-
@Override
protected void post(IServletRequest request, IServletResponse response) throws IOException {
if (isRequestPermitted(request, response)) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfDSApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfDSApiServlet.java
new file mode 100644
index 0000000000..d5311cedde
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfDSApiServlet.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.http.server;
+
+import static org.apache.asterix.external.library.ExternalLibraryManager.DESCRIPTOR_FILE_NAME;
+import static org.apache.asterix.external.library.ExternalLibraryManager.writeDescriptor;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Files;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.api.IRequestReference;
+import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.library.LibraryDescriptor;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
+import org.apache.asterix.external.util.ExternalLibraryUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IFileHandle;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.api.IServletResponse;
+
+import io.netty.buffer.ByteBufInputStream;
+
+public class NCUdfDSApiServlet extends AbstractNCUdfServlet {
+
+ public NCUdfDSApiServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx) {
+ super(ctx, paths, appCtx);
+ }
+
+ protected void distributeLibrary(LibraryUploadData uploadData, DataverseName libDv, String libName, String fileExt,
+ ExternalFunctionLanguage language, MessageDigest digest, Namespace namespace,
+ IRequestReference requestReference, IServletRequest request) throws Exception {
+ writeLibToCloud(uploadData, namespace, libName, digest, language);
+ doCreate(namespace, libName, language, ExternalLibraryUtils.digestToHexString(digest), null, true,
+ getSysAuthHeader(), requestReference, request);
+ }
+
+ private void writeLibToCloud(LibraryUploadData uploadData, Namespace libNamespace, String libName,
+ MessageDigest digest, ExternalFunctionLanguage language) throws IOException {
+ FileReference libDir = libraryManager.getLibraryDir(libNamespace, libName);
+ IIOManager cloudIoMgr = libraryManager.getCloudIOManager();
+ FileReference lib = libDir.getChild(ILibraryManager.LIBRARY_ARCHIVE_NAME);
+ if (!libDir.getFile().exists()) {
+ Files.createDirectories(lib.getFile().toPath().getParent());
+ }
+ if (!lib.getFile().exists()) {
+ Files.createFile(lib.getFile().toPath());
+ }
+ IFileHandle fh = cloudIoMgr.open(lib, IIOManager.FileReadWriteMode.READ_WRITE,
+ IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
+ WritableByteChannel outChannel = cloudIoMgr.newWritableChannel(fh);
+ byte[] writeBuf = new byte[4096];
+ FileReference targetDescFile = libDir.getChild(DESCRIPTOR_FILE_NAME);
+ try (OutputStream outputStream = new DigestOutputStream(Channels.newOutputStream(outChannel), digest);
+ InputStream ui = new ByteBufInputStream((uploadData.fileUpload).getByteBuf())) {
+ IOUtils.copyLarge(ui, outputStream, writeBuf);
+ outputStream.flush();
+ cloudIoMgr.sync(fh, true);
+ writeDescriptor(libraryManager, targetDescFile,
+ new LibraryDescriptor(language, ExternalLibraryUtils.digestToHexString(digest)), true, writeBuf);
+ } finally {
+ cloudIoMgr.close(fh);
+ }
+ }
+
+ protected String getSysAuthHeader() {
+ return sysAuthHeader;
+ }
+
+ @Override
+ protected boolean isRequestPermitted(IServletRequest request, IServletResponse response) throws IOException {
+ return true;
+ }
+
+ @Override
+ protected void post(IServletRequest request, IServletResponse response) throws IOException {
+ if (isRequestPermitted(request, response)) {
+ handleModification(request, response, LibraryOperation.UPSERT);
+ }
+ }
+
+ @Override
+ protected void delete(IServletRequest request, IServletResponse response) throws IOException {
+ if (isRequestPermitted(request, response)) {
+ handleModification(request, response, LibraryOperation.DELETE);
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java
index 2c29d1480e..ca8bc3bcb6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfRecoveryServlet.java
@@ -29,33 +29,33 @@ import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.utils.HttpUtil;
-import io.netty.handler.codec.http.HttpScheme;
-
public class NCUdfRecoveryServlet extends AbstractNCUdfServlet {
- ExternalLibraryManager libraryManager;
-
public static final String GET_ALL_UDF_ENDPOINT = "/all";
- public NCUdfRecoveryServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx,
- HttpScheme httpServerProtocol, int httpServerPort) {
- super(ctx, paths, appCtx, httpServerProtocol, httpServerPort);
+ public NCUdfRecoveryServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx) {
+ super(ctx, paths, appCtx);
}
@Override
public void init() {
appCtx = (INcApplicationContext) plainAppCtx;
srvCtx = this.appCtx.getServiceContext();
- this.libraryManager = (ExternalLibraryManager) appCtx.getLibraryManager();
}
@Override
protected void get(IServletRequest request, IServletResponse response) throws Exception {
String localPath = localPath(request);
if (localPath.equals(GET_ALL_UDF_ENDPOINT)) {
- Path zippedLibs = libraryManager.zipAllLibs();
+ Path zippedLibs = ((ExternalLibraryManager) libraryManager).zipAllLibs();
readFromFile(zippedLibs, response, HttpUtil.ContentType.APPLICATION_ZIP,
StandardOpenOption.DELETE_ON_CLOSE);
}
}
+
+ @Override
+ protected boolean isRequestPermitted(IServletRequest request, IServletResponse response) {
+ //gated by system auth
+ return true;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 7566206087..40a362e418 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -31,12 +31,18 @@ import static org.apache.hyracks.control.common.controllers.ControllerConfig.Opt
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.asterix.algebra.base.ILangExtension;
import org.apache.asterix.api.http.server.BasicAuthServlet;
@@ -44,6 +50,7 @@ import org.apache.asterix.api.http.server.NCQueryResultApiServlet;
import org.apache.asterix.api.http.server.NCQueryServiceServlet;
import org.apache.asterix.api.http.server.NCQueryStatusApiServlet;
import org.apache.asterix.api.http.server.NCUdfApiServlet;
+import org.apache.asterix.api.http.server.NCUdfDSApiServlet;
import org.apache.asterix.api.http.server.NCUdfRecoveryServlet;
import org.apache.asterix.api.http.server.NetDiagnosticsApiServlet;
import org.apache.asterix.api.http.server.ServletConstants;
@@ -98,6 +105,7 @@ import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.client.NodeStatus;
+import org.apache.hyracks.api.config.IApplicationConfig;
import org.apache.hyracks.api.config.IConfigManager;
import org.apache.hyracks.api.control.CcId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -131,6 +139,7 @@ public class NCApplication extends BaseNCApplication {
protected boolean startupCompleted;
protected WebManager webManager;
private HttpServer apiServer;
+ private HttpServer udfServer;
@Override
public void registerConfig(IConfigManager configManager) {
@@ -265,19 +274,38 @@ public class NCApplication extends BaseNCApplication {
apiServer.setAttribute(ServletConstants.CREDENTIAL_MAP,
parseCredentialMap(((NodeControllerService) ncServiceCtx.getControllerService()).getConfiguration()
.getCredentialFilePath()));
- Pair<Map<String, String>, Map<String, String>> auth = BasicAuthServlet.generateSysAuthHeader(apiServer.ctx());
- apiServer
- .addServlet(new BasicAuthServlet(apiServer.ctx(),
- new NCUdfApiServlet(apiServer.ctx(), new String[] { UDF }, getApplicationContext(),
- apiServer.getScheme(), apiServer.getAddress().getPort()),
- auth.getFirst(), auth.getSecond()));
- apiServer.addServlet(new BasicAuthServlet(
- apiServer.ctx(), new NCUdfRecoveryServlet(apiServer.ctx(), new String[] { UDF_RECOVERY },
- getApplicationContext(), apiServer.getScheme(), apiServer.getAddress().getPort()),
- auth.getFirst(), auth.getSecond()));
+ if (!getApplicationContext().isCloudDeployment()) {
+ Pair<Map<String, String>, Map<String, String>> auth =
+ BasicAuthServlet.generateSysAuthHeader(apiServer.ctx());
+ apiServer.addServlet((new BasicAuthServlet(apiServer.ctx(),
+ new NCUdfApiServlet(apiServer.ctx(), new String[] { UDF }, getApplicationContext(),
+ apiServer.getScheme(), externalProperties.getNcApiPort()),
+ auth.getFirst(), auth.getSecond())));
+ apiServer.addServlet(new BasicAuthServlet(apiServer.ctx(),
+ new NCUdfRecoveryServlet(apiServer.ctx(), new String[] { UDF_RECOVERY }, getApplicationContext()),
+ auth.getFirst(), auth.getSecond()));
+ } else {
+ //UDF deployment server
+ IApplicationConfig appCfg = getApplicationContext().getServiceContext().getAppConfig();
+ Path udfSockPath = Path.of(appCfg.getString(NCConfig.Option.UDF_API_DS_PATH)).toAbsolutePath();
+ if (!Files.exists(udfSockPath.getParent())) {
+ Set<PosixFilePermission> noOthers = PosixFilePermissions.fromString("rwxr-x---");
+ FileAttribute<?> permissions = PosixFilePermissions.asFileAttribute(noOthers);
+ Files.createDirectories(udfSockPath.getParent(), permissions);
+ }
+ udfServer = new HttpServer(webManager.getBosses(), webManager.getWorkers(), udfSockPath, config);
+ Pair<Map<String, String>, Map<String, String>> auth =
+ BasicAuthServlet.generateSysAuthHeader(apiServer.ctx());
+ udfServer.addServlet(new BasicAuthServlet(apiServer.ctx(),
+ new NCUdfDSApiServlet(apiServer.ctx(), new String[] { UDF }, getApplicationContext()),
+ auth.getFirst(), auth.getSecond()));
+ webManager.add(udfServer);
+
+ }
apiServer.addServlet(new NCQueryStatusApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_STATUS));
apiServer.addServlet(new NCQueryResultApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_RESULT));
webManager.add(apiServer);
+
}
protected List<AsterixExtension> getExtensions() throws Exception {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java
index b86730b07d..4cac409039 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudStorageIntegrationUtil.java
@@ -28,14 +28,14 @@ import com.adobe.testing.s3mock.testcontainers.S3MockContainer;
public class CloudStorageIntegrationUtil extends AsterixHyracksIntegrationUtil {
public static final String RESOURCES_PATH = joinPath(getProjectPath().toString(), "src", "test", "resources");
- public static final String CONFIG_FILE = joinPath(RESOURCES_PATH, "cc-cloud-storage-main.conf");
- public static final String CONFIG_FILE_TEMPLATE = joinPath(RESOURCES_PATH, "cc-cloud-storage-main.ftl");
+ public static final String TARGET_PATH = joinPath(getProjectPath().toString(), "target");
+ public static final String CONFIG_FILE = joinPath(TARGET_PATH, "cc-cloud-storage.conf");
+ public static final String CONFIG_FILE_TEMPLATE = joinPath(RESOURCES_PATH, "cc-cloud-storage.conf.ftl");
public static void main(String[] args) throws Exception {
- boolean cleanStart = Boolean.getBoolean("cleanup.start");
- LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart);
- final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
+ boolean cleanStart = true;
try (S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(cleanStart)) {
+ final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE,
CONFIG_FILE);
integrationUtil.run(cleanStart, Boolean.getBoolean("cleanup.shutdown"),
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/CloudUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/CloudUDFLibrarian.java
new file mode 100644
index 0000000000..fac50e7f93
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/CloudUDFLibrarian.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.external;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.UnixDomainSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import reactor.core.publisher.Mono;
+import reactor.netty.http.client.HttpClient;
+import reactor.netty.http.client.HttpClientResponse;
+
+@SuppressWarnings("squid:S134")
+public class CloudUDFLibrarian implements IExternalUDFLibrarian {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private String sockPath;
+ private final EventLoopGroup elg;
+
+ public CloudUDFLibrarian(String sockPath) {
+ this.sockPath = sockPath;
+ //force nio even if epoll/kqueue are on classpath
+ this.elg = new NioEventLoopGroup();
+ }
+
+ private static String createAuthHeader(Pair<String, String> credentials) {
+ String auth = credentials.first + ":" + credentials.second;
+ byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.UTF_8));
+ return "Basic " + new String(encodedAuth);
+ }
+
+ public void setCredentials(Pair<String, String> credentials) {
+ }
+
+ public void setAddress(String address) {
+ this.sockPath = address;
+ }
+
+ @Override
+ public void install(String path, String type, String libPath, Pair<String, String> credentials) throws Exception {
+ File lib = new File(libPath);
+ UnixDomainSocketAddress sockAddr = UnixDomainSocketAddress.of(sockPath);
+
+ HttpClient hc = HttpClient.create().runOn(elg)
+ .headers(h -> h.set("Authorization", createAuthHeader(credentials))).remoteAddress(() -> sockAddr);
+ hc.post().uri(path)
+ .sendForm((req, form) -> form.multipart(true).attr("type", type).file("data", lib,
+ "application/octet-stream"))
+ .responseSingle((response, content) -> handleResponse(response, content)).block();
+ }
+
+ @Override
+ public void uninstall(String path, Pair<String, String> credentials) throws IOException, AsterixException {
+ UnixDomainSocketAddress sockAddr = UnixDomainSocketAddress.of(sockPath);
+ HttpClient hc = HttpClient.create().runOn(elg)
+ .headers(h -> h.set("Authorization", createAuthHeader(credentials))).remoteAddress(() -> sockAddr);
+ hc.delete().uri(path).responseSingle((response, content) -> handleResponse(response, content)).block();
+ }
+
+ private Mono<Void> handleResponse(HttpClientResponse response, reactor.netty.ByteBufMono content) {
+ return content.asString().defaultIfEmpty("").flatMap(body -> {
+ int respCode = response.status().code();
+ if (respCode == HttpResponseStatus.OK.code()) {
+ return Mono.empty();
+ }
+
+ String errorMessage;
+ if (respCode == HttpResponseStatus.INTERNAL_SERVER_ERROR.code()
+ || respCode == HttpResponseStatus.BAD_REQUEST.code()) {
+ try {
+ errorMessage = OBJECT_MAPPER.readTree(body).get("error").asText();
+ } catch (IOException e) {
+ errorMessage = "Failed to parse error response: " + body;
+ }
+ } else {
+ errorMessage = response.status().toString();
+ }
+ return Mono.error(new AsterixException(errorMessage));
+ });
+ }
+
+ @Override
+ public SocketType getSocketType() {
+ return SocketType.DOMAIN;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index e32f8eaafc..760adf441f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -57,8 +57,8 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
}
@Override
- public void install(URI path, String type, String libPath, Pair<String, String> credentials) throws Exception {
- HttpClientContext hcCtx = createHttpClientContext(path, credentials);
+ public void install(String path, String type, String libPath, Pair<String, String> credentials) throws Exception {
+ HttpClientContext hcCtx = createHttpClientContext(URI.create(path), credentials);
HttpPost post = new HttpPost(path);
File lib = new File(libPath);
MultipartEntityBuilder entity = MultipartEntityBuilder.create().setMode(HttpMultipartMode.STRICT);
@@ -70,8 +70,8 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
}
@Override
- public void uninstall(URI path, Pair<String, String> credentials) throws IOException, AsterixException {
- HttpClientContext hcCtx = createHttpClientContext(path, credentials);
+ public void uninstall(String path, Pair<String, String> credentials) throws IOException, AsterixException {
+ HttpClientContext hcCtx = createHttpClientContext(URI.create(path), credentials);
HttpDelete del = new HttpDelete(path);
HttpResponse response = hc.execute(del, hcCtx);
handleResponse(response);
@@ -103,4 +103,9 @@ public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
throw new AsterixException(resp);
}
}
+
+ @Override
+ public SocketType getSocketType() {
+ return SocketType.LOOPBACK;
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
index 998fa78ace..b47d07a733 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
@@ -19,13 +19,19 @@
package org.apache.asterix.app.external;
import java.io.IOException;
-import java.net.URI;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.hyracks.algebricks.common.utils.Pair;
public interface IExternalUDFLibrarian {
- void install(URI path, String type, String libPath, Pair<String, String> credentials) throws Exception;
+ public enum SocketType {
+ LOOPBACK,
+ DOMAIN
+ }
- void uninstall(URI path, Pair<String, String> credentials) throws IOException, AsterixException;
+ void install(String path, String type, String libPath, Pair<String, String> credentials) throws Exception;
+
+ void uninstall(String path, Pair<String, String> credentials) throws IOException, AsterixException;
+
+ SocketType getSocketType();
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
index 00dab0eeb3..ad6f6b46a7 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
@@ -21,11 +21,13 @@ package org.apache.asterix.test.cloud_storage;
import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate;
import static org.apache.asterix.test.cloud_storage.CloudStorageTest.MOCK_SERVER_HOSTNAME_FRAGMENT;
import static org.apache.asterix.test.runtime.ExternalPythonFunctionIT.setNcEndpoints;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.util.Collection;
import java.util.List;
import org.apache.asterix.api.common.LocalCloudUtilAdobeMock;
+import org.apache.asterix.app.external.CloudUDFLibrarian;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.test.common.TestConstants;
import org.apache.asterix.test.common.TestExecutor;
@@ -48,7 +50,7 @@ import org.junit.runners.Parameterized.Parameters;
import com.adobe.testing.s3mock.testcontainers.S3MockContainer;
/**
- * Run tests in cloud deployment environment
+ * Run Python UDF tests in cloud deployment environment
*/
@RunWith(Parameterized.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@@ -63,6 +65,7 @@ public class CloudPythonTest {
public static final String CONFIG_FILE_TEMPLATE = "src/test/resources/cc-cloud-storage.conf.ftl";
public static final String CONFIG_FILE = "target/cc-cloud-storage.conf";
private static final String EXCLUDED_TESTS = "MP";
+ private static final String dsPath = joinPath("/", "tmp", "asterixdb_udf", "asterix_nc1_udf.sock");
public CloudPythonTest(TestCaseContext tcCtx) {
this.tcCtx = tcCtx;
@@ -78,7 +81,7 @@ public class CloudPythonTest {
TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH);
testExecutor.executorId = "cloud";
testExecutor.stripSubstring = "//DB:";
- LangExecutionUtil.setUp(CONFIG_FILE, testExecutor);
+ LangExecutionUtil.setUp(CONFIG_FILE, testExecutor, false, false, new CloudUDFLibrarian(dsPath));
setNcEndpoints(testExecutor);
System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE);
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 81ad57a769..bb2aa34c5f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -1475,7 +1475,13 @@ public class TestExecutor {
//TODO: this is not right. URLEncoder does not properly encode paths.
String dataverse = URLEncoder.encode(command[1], StandardCharsets.US_ASCII.name());
String library = URLEncoder.encode(command[2], StandardCharsets.US_ASCII.name());
- URI path = createEndpointURI("/admin/udf/" + dataverse + "/" + library);
+ String basePath = "/admin/udf/" + dataverse + "/" + library;
+ String path = "";
+ switch (librarian.getSocketType()){
+ case DOMAIN -> path = basePath;
+ case LOOPBACK -> path = createEndpointURI(basePath).toString();
+ default -> path = createEndpointURI(basePath).toString();
+ }
if (command.length < 2) {
throw new Exception("invalid library command: " + line);
}
@@ -2742,6 +2748,11 @@ public class TestExecutor {
return uri;
}
+ protected URI createUDFDomainSockURI(String pathAndQuery) {
+ URI uri = URI.create("unix://target/tmp/asterix_nc1/udf.sock" + pathAndQuery);
+ return uri;
+ }
+
public URI getEndpoint(String servlet) throws URISyntaxException {
return createEndpointURI(Servlets.getAbsolutePath(getPath(servlet)));
}
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf
index 8745d9d7fd..9efabaa036 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-main.conf
@@ -21,6 +21,7 @@ core.dump.dir=target/tmp/asterix_nc1/coredump
iodevices=target/tmp/asterix_nc1/iodevice1
iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
nc.api.port=19004
+udf.api.ds.path=target/tmp/asterix_nc1/udf.sock
#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
[nc/asterix_nc2]
@@ -29,6 +30,7 @@ txn.log.dir=target/tmp/asterix_nc2/txnlog
core.dump.dir=target/tmp/asterix_nc2/coredump
iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
nc.api.port=19005
+udf.api.ds.path=target/tmp/asterix_nc2/udf.sock
#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
[nc]
@@ -68,7 +70,7 @@ storage.partitioning=static
cloud.storage.scheme=s3
cloud.storage.bucket=cloud-storage-container
cloud.storage.region=us-west-2
-cloud.storage.endpoint=http://127.0.0.1:8001
+cloud.storage.endpoint=http://127.0.0.1:46677
cloud.storage.anonymous.auth=true
cloud.storage.cache.policy=selective
cloud.max.write.requests.per.second=2000
diff --git a/asterixdb/asterix-doc/src/main/user-defined_function/udf.md b/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
index 71e0fa2e44..9768c81cd5 100644
--- a/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
+++ b/asterixdb/asterix-doc/src/main/user-defined_function/udf.md
@@ -52,7 +52,7 @@ Now,restart the cluster if it was already started to allow the Cluster Controlle
## <a name="installingUDF">Installing a Java UDF Library</a>
To install a UDF package to the cluster, we need to send a Multipart Form-data HTTP request to the `/admin/udf` endpoint
-of the CC at the normal API port (`19004` by default). Any suitable tool will do, but for the example here I will use
+of the cluster through the special UDF domain socket. Any suitable tool will do, but for the example here I will use
`curl` which is widely available.
For example, to install a library with the following criteria:
@@ -65,7 +65,7 @@ For example, to install a library with the following criteria:
we would execute
- curl -v -u admin:admin -X POST -F 'data=@./lib.zip' -F 'type=java' localhost:19004/admin/udf/udfs/testlib
+ curl -v -u admin:admin --unix-socket udf.sock -F 'data=@./lib.zip' -F 'type=java' localhost/admin/udf/udfs/testlib
Any response other than `200` indicates an error in deployment.
diff --git a/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanUDFLibrarian.java b/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanUDFLibrarian.java
index 025f607a30..9a9bd85ebe 100644
--- a/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanUDFLibrarian.java
+++ b/asterixdb/asterix-podman/src/test/java/org/apache/asterix/test/podman/PodmanUDFLibrarian.java
@@ -19,7 +19,6 @@
package org.apache.asterix.test.podman;
import java.io.IOException;
-import java.net.URI;
import org.apache.asterix.app.external.IExternalUDFLibrarian;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -42,7 +41,7 @@ public class PodmanUDFLibrarian implements IExternalUDFLibrarian {
}
@Override
- public void install(URI path, String type, String libPath, Pair<String, String> credentials) throws Exception {
+ public void install(String path, String type, String libPath, Pair<String, String> credentials) throws Exception {
Container.ExecResult curlResult = null;
int retryCt = 0;
while (retryCt < 10) {
@@ -50,7 +49,7 @@ public class PodmanUDFLibrarian implements IExternalUDFLibrarian {
curlResult = asterix.execInContainer("curl", "--no-progress-meter", "-X", "POST", "-u",
credentials.first + ":" + credentials.second, "-F",
"data=@" + "/var/tmp/asterix-app/" + libPath, "-F", "type=" + type,
- "http://localhost:19004" + path.getRawPath());
+ "http://localhost:19004" + path);
handleResponse(curlResult);
return;
} catch (RuntimeException e) {
@@ -62,16 +61,21 @@ public class PodmanUDFLibrarian implements IExternalUDFLibrarian {
}
@Override
- public void uninstall(URI path, Pair<String, String> credentials) throws IOException, AsterixException {
+ public void uninstall(String path, Pair<String, String> credentials) throws IOException, AsterixException {
try {
Container.ExecResult curlResult = asterix.execInContainer("curl", "-X", "DELETE", "-u",
- credentials.first + ":" + credentials.second, "http://localhost:19004" + path.getPath());
+ credentials.first + ":" + credentials.second, "http://localhost:19004" + path);
handleResponse(curlResult);
} catch (InterruptedException e) {
throw new IOException(e);
}
}
+ @Override
+ public SocketType getSocketType() {
+ return SocketType.LOOPBACK;
+ }
+
private void handleResponse(Container.ExecResult result) throws AsterixException, JsonProcessingException {
if (result.getExitCode() != 0) {
throw new AsterixException(result.getStderr());
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index f55dd59c79..c3571934c6 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -27,6 +27,7 @@ import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTE
import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_LONG_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING_ARRAY;
+import static org.apache.hyracks.control.common.utils.ConfigurationUtil.JAVA_IO_TMPDIR;
import java.net.InetAddress;
import java.util.ArrayList;
@@ -103,6 +104,11 @@ public class NCConfig extends ControllerConfig {
PYTHON_ARGS(STRING_ARRAY, (String[]) null),
PYTHON_ENV(STRING_ARRAY, (String[]) null),
PYTHON_DS_PATH(STRING, (String) null),
+ UDF_API_DS_PATH(
+ STRING,
+ appConfig -> FileUtil.joinPath(System.getProperty(JAVA_IO_TMPDIR), "asterixdb_udf",
+ appConfig.getString(NODE_ID) + "_udf.sock"),
+ "Defaults to temporary directory/asterixdb_udf/{$NODE_ID}_udf.sock. Path must be <100 characters"),
LIBRARY_MAX_FILE_SIZE(POSITIVE_LONG_BYTE_UNIT, 250L * 1024 * 1024), //250MB
LIBRARY_MAX_EXTRACTED_SIZE(POSITIVE_LONG_BYTE_UNIT, 1000L * 1024 * 1024), //1GB
LIBRARY_MAX_ARCHIVE_ENTRIES(INTEGER, 4096),
@@ -244,6 +250,8 @@ public class NCConfig extends ControllerConfig {
return "Number of threads per partition used to write and read from storage";
case IO_QUEUE_SIZE:
return "Length of the queue used for requests to write and read";
+ case UDF_API_DS_PATH:
+ return "Path and name for domain socket used to deploy UDFs to the cluster";
case PYTHON_CMD:
return "Absolute path to python interpreter";
case PYTHON_ADDITIONAL_PACKAGES:
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletRequest.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletRequest.java
index 59c0ae05f2..faab5d30cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletRequest.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletRequest.java
@@ -114,5 +114,9 @@ public interface IServletRequest {
*/
InetSocketAddress getLocalAddress();
+ String getHostPort();
+
+ String getRemotePort();
+
Channel getChannel();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/BaseRequest.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/BaseRequest.java
index 4a72d539b4..0aaf8feaf9 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/BaseRequest.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/BaseRequest.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.http.server;
import java.net.InetSocketAddress;
+import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -28,6 +29,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.util.NetworkUtil;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
@@ -114,6 +116,23 @@ public class BaseRequest implements IServletRequest {
return (InetSocketAddress) channel.localAddress();
}
+ @Override
+ public String getHostPort() {
+ return getPort(channel.localAddress());
+ }
+
+ @Override
+ public String getRemotePort() {
+ return getPort(channel.remoteAddress());
+ }
+
+ private String getPort(SocketAddress s) {
+ if (s instanceof InetSocketAddress) {
+ return NetworkUtil.toHostPort((InetSocketAddress) channel.localAddress());
+ }
+ return "0";
+ }
+
@Override
public Channel getChannel() {
return channel;
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
index 6476ab9e3d..16a29aa4af 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
@@ -30,6 +30,8 @@ import org.apache.logging.log4j.Logger;
import io.netty.channel.ChannelDuplexHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPromise;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioDomainSocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultFullHttpResponse;
import io.netty.handler.codec.http.DefaultHttpContent;
@@ -71,7 +73,11 @@ public class CLFLogger extends ChannelDuplexHandler {
if (msg instanceof HttpRequest) {
HttpRequest req = (HttpRequest) msg;
try {
- clientIp = ((NioSocketChannel) ctx.channel()).remoteAddress().getAddress().toString().substring(1);
+ if (ctx.channel() instanceof SocketChannel) {
+ clientIp = ((NioSocketChannel) ctx.channel()).remoteAddress().getAddress().toString().substring(1);
+ } else if (ctx.channel() instanceof NioDomainSocketChannel) {
+ clientIp = ctx.channel().remoteAddress().toString();
+ }
} catch (Exception e) {
LOGGER.debug("ignoring {} obtaining client ip for {}", e, ctx.channel());
clientIp = "-";
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 1198945eb9..4cc8bc9ae7 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -20,6 +20,10 @@ package org.apache.hyracks.http.server;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.UnixDomainSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -52,8 +56,11 @@ import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.FixedRecvByteBufAllocator;
+import io.netty.channel.ServerChannel;
import io.netty.channel.WriteBufferWaterMark;
+import io.netty.channel.socket.DuplexChannel;
import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerDomainSocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpScheme;
@@ -70,6 +77,7 @@ public class HttpServer {
new WriteBufferWaterMark(LOW_WRITE_BUFFER_WATER_MARK, HIGH_WRITE_BUFFER_WATER_MARK);
protected static final int RECEIVE_BUFFER_SIZE = 4096;
private static final Logger LOGGER = LogManager.getLogger();
+ private static final Class<InetSocketAddress> INSA = InetSocketAddress.class;
private static final int FAILED = -1;
private static final int STOPPED = 0;
private static final int STARTING = 1;
@@ -85,7 +93,8 @@ public class HttpServer {
private final ServletRegistry servlets;
private final EventLoopGroup bossGroup;
private final EventLoopGroup workerGroup;
- private final Set<InetSocketAddress> addresses;
+ private final Set<SocketAddress> addresses;
+ private final Set<SocketAddress> boundAddresses;
private final ThreadPoolExecutor executor;
// Mutable members
private volatile int state = STOPPED;
@@ -93,6 +102,7 @@ public class HttpServer {
private final List<Channel> channels;
private Throwable cause;
private HttpServerConfig config;
+ private Class<? extends ServerChannel> channelImpl = NioServerSocketChannel.class;
private final GenericFutureListener<Future<Void>> channelCloseListener = f -> {
// This listener is invoked from within a netty IO thread. Hence, we can never block it
@@ -118,14 +128,30 @@ public class HttpServer {
this(bossGroup, workerGroup, Collections.singletonList(address), config, closeHandler);
}
- public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, Collection<InetSocketAddress> addresses,
- HttpServerConfig config, IChannelClosedHandler closeHandler) {
+ public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, Path path, HttpServerConfig config) {
+ this(bossGroup, workerGroup, Collections.singletonList(UnixDomainSocketAddress.of(path)), config, null);
+ }
+
+ public static String getPortOrPath(SocketAddress addr) {
+ if (addr instanceof InetSocketAddress) {
+ return "port:" + ((InetSocketAddress) addr).getPort();
+ } else if (addr instanceof UnixDomainSocketAddress) {
+ return "path:" + ((UnixDomainSocketAddress) addr).getPath().toString();
+ } else {
+ return addr.toString();
+ }
+ }
+
+ public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup,
+ Collection<? extends SocketAddress> addresses, HttpServerConfig config,
+ IChannelClosedHandler closeHandler) {
if (addresses.isEmpty()) {
throw new IllegalArgumentException("no addresses specified");
}
this.bossGroup = bossGroup;
this.workerGroup = workerGroup;
this.addresses = new LinkedHashSet<>(addresses);
+ this.boundAddresses = new LinkedHashSet<>(addresses);
this.closedHandler = closeHandler;
this.config = config;
channels = new ArrayList<>();
@@ -133,13 +159,21 @@ public class HttpServer {
servlets = new ServletRegistry();
workQueue = new LinkedBlockingQueue<>(config.getRequestQueueSize());
int numExecutorThreads = config.getThreadCount();
- int[] ports = this.addresses.stream().mapToInt(InetSocketAddress::getPort).distinct().toArray();
+ int[] ports = addresses.stream().filter(INSA::isInstance).map(INSA::cast).mapToInt(InetSocketAddress::getPort)
+ .distinct().toArray();
String desc;
if (ports.length > 1) {
- desc = this.addresses.stream().map(a -> a.getAddress().getHostAddress() + ":" + a.getPort())
+ desc = addresses.stream().filter(INSA::isInstance).map(INSA::cast)
+ .map(a -> a.getAddress().getHostAddress() + ":" + a.getPort())
.collect(Collectors.joining(",", "[", "]"));
- } else {
+ } else if (ports.length == 1) {
desc = "port:" + ports[0];
+ } else {
+ List<String> paths = addresses.stream().filter(UnixDomainSocketAddress.class::isInstance)
+ .map(UnixDomainSocketAddress.class::cast).map(UnixDomainSocketAddress::getPath)
+ .map(Object::toString).distinct().toList();
+ desc = "paths: " + " [ " + String.join(" ],[ ", paths) + " ]";
+ channelImpl = NioServerDomainSocketChannel.class;
}
executor = new ThreadPoolExecutor(numExecutorThreads, numExecutorThreads, 0L, TimeUnit.MILLISECONDS, workQueue,
runnable -> new Thread(runnable, "HttpExecutor(" + desc + ")-" + threadId.getAndIncrement()));
@@ -278,18 +312,31 @@ public class HttpServer {
private void bind() throws Exception {
ServerBootstrap b = new ServerBootstrap();
- b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class)
+ b.group(bossGroup, workerGroup).channel(channelImpl)
.childOption(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(RECEIVE_BUFFER_SIZE))
.childOption(ChannelOption.AUTO_READ, Boolean.FALSE)
.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
.childOption(ChannelOption.WRITE_BUFFER_WATER_MARK, WRITE_BUFFER_WATER_MARK)
.handler(new LoggingHandler(LogLevel.DEBUG)).childHandler(getChannelInitializer());
- List<Pair<InetSocketAddress, ChannelFuture>> channelFutures = new ArrayList<>();
- for (InetSocketAddress address : addresses) {
+ List<Pair<SocketAddress, ChannelFuture>> channelFutures = new ArrayList<>();
+ for (SocketAddress address : addresses) {
+ if(address instanceof UnixDomainSocketAddress udr){
+ Path udrPath = udr.getPath();
+ //if the socket can't be connected to, but:
+ // - the node exists
+ // - it is not a regular file
+ // - it is not a directory
+ // - and is 0 length
+ // then it should be safe to unlink
+ if(!sockAlive(udr) && Files.exists(udrPath) && !Files.isRegularFile(udrPath) && !Files.isDirectory(udrPath) && udrPath.toFile().length() == 0){
+ Files.deleteIfExists(udrPath);
+ }
+ }
channelFutures.add(org.apache.commons.lang3.tuple.Pair.of(address, b.bind(address)));
+ boundAddresses.add(address);
}
Exception failure = null;
- for (Pair<InetSocketAddress, ChannelFuture> addressFuture : channelFutures) {
+ for (Pair<SocketAddress, ChannelFuture> addressFuture : channelFutures) {
try {
Channel channel = addressFuture.getRight().sync().channel();
channel.closeFuture().addListener(channelCloseListener);
@@ -384,6 +431,7 @@ public class HttpServer {
LOGGER.log(Level.ERROR, "Error while shutting down http server executor", e);
}
closeChannels();
+ deleteDomainSockFiles();
}
public IServlet getServlet(FullHttpRequest request) {
@@ -394,7 +442,7 @@ public class HttpServer {
return new HttpServerHandler<>(this, chunkSize);
}
- protected ChannelInitializer<SocketChannel> getChannelInitializer() {
+ protected ChannelInitializer<DuplexChannel> getChannelInitializer() {
return new HttpServerInitializer(this);
}
@@ -429,7 +477,7 @@ public class HttpServer {
}
@Deprecated // this returns an arbitrary (the first supplied if collection is ordered) address
- public InetSocketAddress getAddress() {
+ public SocketAddress getAddress() {
return addresses.iterator().next();
}
@@ -443,4 +491,27 @@ public class HttpServer {
channels.clear();
}
}
+
+ // Domain socket nodes exist outside the lifetime of the socket itself, so we should attempt to remove them on close
+ void deleteDomainSockFiles() {
+ for (UnixDomainSocketAddress address : boundAddresses.stream().filter(UnixDomainSocketAddress.class::isInstance)
+ .map(UnixDomainSocketAddress.class::cast).toList()) {
+ try {
+ Files.deleteIfExists(address.getPath());
+ LOGGER.info("Deleted domain socket file {}", address.getPath());
+ } catch (IOException e) {
+ LOGGER.warn("Failed to delete domain socket file {}", address.getPath(), e);
+ }
+ }
+ }
+
+ //the only real way to know if the file given to us in the config is an active socket is to attempt to connect to it
+ boolean sockAlive(UnixDomainSocketAddress addr) {
+ try (var chan = java.nio.channels.SocketChannel.open(addr)) {
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
index ad8a61f4a0..32b1dbbd22 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
@@ -20,11 +20,11 @@ package org.apache.hyracks.http.server;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
-import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.DuplexChannel;
import io.netty.handler.codec.http.HttpRequestDecoder;
import io.netty.handler.codec.http.HttpResponseEncoder;
-public class HttpServerInitializer extends ChannelInitializer<SocketChannel> {
+public class HttpServerInitializer extends ChannelInitializer<DuplexChannel> {
private final HttpServer server;
private final int maxRequestSize;
@@ -44,7 +44,7 @@ public class HttpServerInitializer extends ChannelInitializer<SocketChannel> {
}
@Override
- public void initChannel(SocketChannel ch) {
+ public void initChannel(DuplexChannel ch) {
ChannelPipeline p = ch.pipeline();
p.addLast(new HttpRequestCapacityController(server));
p.addLast(new HttpRequestDecoder(maxRequestInitialLineLength, maxRequestHeaderSize, maxRequestChunkSize));
(asterixdb) 04/18: [ASTERIXDB-3677][COMP] Fix nested star operators failure
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1933dfca4b4292c1994dbe88092470605e0ea488
Author: preetham0202 <pr...@couchbase.com>
AuthorDate: Wed Jan 14 18:14:06 2026 +0530
[ASTERIXDB-3677][COMP] Fix nested star operators failure
- user model changes: no
- storage format changes: no
- interface changes: no
Ext-ref: MB-70076
Details: Do not extract record-constructor() functions so that nested star operators don't fail.
Change-Id: I557b9992d047c6fc53bddd5b90ff814bb678cedf
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20864
Tested-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20794
Reviewed-by: Preetham Poluparthi <pr...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../translator/LangExpressionToPlanTranslator.java | 3 +
.../SqlppExpressionToPlanTranslator.java | 5 +-
.../ASTERIXDB-3682-field-access-in-join.plan | 590 ++++++++++-----------
.../select-star/nested/nested.1.ddl.sqlpp | 32 ++
.../select-star/nested/nested.2.update.sqlpp | 25 +
.../select-star/nested/nested.3.query.sqlpp | 22 +
.../results/select-star/nested/nested.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +
.../DoNotExtractExpressionAnnotation.java | 27 +
.../rules/ExtractCommonExpressionsRule.java | 11 +-
10 files changed, 424 insertions(+), 298 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index b1638d4e71..fee6816ed1 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -1482,6 +1482,9 @@ abstract class LangExpressionToPlanTranslator
throws CompilationException {
AbstractFunctionCallExpression f = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR));
+ if (rc.hasHints()) {
+ f.putAnnotations(rc.getHints());
+ }
f.setSourceLocation(rc.getSourceLocation());
LogicalVariable v1 = context.newVar();
AssignOperator a = new AssignOperator(v1, new MutableObject<>(f));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
index 02f1f0e9f8..d813a4c5a2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -114,6 +114,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCa
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.DoNotExtractExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
@@ -916,8 +917,10 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
CallExpr toObjectExpr = new CallExpr(new FunctionSignature(BuiltinFunctions.TO_OBJECT),
Collections.singletonList(projectionExpr));
toObjectExpr.setSourceLocation(sourceLoc);
+ RecordConstructor recordConstructor = new RecordConstructor(Collections.emptyList());
+ recordConstructor.addHint(DoNotExtractExpressionAnnotation.INSTANCE);
CallExpr ifMissingOrNullExpr = new CallExpr(new FunctionSignature(BuiltinFunctions.IF_MISSING_OR_NULL),
- Arrays.asList(toObjectExpr, new RecordConstructor(Collections.emptyList())));
+ Arrays.asList(toObjectExpr, recordConstructor));
ifMissingOrNullExpr.setSourceLocation(sourceLoc);
return ifMissingOrNullExpr;
}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan
index a44e1cb635..be7926f2c9 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan
@@ -4,327 +4,325 @@ distribute result [$$446]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
project ([$$446])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$446] <- [if-missing-or-null(to-object($$593), cast($$456))]
+ assign [$$446] <- [if-missing-or-null(to-object($$592), cast({ }))]
-- ASSIGN |PARTITIONED|
- project ([$$456, $$593])
+ project ([$$592])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$529(ASC), $$530(ASC), $$531(ASC), $$532(ASC) ] |PARTITIONED|
- order (ASC, $$529) (ASC, $$530) (ASC, $$531) (ASC, $$532)
- -- STABLE_SORT [$$529(ASC), $$530(ASC), $$531(ASC), $$532(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$528(ASC), $$529(ASC), $$530(ASC), $$531(ASC) ] |PARTITIONED|
+ order (ASC, $$528) (ASC, $$529) (ASC, $$530) (ASC, $$531)
+ -- STABLE_SORT [$$528(ASC), $$529(ASC), $$530(ASC), $$531(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$529, $$530, $$531, $$532] <- [$$593.getField("join_key2"), $$593.getField("wsc"), $$593.getField("lcd"), $$593.getField("wsd")]
+ assign [$$528, $$529, $$530, $$531] <- [$$592.getField("join_key2"), $$592.getField("wsc"), $$592.getField("lcd"), $$592.getField("wsd")]
-- ASSIGN |PARTITIONED|
- project ([$$456, $$593])
+ project ([$$592])
-- STREAM_PROJECT |PARTITIONED|
select (and(like($$455, "%"), not(is-missing($$455))))
-- STREAM_SELECT |PARTITIONED|
- project ([$$456, $$593, $$455])
+ project ([$$592, $$455])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$593, $$455] <- [object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$603, "wsm": $$604, "wsc": $$513, "scKey": $$602, "wpdi": $$509, "bch": $$525, "omaa": $$526, "bco": uppercase(substring($$525, 0, position($$525, "_")))}), object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$513, "scKey": $$602, "wpdi": $$5 [...]
+ assign [$$592, $$455] <- [object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$602, "wsm": $$603, "wsc": $$512, "scKey": $$601, "wpdi": $$508, "bch": $$524, "omaa": $$525, "bco": uppercase(substring($$524, 0, position($$524, "_")))}), object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$512, "scKey": $$601, "wpdi": $$508 [...]
-- ASSIGN |PARTITIONED|
- select (and(like(object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$388.getField("wsc"), "scKey": $$602, "wpdi": $$509, "bch": $$525, "omaa": $$526, "bco": uppercase(substring($$525, 0, position($$525, "_")))}).getField("bch"), "%"), like(object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$388.getField(3), "wsm": [...]
+ select (and(like(object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$388.getField("wsc"), "scKey": $$601, "wpdi": $$508, "bch": $$524, "omaa": $$525, "bco": uppercase(substring($$524, 0, position($$524, "_")))}).getField("bch"), "%"), like(object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$388.getField(3), "wsm": $$ [...]
-- STREAM_SELECT |PARTITIONED|
- project ([$$323, $$466, $$603, $$604, $$513, $$602, $$509, $$388, $$456, $$525, $$526])
+ project ([$$323, $$465, $$602, $$603, $$512, $$601, $$508, $$388, $$524, $$525])
-- STREAM_PROJECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$476, $$509))
- -- HYBRID_HASH_JOIN [$$509][$$476] |PARTITIONED|
+ join (eq($$475, $$508))
+ -- HYBRID_HASH_JOIN [$$508][$$475] |PARTITIONED|
exchange
- -- HASH_PARTITION_EXCHANGE [$$509] |PARTITIONED|
- assign [$$456] <- [{ }]
- -- ASSIGN |PARTITIONED|
- project ([$$323, $$466, $$603, $$604, $$513, $$602, $$509, $$388])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (and(eq($$599, $$601), eq($$466, $$602)))
- -- HYBRID_HASH_JOIN [$$599, $$466][$$601, $$602] |PARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$599, $$466] |PARTITIONED|
- project ([$$323, $$466, $$599])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$uniqueID, $$489))
- -- HYBRID_HASH_JOIN [$$489][$$uniqueID] |PARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$489] |PARTITIONED|
- project ([$$323, $$599, $$489])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$323] <- [{"join_key2": $$599, "lov13_wsx1": substring($$465, 0, 7), "lcd": substring($$465, 0, 10), "lov13_name": $$467, "lov13_uunn": $$489, "lov13_t0_dd": $$600, "lov13_t0_ff": substring($$600, 0, 19)}]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$467, $$599, $$600])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$600] <- [to-string($$u_val.getField("t0"))]
- -- ASSIGN |PARTITIONED|
- select (not(is-null($$599)))
- -- STREAM_SELECT |PARTITIONED|
- project ([$$465, $$489, $$467, $$u_val, $$599])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$599] <- [substring(if-missing-or-null(if-missing-or-null($$461.getField("someId1"), $$461.getField("someId2")), $$461.getField("someId3")), 0, 38)]
+ -- HASH_PARTITION_EXCHANGE [$$508] |PARTITIONED|
+ project ([$$323, $$465, $$602, $$603, $$512, $$601, $$508, $$388])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (and(eq($$598, $$600), eq($$465, $$601)))
+ -- HYBRID_HASH_JOIN [$$598, $$465][$$600, $$601] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$598, $$465] |PARTITIONED|
+ project ([$$323, $$465, $$598])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$uniqueID, $$488))
+ -- HYBRID_HASH_JOIN [$$488][$$uniqueID] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$488] |PARTITIONED|
+ project ([$$323, $$598, $$488])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$323] <- [{"join_key2": $$598, "lov13_wsx1": substring($$464, 0, 7), "lcd": substring($$464, 0, 10), "lov13_name": $$466, "lov13_uunn": $$488, "lov13_t0_dd": $$599, "lov13_t0_ff": substring($$599, 0, 19)}]
+ -- ASSIGN |PARTITIONED|
+ project ([$$464, $$488, $$466, $$598, $$599])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$599] <- [to-string($$u_val.getField("t0"))]
+ -- ASSIGN |PARTITIONED|
+ select (not(is-null($$598)))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$464, $$488, $$466, $$u_val, $$598])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$598] <- [substring(if-missing-or-null(if-missing-or-null($$460.getField("someId1"), $$460.getField("someId2")), $$460.getField("someId3")), 0, 38)]
+ -- ASSIGN |PARTITIONED|
+ assign [$$460] <- [$$u_val.getField("marop")]
-- ASSIGN |PARTITIONED|
- assign [$$461] <- [$$u_val.getField("marop")]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$467, $$u_val])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$u_val <- scan-collection($$290)
- -- UNNEST |PARTITIONED|
- project ([$$465, $$489, $$467, $$290])
- -- STREAM_PROJECT |PARTITIONED|
- subplan {
- aggregate [$$290] <- [listify($$289)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- assign [$$289] <- [{"marop": $$468, "t0": $$488}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- select (and(eq($$487, "iap"), not(is-unknown($$468)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- STREAM_SELECT |LOCAL|
- assign [$$488, $$468, $$487] <- [$$item2.getField("t0"), $$item2.getField("marop"), $$item2.getField("ua")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- unnest $$item2 <- scan-collection($$486) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- UNNEST |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- SUBPLAN |PARTITIONED|
- select (not(or(eq($$467, "p1"), eq($$467, "p2"), eq($$467, "p3"))))
- -- STREAM_SELECT |PARTITIONED|
- project ([$$465, $$489, $$486, $$467])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$470, $$275))
- -- HYBRID_HASH_JOIN [$$275][$$470] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$275] <- [get-item(split($$467, " "), 0)]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$486, $$467])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$486, $$467] <- [$$item.getField("value"), $$item.getField("name")]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$item])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$item <- scan-collection(object-pairs($$477))
- -- UNNEST |PARTITIONED|
- project ([$$465, $$489, $$477])
- -- STREAM_PROJECT |PARTITIONED|
- select ($$262)
- -- STREAM_SELECT |PARTITIONED|
- project ([$$262, $$465, $$489, $$477])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- group by ([$$551 := $$457]) decor ([$$465; $$489; $$477]) {
- aggregate [$$262] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- select (not(is-missing($$550))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- STREAM_SELECT |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- PRE_CLUSTERED_GROUP_BY[$$457] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$489, $$465, $$477, $$550, $$457])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- left outer join (eq($$457, $$549))
- -- HYBRID_HASH_JOIN [$$457][$$549] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$489, $$465, $$477, $$457])
- -- STREAM_PROJECT |PARTITIONED|
- select (and(eq($$b.getField("t"), "lovStat"), not(and(eq(len($$597), 1), eq(get-item($$597, 0), "some_str")))))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$597] <- [object-names($$477)]
+ project ([$$464, $$488, $$466, $$u_val])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$u_val <- scan-collection($$290)
+ -- UNNEST |PARTITIONED|
+ project ([$$464, $$488, $$466, $$290])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$290] <- [listify($$289)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ assign [$$289] <- [{"marop": $$467, "t0": $$487}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ select (and(eq($$486, "iap"), not(is-unknown($$467)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |LOCAL|
+ assign [$$487, $$467, $$486] <- [$$item2.getField("t0"), $$item2.getField("marop"), $$item2.getField("ua")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ unnest $$item2 <- scan-collection($$485) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNNEST |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ select (not(or(eq($$466, "p1"), eq($$466, "p2"), eq($$466, "p3"))))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$464, $$488, $$485, $$466])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$469, $$275))
+ -- HYBRID_HASH_JOIN [$$275][$$469] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$275] <- [get-item(split($$466, " "), 0)]
+ -- ASSIGN |PARTITIONED|
+ project ([$$464, $$488, $$485, $$466])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$485, $$466] <- [$$item.getField("value"), $$item.getField("name")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$464, $$488, $$item])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$item <- scan-collection(object-pairs($$476))
+ -- UNNEST |PARTITIONED|
+ project ([$$464, $$488, $$476])
+ -- STREAM_PROJECT |PARTITIONED|
+ select ($$262)
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$262, $$464, $$488, $$476])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ group by ([$$550 := $$456]) decor ([$$464; $$488; $$476]) {
+ aggregate [$$262] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ select (not(is-missing($$549))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- PRE_CLUSTERED_GROUP_BY[$$456] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$488, $$464, $$476, $$549, $$456])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ left outer join (eq($$456, $$548))
+ -- HYBRID_HASH_JOIN [$$456][$$548] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$488, $$464, $$476, $$456])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (and(eq($$b.getField("t"), "lovStat"), not(and(eq(len($$596), 1), eq(get-item($$596, 0), "some_str")))))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$596] <- [object-names($$476)]
+ -- ASSIGN |PARTITIONED|
+ assign [$$488, $$464, $$476] <- [$$b.getField("uunn"), $$b.getField("qwer"), $$b.getField("ihsaa")]
-- ASSIGN |PARTITIONED|
- assign [$$489, $$465, $$477] <- [$$b.getField("uunn"), $$b.getField("qwer"), $$b.getField("ihsaa")]
- -- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$457, $$b] <- test.col2
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$550] <- [true]
- -- ASSIGN |PARTITIONED|
- project ([$$549])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$469, $$251))
- -- HYBRID_HASH_JOIN [$$251][$$469] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$549, $$251])
- -- STREAM_PROJECT |PARTITIONED|
- select ($$260)
- -- STREAM_SELECT |PARTITIONED|
- project ([$$549, $$251, $$260])
- -- STREAM_PROJECT |PARTITIONED|
- subplan {
- aggregate [$$260] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- select (and(eq($$484, "iap"), not(is-unknown($$483)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- STREAM_SELECT |LOCAL|
- assign [$$484, $$483] <- [$$item2.getField("ua"), $$item2.getField("marop")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- unnest $$item2 <- scan-collection($$482) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- UNNEST |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- SUBPLAN |PARTITIONED|
- project ([$$549, $$251, $$482])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$251, $$482] <- [get-item(split($$item.getField("name"), " "), 0), $$item.getField("value")]
- -- ASSIGN |PARTITIONED|
- project ([$$549, $$item])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$item <- scan-collection(object-pairs($$545))
- -- UNNEST |PARTITIONED|
- project ([$$549, $$545])
- -- STREAM_PROJECT |PARTITIONED|
- select (and(not(and(eq(len($$598), 1), eq(get-item($$598, 0), "some_str"))), eq($$546.getField("t"), "lovStat")))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$598] <- [object-names($$545)]
- -- ASSIGN |PARTITIONED|
- assign [$$545] <- [$$546.getField("ihsaa")]
- -- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$549, $$546] <- test.col2
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- BROADCAST_EXCHANGE |PARTITIONED|
- unnest $$469 <- scan-collection(array: [ "X", "Y" ])
- -- UNNEST |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$456, $$b] <- test.col2
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
- exchange
- -- BROADCAST_EXCHANGE |PARTITIONED|
- unnest $$470 <- scan-collection(array: [ "X", "Y" ])
- -- UNNEST |UNPARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$uniqueID] |PARTITIONED|
- project ([$$466, $$uniqueID])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$uniqueID <- scan-collection($$505)
- -- UNNEST |PARTITIONED|
- project ([$$466, $$505])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$505] <- [array-star($$p.getField("members")).getField("uniqueID")]
- -- ASSIGN |PARTITIONED|
- select (and(eq($$p.getField("r"), "uuu"), eq($$p.getField("t"), "ccc"), not(is-missing($$466))))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$466] <- [$$p.getField("cKey")]
- -- ASSIGN |PARTITIONED|
- project ([$$p])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$p] <- [$$p2]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$549] <- [true]
+ -- ASSIGN |PARTITIONED|
+ project ([$$548])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$468, $$251))
+ -- HYBRID_HASH_JOIN [$$251][$$468] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$548, $$251])
+ -- STREAM_PROJECT |PARTITIONED|
+ select ($$260)
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$548, $$251, $$260])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$260] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ select (and(eq($$483, "iap"), not(is-unknown($$482)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |LOCAL|
+ assign [$$483, $$482] <- [$$item2.getField("ua"), $$item2.getField("marop")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ unnest $$item2 <- scan-collection($$481) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNNEST |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ project ([$$548, $$251, $$481])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$251, $$481] <- [get-item(split($$item.getField("name"), " "), 0), $$item.getField("value")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$548, $$item])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$item <- scan-collection(object-pairs($$544))
+ -- UNNEST |PARTITIONED|
+ project ([$$548, $$544])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (and(not(and(eq(len($$597), 1), eq(get-item($$597, 0), "some_str"))), eq($$545.getField("t"), "lovStat")))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$597] <- [object-names($$544)]
+ -- ASSIGN |PARTITIONED|
+ assign [$$544] <- [$$545.getField("ihsaa")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$548, $$545] <- test.col2
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ unnest $$468 <- scan-collection(array: [ "X", "Y" ])
+ -- UNNEST |UNPARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ exchange
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ unnest $$469 <- scan-collection(array: [ "X", "Y" ])
+ -- UNNEST |UNPARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$uniqueID] |PARTITIONED|
+ project ([$$465, $$uniqueID])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$uniqueID <- scan-collection($$504)
+ -- UNNEST |PARTITIONED|
+ project ([$$465, $$504])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$504] <- [array-star($$p.getField("members")).getField("uniqueID")]
+ -- ASSIGN |PARTITIONED|
+ select (and(eq($$p.getField("r"), "uuu"), eq($$p.getField("t"), "ccc"), not(is-missing($$465))))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$465] <- [$$p.getField("cKey")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$p])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$p] <- [$$p2]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ replicate
+ -- REPLICATE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$p2])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$458, $$p2] <- test.col1
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$600, $$601] |PARTITIONED|
+ project ([$$602, $$603, $$512, $$601, $$508, $$388, $$600])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$388] <- [{"join_key1": $$600, "wpdi": $$508, "wsc": $$512, "scKey": $$601, "wwlps": $$381, "wsd": $$602, "wsm": $$603}]
+ -- ASSIGN |PARTITIONED|
+ project ([$$512, $$508, $$381, $$603, $$602, $$601, $$600])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$603, $$602, $$601, $$600] <- [substring($$515, 0, 7), substring($$515, 0, 10), uppercase(substring($$512, 0, position($$512, "_"))), substring($$510, 0, 38)]
+ -- ASSIGN |PARTITIONED|
+ project ([$$512, $$508, $$515, $$510, $$381])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$381] <- [listify($$380)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ assign [$$380] <- [$$item.getField("wkps")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ unnest $$item <- scan-collection($$509) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNNEST |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ project ([$$512, $$508, $$515, $$510, $$509])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$472, $$353))
+ -- HYBRID_HASH_JOIN [$$353][$$472] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$512, $$508, $$515, $$510, $$509, $$353])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (and(eq($$p2.getField("t"), "ak1"), not(is-missing($$508)), not(is-missing($$540))))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$515, $$512, $$510, $$509, $$508] <- [$$540.getField("ssatt"), $$540.getField("stchk"), $$p2.getField("ed"), $$540.getField("intib"), $$540.getField("coaed")]
-- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- replicate
- -- REPLICATE |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$p2])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$459, $$p2] <- test.col1
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$601, $$602] |PARTITIONED|
- project ([$$603, $$604, $$513, $$602, $$509, $$388, $$601])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$388] <- [{"join_key1": $$601, "wpdi": $$509, "wsc": $$513, "scKey": $$602, "wwlps": $$381, "wsd": $$603, "wsm": $$604}]
- -- ASSIGN |PARTITIONED|
- project ([$$513, $$509, $$381, $$604, $$603, $$602, $$601])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$604, $$603, $$602, $$601] <- [substring($$516, 0, 7), substring($$516, 0, 10), uppercase(substring($$513, 0, position($$513, "_"))), substring($$511, 0, 38)]
- -- ASSIGN |PARTITIONED|
- project ([$$513, $$509, $$516, $$511, $$381])
- -- STREAM_PROJECT |PARTITIONED|
- subplan {
- aggregate [$$381] <- [listify($$380)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- assign [$$380] <- [$$item.getField("wkps")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- unnest $$item <- scan-collection($$510) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- UNNEST |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- SUBPLAN |PARTITIONED|
- project ([$$513, $$509, $$516, $$511, $$510])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$473, $$353))
- -- HYBRID_HASH_JOIN [$$353][$$473] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$513, $$509, $$516, $$511, $$510, $$353])
- -- STREAM_PROJECT |PARTITIONED|
- select (and(eq($$p2.getField("t"), "ak1"), not(is-missing($$509)), not(is-missing($$541))))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$516, $$513, $$511, $$510, $$509] <- [$$541.getField("ssatt"), $$541.getField("stchk"), $$p2.getField("ed"), $$541.getField("intib"), $$541.getField("coaed")]
+ assign [$$353, $$540] <- [$$p2.getField("ha"), $$p2.getField("kat")]
-- ASSIGN |PARTITIONED|
- assign [$$353, $$541] <- [$$p2.getField("ha"), $$p2.getField("kat")]
- -- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- replicate
- -- REPLICATE |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$p2])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$459, $$p2] <- test.col1
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- BROADCAST_EXCHANGE |PARTITIONED|
- unnest $$473 <- scan-collection(array: [ "l1", "l2" ])
- -- UNNEST |UNPARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ replicate
+ -- REPLICATE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$p2])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$458, $$p2] <- test.col1
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ unnest $$472 <- scan-collection(array: [ "l1", "l2" ])
+ -- UNNEST |UNPARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
exchange
- -- HASH_PARTITION_EXCHANGE [$$476] |PARTITIONED|
- project ([$$525, $$526, $$476])
+ -- HASH_PARTITION_EXCHANGE [$$475] |PARTITIONED|
+ project ([$$524, $$525, $$475])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$526, $$525] <- [$$454.getField("omaa"), $$454.getField("qaz")]
+ assign [$$525, $$524] <- [$$454.getField("omaa"), $$454.getField("qaz")]
-- ASSIGN |PARTITIONED|
select (not(is-missing($$454)))
-- STREAM_SELECT |PARTITIONED|
- project ([$$476, $$454])
+ project ([$$475, $$454])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$476, $$454] <- [$$sbr.getField("ed"), $$sbr.getField("tohac")]
+ assign [$$475, $$454] <- [$$sbr.getField("ed"), $$sbr.getField("tohac")]
-- ASSIGN |PARTITIONED|
project ([$$sbr])
-- STREAM_PROJECT |PARTITIONED|
@@ -340,9 +338,9 @@ distribute result [$$446]
-- STREAM_PROJECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$459, $$p2] <- test.col1
+ data-scan []<-[$$458, $$p2] <- test.col1
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.1.ddl.sqlpp
new file mode 100644
index 0000000000..36024d85d8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.1.ddl.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type prim as open {
+ id: bigint
+};
+
+create dataset A(prim)
+primary key id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.2.update.sqlpp
new file mode 100644
index 0000000000..fe77706e1d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.2.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+INSERT INTO A(
+ SELECT VALUE {"id": intVal, "a": intVal%2, "b": intVal%5 , "d": intVal%25}
+ FROM range(1, 2) intVal
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.3.query.sqlpp
new file mode 100644
index 0000000000..02ab319123
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.3.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+SELECT sub.* FROM (SELECT a.* FROM A a) sub;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-star/nested/nested.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-star/nested/nested.3.adm
new file mode 100644
index 0000000000..125daf12f9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-star/nested/nested.3.adm
@@ -0,0 +1,2 @@
+{ "id": 1, "a": 1, "b": 1, "d": 1 }
+{ "id": 2, "a": 0, "b": 2, "d": 2 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index af63f0b1e4..eb46490c2f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -10052,6 +10052,11 @@
<output-dir compare="Text">mixed</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="nested">
+ <output-dir compare="Text">nested</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="select-star">
<compilation-unit name="no_star">
<output-dir compare="Text">no_star</output-dir>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/DoNotExtractExpressionAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/DoNotExtractExpressionAnnotation.java
new file mode 100644
index 0000000000..fa19e22dbc
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/DoNotExtractExpressionAnnotation.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.algebricks.core.algebra.expressions;
+
+public class DoNotExtractExpressionAnnotation implements IExpressionAnnotation {
+ public static final DoNotExtractExpressionAnnotation INSTANCE = new DoNotExtractExpressionAnnotation();
+
+ private DoNotExtractExpressionAnnotation() {
+
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
index e2ba5571b7..7c4d62c56c 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
@@ -39,6 +39,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.DoNotExtractExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
@@ -260,6 +261,14 @@ public class ExtractCommonExpressionsRule implements IAlgebraicRewriteRule {
return modified;
}
+ private static boolean isExprExtractable(ILogicalExpression expr) {
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return true;
+ }
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ return !funcExpr.hasAnnotation(DoNotExtractExpressionAnnotation.class);
+ }
+
private class CommonExpressionSubstitutionVisitor implements ILogicalExpressionReferenceTransform {
private IOptimizationContext context;
@@ -302,7 +311,7 @@ public class ExtractCommonExpressionsRule implements IAlgebraicRewriteRule {
}
}
} else {
- if (expr.isFunctional() && assignCommonExpression(exprEqClass, expr)) {
+ if (expr.isFunctional() && isExprExtractable(expr) && assignCommonExpression(exprEqClass, expr)) {
modified = true;
//re-obtain the live vars after rewriting in the method called in the if condition
Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
(asterixdb) 10/18: Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a2e1e197ca3c447720c12961db28b3fc797f4ed9
Merge: 5510be7dc6 1ed3c060ba
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Feb 22 23:05:19 2026 -0500
Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
* [NO ISSUE][*DB][EXT] Update Apache Hadoop -> 3.4.2 for CVE (MB-70644)
Ext-ref: MB-70644
Change-Id: If82ce57c6f816f3bf5d9ce2a745ba98a396a317d
asterixdb/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
(asterixdb) 16/18: [ASTERIXDB-3705][GEO] Add new geospatial functions
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 72a01fe09f0913eb8f7850a09458086586c41bf9
Author: sshiv012 <ss...@ucr.edu>
AuthorDate: Tue Feb 17 12:43:32 2026 -0800
[ASTERIXDB-3705][GEO] Add new geospatial functions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add new geospatial functions: st_convex_hull, st_centroid,
st_is_valid, st_reverse, st_flip_coordinates,
st_distance_sphere (haversine), st_dwithin, and st_buffer
- Register all new functions in BuiltinFunctions and GeoFunctionRegistrant
- Add SQL++ integration tests and expected results for all new functions
- Updates docs
Bug fix:
- Fix st_point_n (AbstractSTGeometryNDescriptor) returning stale results
by adding missing resultStorage.reset() call before each evaluation
Change-Id: Ic4791ce4934ff3a1dd9051ddb13846a4bea4e5c1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20904
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
---
.../single-method/single-method.16.query.sqlpp | 24 ++++
.../single-method/single-method.17.query.sqlpp | 22 ++++
.../single-method/single-method.18.query.sqlpp | 22 ++++
.../single-method/single-method.19.query.sqlpp | 23 ++++
.../single-method/single-method.20.query.sqlpp | 24 ++++
.../single-method/single-method.21.query.sqlpp | 22 ++++
.../two-geometries/two-geometries.06.query.sqlpp | 25 ++++
.../results/geojson/datatype/result.08.adm | 4 +-
.../results/geojson/single-method/result.16.adm | 1 +
.../results/geojson/single-method/result.17.adm | 1 +
.../results/geojson/single-method/result.18.adm | 1 +
.../results/geojson/single-method/result.19.adm | 1 +
.../results/geojson/single-method/result.20.adm | 1 +
.../results/geojson/single-method/result.21.adm | 1 +
.../results/geojson/two-geometries/result.06.adm | 10 ++
.../asterix-doc/src/site/markdown/geo/functions.md | 102 +++++++++++++++
.../geo/evaluators/GeoFunctionRegistrant.java | 16 +++
.../functions/AbstractSTGeometryNDescriptor.java | 1 +
...tryNDescriptor.java => STBufferDescriptor.java} | 77 ++++++------
.../evaluators/functions/STCentroidDescriptor.java | 42 +++++++
.../functions/STConvexHullDescriptor.java | 42 +++++++
.../evaluators/functions/STDWithinDescriptor.java | 139 +++++++++++++++++++++
.../functions/STDistanceSphereDescriptor.java | 59 +++++++++
.../functions/STFlipCoordinatesDescriptor.java | 69 ++++++++++
.../evaluators/functions/STIsValidDescriptor.java | 42 +++++++
.../evaluators/functions/STReverseDescriptor.java | 42 +++++++
.../asterix/om/functions/BuiltinFunctions.java | 19 +++
27 files changed, 793 insertions(+), 39 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.16.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.16.query.sqlpp
new file mode 100644
index 0000000000..85e39a3948
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.16.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+{
+"a": st_as_text(st_convex_hull(st_geom_from_text("MULTIPOINT(0 0, 1 0, 1 1, 0 1, 0.5 0.5)"))),
+"b": st_as_text(st_centroid(st_geom_from_text("POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))"))),
+"c": st_is_valid(st_geom_from_text("POLYGON((0 0, 1 1, 1 0, 0 1, 0 0))")),
+"d": st_is_valid(st_geom_from_text("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"))
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.17.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.17.query.sqlpp
new file mode 100644
index 0000000000..97d34b302e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.17.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+{
+"a": st_as_text(st_reverse(st_geom_from_text("LINESTRING(0 0, 1 1, 2 2)"))),
+"b": st_as_text(st_reverse(st_geom_from_text("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))")))
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.18.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.18.query.sqlpp
new file mode 100644
index 0000000000..bfeaf82142
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.18.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+{
+"a": st_as_text(st_flip_coordinates(st_geom_from_text("POINT(1 2)"))),
+"b": st_as_text(st_flip_coordinates(st_geom_from_text("LINESTRING(1 2, 3 4)")))
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.19.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.19.query.sqlpp
new file mode 100644
index 0000000000..41036d44d7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.19.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+{
+"a": st_as_text(st_buffer(st_geom_from_text("POINT(0 0)"), 0.0)),
+"b": st_area(st_buffer(st_geom_from_text("POINT(0 0)"), 1.0)) > 3.0,
+"c": st_area(st_buffer(st_geom_from_text("POINT(0 0)"), 1.0)) > 4.0
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.20.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.20.query.sqlpp
new file mode 100644
index 0000000000..972e66b7ce
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.20.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+{
+"a": st_distance_sphere(st_geom_from_text("POINT(0 0)"), st_geom_from_text("POINT(0 0)")),
+"b": st_distance_sphere(st_geom_from_text("POINT(0 0)"), st_geom_from_text("POINT(1 0)")),
+"c": st_distance_sphere(st_geom_from_text("POINT(0 0)"), st_geom_from_text("POINT(0 1)")),
+"d": st_distance_sphere(st_geom_from_text("POINT(0 0)"), st_geom_from_text("POINT(0.001 0)"))
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.21.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.21.query.sqlpp
new file mode 100644
index 0000000000..27d1625771
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/single-method/single-method.21.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+{
+"a": st_dwithin(st_geom_from_text("POINT(0 0)"), st_geom_from_text("POINT(1 0)"), 2.0),
+"b": st_dwithin(st_geom_from_text("POINT(0 0)"), st_geom_from_text("POINT(5 0)"), 2.0)
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/two-geometries/two-geometries.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/two-geometries/two-geometries.06.query.sqlpp
new file mode 100644
index 0000000000..6233489a8b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/geojson/two-geometries/two-geometries.06.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use GeoJSON1;
+
+SELECT VALUE {
+ "DWithin_0_5": st_dwithin(geo.myGeometry1, geo.myGeometry2, 0.5),
+ "DWithin5": st_dwithin(geo.myGeometry1, geo.myGeometry2, 5.0)
+}
+FROM Geometries geo ORDER BY geo.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/datatype/result.08.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/datatype/result.08.adm
index 198797093f..cbadf0bb89 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/datatype/result.08.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/datatype/result.08.adm
@@ -1,3 +1,3 @@
{ "PointN": {"type":"Point","coordinates":[-69.199136,-12.599842]}, "StartPoint": {"type":"Point","coordinates":[-69.1991349,-12.6006222]}, "Envelope": {"type":"Polygon","coordinates":[[[-69.199136,-12.6010968],[-69.199136,-12.5998133],[-69.1972972,-12.5998133],[-69.1972972,-12.6010968],[-69.199136,-12.6010968]]]} }
-{ "PointN": {"type":"Point","coordinates":[-69.199136,-12.599842]}, "StartPoint": {"type":"Point","coordinates":[-113.98,39.198]}, "Envelope": {"type":"Polygon","coordinates":[[[-113.981,39.195],[-113.981,39.198],[-113.98,39.198],[-113.98,39.195],[-113.981,39.195]]]} }
-{ "PointN": {"type":"Point","coordinates":[-69.199136,-12.599842]}, "StartPoint": {"type":"Point","coordinates":[1,2]}, "Envelope": {"type":"Polygon","coordinates":[[[1,2],[1,8],[7,8],[7,2],[1,2]]]} }
\ No newline at end of file
+{ "PointN": {"type":"Point","coordinates":[-113.981,39.195]}, "StartPoint": {"type":"Point","coordinates":[-113.98,39.198]}, "Envelope": {"type":"Polygon","coordinates":[[[-113.981,39.195],[-113.981,39.198],[-113.98,39.198],[-113.98,39.195],[-113.981,39.195]]]} }
+{ "PointN": {"type":"Point","coordinates":[4,5]}, "StartPoint": {"type":"Point","coordinates":[1,2]}, "Envelope": {"type":"Polygon","coordinates":[[[1,2],[1,8],[7,8],[7,2],[1,2]]]} }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.16.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.16.adm
new file mode 100644
index 0000000000..606c5c613f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.16.adm
@@ -0,0 +1 @@
+{ "a": "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", "b": "POINT (5 5)", "c": false, "d": true }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.17.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.17.adm
new file mode 100644
index 0000000000..ee0219f578
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.17.adm
@@ -0,0 +1 @@
+{ "a": "LINESTRING (2 2, 1 1, 0 0)", "b": "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.18.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.18.adm
new file mode 100644
index 0000000000..e73d9f3352
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.18.adm
@@ -0,0 +1 @@
+{ "a": "POINT (2 1)", "b": "LINESTRING (2 1, 4 3)" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.19.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.19.adm
new file mode 100644
index 0000000000..e35cdf0b80
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.19.adm
@@ -0,0 +1 @@
+{ "a": "POLYGON EMPTY", "b": true, "c": false }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.20.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.20.adm
new file mode 100644
index 0000000000..a591562fbe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.20.adm
@@ -0,0 +1 @@
+{ "a": 0.0, "b": 111195.0662708989, "c": 111195.0662708989, "d": 111.1950662708989 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.21.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.21.adm
new file mode 100644
index 0000000000..7846424f35
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/single-method/result.21.adm
@@ -0,0 +1 @@
+{ "a": true, "b": false }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/two-geometries/result.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/two-geometries/result.06.adm
new file mode 100644
index 0000000000..9ed21094e6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/geojson/two-geometries/result.06.adm
@@ -0,0 +1,10 @@
+{ "DWithin_0_5": true, "DWithin5": true }
+{ "DWithin_0_5": true, "DWithin5": true }
+{ "DWithin_0_5": true, "DWithin5": true }
+{ "DWithin_0_5": true, "DWithin5": true }
+{ "DWithin_0_5": true, "DWithin5": true }
+{ "DWithin_0_5": false, "DWithin5": true }
+{ "DWithin_0_5": true, "DWithin5": true }
+{ "DWithin_0_5": false, "DWithin5": true }
+{ "DWithin_0_5": true, "DWithin5": true }
+{ "DWithin_0_5": true, "DWithin5": true }
diff --git a/asterixdb/asterix-doc/src/site/markdown/geo/functions.md b/asterixdb/asterix-doc/src/site/markdown/geo/functions.md
index 2bd22507f4..e91568f4d4 100644
--- a/asterixdb/asterix-doc/src/site/markdown/geo/functions.md
+++ b/asterixdb/asterix-doc/src/site/markdown/geo/functions.md
@@ -333,6 +333,17 @@ There are primitive functions that take as input geometry/es and return a primit
0.30901547439030225
+### st_distance_sphere ###
+* Return the minimum distance in meters between two point geometries using spherical (haversine) calculation.
+
+* Example:
+ * Command:
+
+ st_distance_sphere(st_geom_from_text('POINT(0 0)'), st_geom_from_text('POINT(1 0)'));
+ * Result:
+
+ 111195.0662708989
+
## <a id="predicate">Spatial Predicate</a>
Spatial predicate functions test for a relationship between two geometries and return a Boolean value (true/false).
@@ -482,6 +493,42 @@ Spatial predicate functions test for a relationship between two geometries and r
### st_within ###
* Return true if the geometry A is completely inside geometry B.
+### st_is_valid ###
+* Return TRUE if the geometry is topologically valid.
+
+* Example:
+ * Command:
+
+ st_is_valid(st_geom_from_text("POLYGON((0 0, 1 1, 1 0, 0 1, 0 0))"));
+ * Result:
+
+ false
+* Example:
+ * Command:
+
+ st_is_valid(st_geom_from_text("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))"));
+ * Result:
+
+ true
+
+### st_dwithin ###
+* Return TRUE if the two geometries are within the specified distance of one another.
+
+* Example:
+ * Command:
+
+ st_dwithin(st_geom_from_text('POINT(0 0)'), st_geom_from_text('POINT(1 0)'), 2.0);
+ * Result:
+
+ true
+* Example:
+ * Command:
+
+ st_dwithin(st_geom_from_text('POINT(0 0)'), st_geom_from_text('POINT(5 0)'), 2.0);
+ * Result:
+
+ false
+
## <a id="analysis">Spatial Analysis</a>
Spatial analysis functions take as input one or more geometries and return a geometry as output.
@@ -617,6 +664,61 @@ Spatial analysis functions take as input one or more geometries and return a geo
{"type":"MultiLineString","coordinates":[[[0,2],[1,2],[2,2],[1,1]],[[5,2],[4,2],[3,3],[4,4],[5,5],[6,6]]],"crs":null}
+### st_convex_hull ###
+* Return a geometry that represents the convex hull of the input geometry.
+
+* Example:
+ * Command:
+
+ st_convex_hull(st_geom_from_text('MULTIPOINT(0 0, 1 0, 1 1, 0 1, 0.5 0.5)'));
+ * Result:
+
+ {"type":"Polygon","coordinates":[[[0,0],[0,1],[1,1],[1,0],[0,0]]]}
+
+### st_centroid ###
+* Return a point geometry representing the geometric center of the input geometry.
+
+* Example:
+ * Command:
+
+ st_centroid(st_geom_from_text('POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))'));
+ * Result:
+
+ {"type":"Point","coordinates":[5,5]}
+
+### st_reverse ###
+* Return the geometry with vertex order reversed.
+
+* Example:
+ * Command:
+
+ st_reverse(st_geom_from_text('LINESTRING(0 0, 1 1, 2 2)'));
+ * Result:
+
+ {"type":"LineString","coordinates":[[2,2],[1,1],[0,0]]}
+
+### st_flip_coordinates ###
+* Return the geometry with X and Y coordinates swapped.
+
+* Example:
+ * Command:
+
+ st_flip_coordinates(st_geom_from_text('POINT(1 2)'));
+ * Result:
+
+ {"type":"Point","coordinates":[2,1]}
+
+### st_buffer ###
+* Return a geometry that represents all points whose distance from the input geometry is less than or equal to the specified distance.
+
+* Example:
+ * Command:
+
+ st_area(st_buffer(st_geom_from_text('POINT(0 0)'), 1.0)) > 3.0;
+ * Result:
+
+ true
+
### st_polygonize ###
* Aggregate. Creates a GeometryCollection containing possible polygons formed from the constituent linework of a set of geometries.
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/GeoFunctionRegistrant.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/GeoFunctionRegistrant.java
index 75927cc1ba..ba20fa4c3d 100644
--- a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/GeoFunctionRegistrant.java
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/GeoFunctionRegistrant.java
@@ -27,17 +27,23 @@ import org.apache.asterix.geo.evaluators.functions.STAsBinaryDescriptor;
import org.apache.asterix.geo.evaluators.functions.STAsGeoJSONDescriptor;
import org.apache.asterix.geo.evaluators.functions.STAsTextDescriptor;
import org.apache.asterix.geo.evaluators.functions.STBoundaryDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STBufferDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STCentroidDescriptor;
import org.apache.asterix.geo.evaluators.functions.STContainsDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STConvexHullDescriptor;
import org.apache.asterix.geo.evaluators.functions.STCoordDimDescriptor;
import org.apache.asterix.geo.evaluators.functions.STCrossesDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STDWithinDescriptor;
import org.apache.asterix.geo.evaluators.functions.STDifferenceDescriptor;
import org.apache.asterix.geo.evaluators.functions.STDimensionDescriptor;
import org.apache.asterix.geo.evaluators.functions.STDisjointDescriptor;
import org.apache.asterix.geo.evaluators.functions.STDistanceDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STDistanceSphereDescriptor;
import org.apache.asterix.geo.evaluators.functions.STEndPointDescriptor;
import org.apache.asterix.geo.evaluators.functions.STEnvelopeDescriptor;
import org.apache.asterix.geo.evaluators.functions.STEqualsDescriptor;
import org.apache.asterix.geo.evaluators.functions.STExteriorRingDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STFlipCoordinatesDescriptor;
import org.apache.asterix.geo.evaluators.functions.STGeomFromTextDescriptor;
import org.apache.asterix.geo.evaluators.functions.STGeomFromTextSRIDDescriptor;
import org.apache.asterix.geo.evaluators.functions.STGeomFromWKBDescriptor;
@@ -51,6 +57,7 @@ import org.apache.asterix.geo.evaluators.functions.STIsCollectionDescriptor;
import org.apache.asterix.geo.evaluators.functions.STIsEmptyDescriptor;
import org.apache.asterix.geo.evaluators.functions.STIsRingDescriptor;
import org.apache.asterix.geo.evaluators.functions.STIsSimpleDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STIsValidDescriptor;
import org.apache.asterix.geo.evaluators.functions.STLengthDescriptor;
import org.apache.asterix.geo.evaluators.functions.STLineFromMultiPointDescriptor;
import org.apache.asterix.geo.evaluators.functions.STMBRDescriptor;
@@ -68,6 +75,7 @@ import org.apache.asterix.geo.evaluators.functions.STOverlapsDescriptor;
import org.apache.asterix.geo.evaluators.functions.STPointNDescriptor;
import org.apache.asterix.geo.evaluators.functions.STPolygonizeDescriptor;
import org.apache.asterix.geo.evaluators.functions.STRelateDescriptor;
+import org.apache.asterix.geo.evaluators.functions.STReverseDescriptor;
import org.apache.asterix.geo.evaluators.functions.STSRIDDescriptor;
import org.apache.asterix.geo.evaluators.functions.STStartPointDescriptor;
import org.apache.asterix.geo.evaluators.functions.STSymDifferenceDescriptor;
@@ -160,5 +168,13 @@ public class GeoFunctionRegistrant implements IFunctionRegistrant {
fc.add(STPolygonizeDescriptor.FACTORY);
fc.add(STMBRDescriptor.FACTORY);
fc.add(STMBREnlargeDescriptor.FACTORY);
+ fc.add(STConvexHullDescriptor.FACTORY);
+ fc.add(STCentroidDescriptor.FACTORY);
+ fc.add(STIsValidDescriptor.FACTORY);
+ fc.add(STReverseDescriptor.FACTORY);
+ fc.add(STFlipCoordinatesDescriptor.FACTORY);
+ fc.add(STDistanceSphereDescriptor.FACTORY);
+ fc.add(STDWithinDescriptor.FACTORY);
+ fc.add(STBufferDescriptor.FACTORY);
}
}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/AbstractSTGeometryNDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/AbstractSTGeometryNDescriptor.java
index c3096c45b1..a804631541 100644
--- a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/AbstractSTGeometryNDescriptor.java
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/AbstractSTGeometryNDescriptor.java
@@ -79,6 +79,7 @@ public abstract class AbstractSTGeometryNDescriptor extends AbstractScalarFuncti
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
eval.evaluate(tuple, inputArg);
byte[] data = inputArg.getByteArray();
int offset = inputArg.getStartOffset();
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/AbstractSTGeometryNDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STBufferDescriptor.java
similarity index 61%
copy from asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/AbstractSTGeometryNDescriptor.java
copy to asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STBufferDescriptor.java
index c3096c45b1..9af28b85d3 100644
--- a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/AbstractSTGeometryNDescriptor.java
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STBufferDescriptor.java
@@ -24,11 +24,15 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.asterix.dataflow.data.nontagged.serde.AGeometrySerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AInt64SerializerDeserializer;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.asterix.runtime.exceptions.InvalidDataFormatException;
import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.context.IEvaluatorContext;
@@ -39,11 +43,16 @@ import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import org.locationtech.jts.geom.Geometry;
-public abstract class AbstractSTGeometryNDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+public class STBufferDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ public static final IFunctionDescriptorFactory FACTORY = STBufferDescriptor::new;
private static final long serialVersionUID = 1L;
- abstract protected Geometry evaluateOGCGeometry(Geometry geometry, int n) throws HyracksDataException;
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_BUFFER;
+ }
@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
@@ -52,65 +61,59 @@ public abstract class AbstractSTGeometryNDescriptor extends AbstractScalarFuncti
@Override
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
-
- return new AbstractSTGeometryNEvaluator(args, ctx);
+ return new STBufferEvaluator(args, ctx);
}
};
}
- private class AbstractSTGeometryNEvaluator implements IScalarEvaluator {
+ private class STBufferEvaluator implements IScalarEvaluator {
+ private final ArrayBackedValueStorage resultStorage;
+ private final DataOutput out;
+ private final IPointable inputArg0;
+ private final IScalarEvaluator eval0;
+ private final IPointable inputArg1;
+ private final IScalarEvaluator eval1;
- private ArrayBackedValueStorage resultStorage;
- private DataOutput out;
- private IPointable inputArg;
- private IScalarEvaluator eval;
- private IPointable inputArg0;
- private IScalarEvaluator eval0;
-
- public AbstractSTGeometryNEvaluator(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx)
- throws HyracksDataException {
+ public STBufferEvaluator(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx) throws HyracksDataException {
resultStorage = new ArrayBackedValueStorage();
out = resultStorage.getDataOutput();
- inputArg = new VoidPointable();
- eval = args[0].createScalarEvaluator(ctx);
inputArg0 = new VoidPointable();
- eval0 = args[1].createScalarEvaluator(ctx);
+ eval0 = args[0].createScalarEvaluator(ctx);
+ inputArg1 = new VoidPointable();
+ eval1 = args[1].createScalarEvaluator(ctx);
}
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- eval.evaluate(tuple, inputArg);
- byte[] data = inputArg.getByteArray();
- int offset = inputArg.getStartOffset();
- int len = inputArg.getLength();
-
+ resultStorage.reset();
eval0.evaluate(tuple, inputArg0);
- byte[] data0 = inputArg0.getByteArray();
+ byte[] bytes0 = inputArg0.getByteArray();
int offset0 = inputArg0.getStartOffset();
+ int len0 = inputArg0.getLength();
- if (data[offset] != ATypeTag.SERIALIZED_GEOMETRY_TYPE_TAG) {
- throw new TypeMismatchException(sourceLoc, getIdentifier(), 0, data[offset],
+ eval1.evaluate(tuple, inputArg1);
+ byte[] bytes1 = inputArg1.getByteArray();
+ int offset1 = inputArg1.getStartOffset();
+
+ ATypeTag tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes0[offset0]);
+ if (tag != ATypeTag.GEOMETRY) {
+ throw new TypeMismatchException(sourceLoc, getIdentifier(), 0, bytes0[offset0],
ATypeTag.SERIALIZED_GEOMETRY_TYPE_TAG);
}
- if (data0[offset0] != ATypeTag.SERIALIZED_INT64_TYPE_TAG) {
- throw new TypeMismatchException(sourceLoc, getIdentifier(), 0, data0[offset0],
- ATypeTag.SERIALIZED_INT64_TYPE_TAG);
- }
- ByteArrayInputStream inStream = new ByteArrayInputStream(data, offset + 1, len - 1);
- DataInputStream dataIn = new DataInputStream(inStream);
- Geometry geometry = AGeometrySerializerDeserializer.INSTANCE.deserialize(dataIn).getGeometry();
- int n = (int) AInt64SerializerDeserializer.getLong(data0, offset0 + 1);
+ double distance = ATypeHierarchy.getDoubleValue(getIdentifier().getName(), 1, bytes1, offset1);
- Geometry geometryN = evaluateOGCGeometry(geometry, n);
+ DataInputStream dataIn0 = new DataInputStream(new ByteArrayInputStream(bytes0, offset0 + 1, len0 - 1));
+ Geometry geometry = AGeometrySerializerDeserializer.INSTANCE.deserialize(dataIn0).getGeometry();
try {
+ Geometry buffered = geometry.buffer(distance);
out.writeByte(ATypeTag.SERIALIZED_GEOMETRY_TYPE_TAG);
- AGeometrySerializerDeserializer.INSTANCE.serialize(geometryN, out);
- result.set(resultStorage);
+ AGeometrySerializerDeserializer.INSTANCE.serialize(buffered, out);
} catch (IOException e) {
throw new InvalidDataFormatException(sourceLoc, getIdentifier(), e,
ATypeTag.SERIALIZED_GEOMETRY_TYPE_TAG);
}
+ result.set(resultStorage);
}
}
}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STCentroidDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STCentroidDescriptor.java
new file mode 100644
index 0000000000..0d280daed6
--- /dev/null
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STCentroidDescriptor.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.geo.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.locationtech.jts.geom.Geometry;
+
+public class STCentroidDescriptor extends AbstractSTSingleGeometryDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = STCentroidDescriptor::new;
+
+ @Override
+ protected Object evaluateOGCGeometry(Geometry geometry) throws HyracksDataException {
+ return geometry.getCentroid();
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_CENTROID;
+ }
+
+}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STConvexHullDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STConvexHullDescriptor.java
new file mode 100644
index 0000000000..9241ec8c5d
--- /dev/null
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STConvexHullDescriptor.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.geo.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.locationtech.jts.geom.Geometry;
+
+public class STConvexHullDescriptor extends AbstractSTSingleGeometryDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = STConvexHullDescriptor::new;
+
+ @Override
+ protected Object evaluateOGCGeometry(Geometry geometry) throws HyracksDataException {
+ return geometry.convexHull();
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_CONVEX_HULL;
+ }
+
+}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDWithinDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDWithinDescriptor.java
new file mode 100644
index 0000000000..e623a51ccc
--- /dev/null
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDWithinDescriptor.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.geo.evaluators.functions;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AGeometrySerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.exceptions.InvalidDataFormatException;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IEvaluatorContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import org.locationtech.jts.geom.Geometry;
+
+public class STDWithinDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ public static final IFunctionDescriptorFactory FACTORY = STDWithinDescriptor::new;
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_DWITHIN;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
+ return new STDWithinEvaluator(args, ctx);
+ }
+ };
+ }
+
+ private class STDWithinEvaluator implements IScalarEvaluator {
+ private final ArrayBackedValueStorage resultStorage;
+ private final DataOutput out;
+ private final IPointable inputArg0;
+ private final IScalarEvaluator eval0;
+ private final IPointable inputArg1;
+ private final IScalarEvaluator eval1;
+ private final IPointable inputArg2;
+ private final IScalarEvaluator eval2;
+
+ public STDWithinEvaluator(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx) throws HyracksDataException {
+ resultStorage = new ArrayBackedValueStorage();
+ out = resultStorage.getDataOutput();
+ inputArg0 = new VoidPointable();
+ eval0 = args[0].createScalarEvaluator(ctx);
+ inputArg1 = new VoidPointable();
+ eval1 = args[1].createScalarEvaluator(ctx);
+ inputArg2 = new VoidPointable();
+ eval2 = args[2].createScalarEvaluator(ctx);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ eval0.evaluate(tuple, inputArg0);
+ byte[] bytes0 = inputArg0.getByteArray();
+ int offset0 = inputArg0.getStartOffset();
+ int len0 = inputArg0.getLength();
+
+ eval1.evaluate(tuple, inputArg1);
+ byte[] bytes1 = inputArg1.getByteArray();
+ int offset1 = inputArg1.getStartOffset();
+ int len1 = inputArg1.getLength();
+
+ eval2.evaluate(tuple, inputArg2);
+ byte[] bytes2 = inputArg2.getByteArray();
+ int offset2 = inputArg2.getStartOffset();
+
+ ATypeTag tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes0[offset0]);
+ if (tag != ATypeTag.GEOMETRY) {
+ throw new TypeMismatchException(sourceLoc, getIdentifier(), 0, bytes0[offset0],
+ ATypeTag.SERIALIZED_GEOMETRY_TYPE_TAG);
+ }
+ tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes1[offset1]);
+ if (tag != ATypeTag.GEOMETRY) {
+ throw new TypeMismatchException(sourceLoc, getIdentifier(), 1, bytes1[offset1],
+ ATypeTag.SERIALIZED_GEOMETRY_TYPE_TAG);
+ }
+
+ double distance = ATypeHierarchy.getDoubleValue(getIdentifier().getName(), 2, bytes2, offset2);
+
+ DataInputStream dataIn0 = new DataInputStream(new ByteArrayInputStream(bytes0, offset0 + 1, len0 - 1));
+ Geometry geometry0 = AGeometrySerializerDeserializer.INSTANCE.deserialize(dataIn0).getGeometry();
+ DataInputStream dataIn1 = new DataInputStream(new ByteArrayInputStream(bytes1, offset1 + 1, len1 - 1));
+ Geometry geometry1 = AGeometrySerializerDeserializer.INSTANCE.deserialize(dataIn1).getGeometry();
+ try {
+ boolean val = geometry0.isWithinDistance(geometry1, distance);
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN)
+ .serialize(val ? ABoolean.TRUE : ABoolean.FALSE, out);
+ } catch (IOException e) {
+ throw new InvalidDataFormatException(sourceLoc, getIdentifier(), e,
+ ATypeTag.SERIALIZED_GEOMETRY_TYPE_TAG);
+ }
+ result.set(resultStorage);
+ }
+ }
+}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDistanceSphereDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDistanceSphereDescriptor.java
new file mode 100644
index 0000000000..b882dd9ae4
--- /dev/null
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STDistanceSphereDescriptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.geo.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.Geometry;
+
+public class STDistanceSphereDescriptor extends AbstractSTDoubleGeometryDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = STDistanceSphereDescriptor::new;
+
+ private static final double EARTH_RADIUS_METERS = 6371008.0;
+
+ @Override
+ protected Object evaluateOGCGeometry(Geometry geometry0, Geometry geometry1) throws HyracksDataException {
+ Coordinate c0 = geometry0.getCoordinate();
+ Coordinate c1 = geometry1.getCoordinate();
+ return haversineDistance(c0.y, c0.x, c1.y, c1.x);
+ }
+
+ private static double haversineDistance(double lat1, double lon1, double lat2, double lon2) {
+ double dLat = Math.toRadians(lat2 - lat1);
+ double dLon = Math.toRadians(lon2 - lon1);
+ double rLat1 = Math.toRadians(lat1);
+ double rLat2 = Math.toRadians(lat2);
+
+ double a =
+ Math.pow(Math.sin(dLat / 2), 2) + Math.cos(rLat1) * Math.cos(rLat2) * Math.pow(Math.sin(dLon / 2), 2);
+ double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+ return EARTH_RADIUS_METERS * c;
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_DISTANCE_SPHERE;
+ }
+
+}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STFlipCoordinatesDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STFlipCoordinatesDescriptor.java
new file mode 100644
index 0000000000..7ad3277189
--- /dev/null
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STFlipCoordinatesDescriptor.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.geo.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.locationtech.jts.geom.CoordinateSequence;
+import org.locationtech.jts.geom.CoordinateSequenceFilter;
+import org.locationtech.jts.geom.Geometry;
+
+public class STFlipCoordinatesDescriptor extends AbstractSTSingleGeometryDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = STFlipCoordinatesDescriptor::new;
+
+ @Override
+ protected Object evaluateOGCGeometry(Geometry geometry) throws HyracksDataException {
+ Geometry flipped = geometry.copy();
+ flipped.apply(FlipCoordinatesFilter.INSTANCE);
+ return flipped;
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_FLIP_COORDINATES;
+ }
+
+ private static final class FlipCoordinatesFilter implements CoordinateSequenceFilter {
+ private static final FlipCoordinatesFilter INSTANCE = new FlipCoordinatesFilter();
+
+ @Override
+ public void filter(CoordinateSequence seq, int i) {
+
+ double x = seq.getX(i);
+ double y = seq.getY(i);
+ seq.setOrdinate(i, CoordinateSequence.X, y);
+ seq.setOrdinate(i, CoordinateSequence.Y, x);
+ }
+
+ @Override
+ public boolean isDone() {
+ return false;
+ }
+
+ @Override
+ public boolean isGeometryChanged() {
+ return true;
+ }
+ }
+
+}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STIsValidDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STIsValidDescriptor.java
new file mode 100644
index 0000000000..3f9f4cb4bc
--- /dev/null
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STIsValidDescriptor.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.geo.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.locationtech.jts.geom.Geometry;
+
+public class STIsValidDescriptor extends AbstractSTSingleGeometryDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = STIsValidDescriptor::new;
+
+ @Override
+ protected Object evaluateOGCGeometry(Geometry geometry) throws HyracksDataException {
+ return geometry.isValid();
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_IS_VALID;
+ }
+
+}
diff --git a/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STReverseDescriptor.java b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STReverseDescriptor.java
new file mode 100644
index 0000000000..93935fb93c
--- /dev/null
+++ b/asterixdb/asterix-geo/src/main/java/org/apache/asterix/geo/evaluators/functions/STReverseDescriptor.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.geo.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.locationtech.jts.geom.Geometry;
+
+public class STReverseDescriptor extends AbstractSTSingleGeometryDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = STReverseDescriptor::new;
+
+ @Override
+ protected Object evaluateOGCGeometry(Geometry geometry) throws HyracksDataException {
+ return geometry.reverse();
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.ST_REVERSE;
+ }
+
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index 51ad36b4bd..144a389a36 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1128,6 +1128,16 @@ public class BuiltinFunctions {
public static final FunctionIdentifier ST_MBR = FunctionConstants.newAsterix("st-mbr", 1);
public static final FunctionIdentifier ST_MBR_ENLARGE = FunctionConstants.newAsterix("st-mbr-enlarge", 2);
+ public static final FunctionIdentifier ST_CONVEX_HULL = FunctionConstants.newAsterix("st-convex-hull", 1);
+ public static final FunctionIdentifier ST_CENTROID = FunctionConstants.newAsterix("st-centroid", 1);
+ public static final FunctionIdentifier ST_IS_VALID = FunctionConstants.newAsterix("st-is-valid", 1);
+ public static final FunctionIdentifier ST_REVERSE = FunctionConstants.newAsterix("st-reverse", 1);
+ public static final FunctionIdentifier ST_FLIP_COORDINATES = FunctionConstants.newAsterix("st-flip-coordinates", 1);
+ public static final FunctionIdentifier ST_SET_SRID = FunctionConstants.newAsterix("st-set-srid", 2);
+ public static final FunctionIdentifier ST_DISTANCE_SPHERE = FunctionConstants.newAsterix("st-distance-sphere", 2);
+ public static final FunctionIdentifier ST_DWITHIN = FunctionConstants.newAsterix("st-dwithin", 3);
+ public static final FunctionIdentifier ST_BUFFER = FunctionConstants.newAsterix("st-buffer", 2);
+
// Spatial and temporal type accessors
public static final FunctionIdentifier ACCESSOR_TEMPORAL_YEAR = FunctionConstants.newAsterix("get-year", 1);
public static final FunctionIdentifier ACCESSOR_TEMPORAL_MONTH = FunctionConstants.newAsterix("get-month", 1);
@@ -2009,6 +2019,15 @@ public class BuiltinFunctions {
addPrivateFunction(ST_MBR, ARectangleTypeComputer.INSTANCE, true);
addPrivateFunction(ST_MBR_ENLARGE, ARectangleTypeComputer.INSTANCE, true);
+ addFunction(ST_CONVEX_HULL, AGeometryTypeComputer.INSTANCE, true);
+ addFunction(ST_CENTROID, AGeometryTypeComputer.INSTANCE, true);
+ addFunction(ST_IS_VALID, ABooleanTypeComputer.INSTANCE, true);
+ addFunction(ST_REVERSE, AGeometryTypeComputer.INSTANCE, true);
+ addFunction(ST_FLIP_COORDINATES, AGeometryTypeComputer.INSTANCE, true);
+ addFunction(ST_DISTANCE_SPHERE, ADoubleTypeComputer.INSTANCE, true);
+ addFunction(ST_DWITHIN, ABooleanTypeComputer.INSTANCE, true);
+ addFunction(ST_BUFFER, AGeometryTypeComputer.INSTANCE, true);
+
// Binary functions
addFunction(BINARY_HEX_CONSTRUCTOR, ABinaryTypeComputer.INSTANCE_NULLABLE, true);
addFunction(BINARY_BASE64_CONSTRUCTOR, ABinaryTypeComputer.INSTANCE_NULLABLE, true);
(asterixdb) 05/18: [NO ISSUE][HYR][NET] Upgrade gRPC to address Netty CVE
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit bfdc44e92a7c9e1e20530322aa68994d356b878f
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Feb 21 10:31:38 2026 -0500
[NO ISSUE][HYR][NET] Upgrade gRPC to address Netty CVE
See https://nvd.nist.gov/vuln/detail/CVE-2025-67735
Ext-ref: MB-70641
Change-Id: I83fd3948d973949343e953e9e30b534516c0a282
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20924
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
asterixdb/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 3442706d67..96a831e070 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1847,7 +1847,7 @@
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-bom</artifactId>
- <version>1.78.0</version>
+ <version>1.79.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
(asterixdb) 14/18: Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b4f30987634a477e7a6a6262753ba7200a263924
Merge: a2e1e197ca 996a015003
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Tue Feb 24 23:37:23 2026 -0500
Merge branch 'gerrit/trinity' into 'gerrit/morpheus'
* MB-70678: re-resolve cluster address on initial connect retry
Ext-ref: MB-70678
Change-Id: Ia839ec67e5a4f586604ae861ed60097dc1ac812c
.../src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java | 1 +
hyracks-fullstack/pom.xml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
(asterixdb) 01/18: [ASTERIXDB-3677][COMP] Fix nested star operators failure
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8f3b0f649583feec9f00fc57c7f73afab585a0e0
Author: preetham0202 <pr...@couchbase.com>
AuthorDate: Wed Jan 14 18:14:06 2026 +0530
[ASTERIXDB-3677][COMP] Fix nested star operators failure
- user model changes: no
- storage format changes: no
- interface changes: no
Ext-ref: MB-70076
Details: Do not extract record-constructor() functions so that nested star operators don't fail.
Change-Id: I557b9992d047c6fc53bddd5b90ff814bb678cedf
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20864
Tested-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../translator/LangExpressionToPlanTranslator.java | 3 +
.../SqlppExpressionToPlanTranslator.java | 5 +-
.../ASTERIXDB-3682-field-access-in-join.plan | 590 ++++++++++-----------
.../select-star/nested/nested.1.ddl.sqlpp | 32 ++
.../select-star/nested/nested.2.update.sqlpp | 25 +
.../select-star/nested/nested.3.query.sqlpp | 22 +
.../results/select-star/nested/nested.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +
.../DoNotExtractExpressionAnnotation.java | 27 +
.../rules/ExtractCommonExpressionsRule.java | 11 +-
10 files changed, 424 insertions(+), 298 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index b1638d4e71..fee6816ed1 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -1482,6 +1482,9 @@ abstract class LangExpressionToPlanTranslator
throws CompilationException {
AbstractFunctionCallExpression f = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR));
+ if (rc.hasHints()) {
+ f.putAnnotations(rc.getHints());
+ }
f.setSourceLocation(rc.getSourceLocation());
LogicalVariable v1 = context.newVar();
AssignOperator a = new AssignOperator(v1, new MutableObject<>(f));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
index 02f1f0e9f8..d813a4c5a2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -114,6 +114,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCa
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.DoNotExtractExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
@@ -916,8 +917,10 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
CallExpr toObjectExpr = new CallExpr(new FunctionSignature(BuiltinFunctions.TO_OBJECT),
Collections.singletonList(projectionExpr));
toObjectExpr.setSourceLocation(sourceLoc);
+ RecordConstructor recordConstructor = new RecordConstructor(Collections.emptyList());
+ recordConstructor.addHint(DoNotExtractExpressionAnnotation.INSTANCE);
CallExpr ifMissingOrNullExpr = new CallExpr(new FunctionSignature(BuiltinFunctions.IF_MISSING_OR_NULL),
- Arrays.asList(toObjectExpr, new RecordConstructor(Collections.emptyList())));
+ Arrays.asList(toObjectExpr, recordConstructor));
ifMissingOrNullExpr.setSourceLocation(sourceLoc);
return ifMissingOrNullExpr;
}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan
index a44e1cb635..be7926f2c9 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/ASTERIXDB-3682-field-access-in-join.plan
@@ -4,327 +4,325 @@ distribute result [$$446]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
project ([$$446])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$446] <- [if-missing-or-null(to-object($$593), cast($$456))]
+ assign [$$446] <- [if-missing-or-null(to-object($$592), cast({ }))]
-- ASSIGN |PARTITIONED|
- project ([$$456, $$593])
+ project ([$$592])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$529(ASC), $$530(ASC), $$531(ASC), $$532(ASC) ] |PARTITIONED|
- order (ASC, $$529) (ASC, $$530) (ASC, $$531) (ASC, $$532)
- -- STABLE_SORT [$$529(ASC), $$530(ASC), $$531(ASC), $$532(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$528(ASC), $$529(ASC), $$530(ASC), $$531(ASC) ] |PARTITIONED|
+ order (ASC, $$528) (ASC, $$529) (ASC, $$530) (ASC, $$531)
+ -- STABLE_SORT [$$528(ASC), $$529(ASC), $$530(ASC), $$531(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$529, $$530, $$531, $$532] <- [$$593.getField("join_key2"), $$593.getField("wsc"), $$593.getField("lcd"), $$593.getField("wsd")]
+ assign [$$528, $$529, $$530, $$531] <- [$$592.getField("join_key2"), $$592.getField("wsc"), $$592.getField("lcd"), $$592.getField("wsd")]
-- ASSIGN |PARTITIONED|
- project ([$$456, $$593])
+ project ([$$592])
-- STREAM_PROJECT |PARTITIONED|
select (and(like($$455, "%"), not(is-missing($$455))))
-- STREAM_SELECT |PARTITIONED|
- project ([$$456, $$593, $$455])
+ project ([$$592, $$455])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$593, $$455] <- [object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$603, "wsm": $$604, "wsc": $$513, "scKey": $$602, "wpdi": $$509, "bch": $$525, "omaa": $$526, "bco": uppercase(substring($$525, 0, position($$525, "_")))}), object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$513, "scKey": $$602, "wpdi": $$5 [...]
+ assign [$$592, $$455] <- [object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$602, "wsm": $$603, "wsc": $$512, "scKey": $$601, "wpdi": $$508, "bch": $$524, "omaa": $$525, "bco": uppercase(substring($$524, 0, position($$524, "_")))}), object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$512, "scKey": $$601, "wpdi": $$508 [...]
-- ASSIGN |PARTITIONED|
- select (and(like(object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$388.getField("wsc"), "scKey": $$602, "wpdi": $$509, "bch": $$525, "omaa": $$526, "bco": uppercase(substring($$525, 0, position($$525, "_")))}).getField("bch"), "%"), like(object-concat-strict(if-missing-or-null(to-object($$323), cast($$456)), {"lov_cKey": $$466, "wsd": $$388.getField(3), "wsm": [...]
+ select (and(like(object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$388.getField(3), "wsm": $$388.getField(4), "wsc": $$388.getField("wsc"), "scKey": $$601, "wpdi": $$508, "bch": $$524, "omaa": $$525, "bco": uppercase(substring($$524, 0, position($$524, "_")))}).getField("bch"), "%"), like(object-concat-strict(if-missing-or-null(to-object($$323), cast({ })), {"lov_cKey": $$465, "wsd": $$388.getField(3), "wsm": $$ [...]
-- STREAM_SELECT |PARTITIONED|
- project ([$$323, $$466, $$603, $$604, $$513, $$602, $$509, $$388, $$456, $$525, $$526])
+ project ([$$323, $$465, $$602, $$603, $$512, $$601, $$508, $$388, $$524, $$525])
-- STREAM_PROJECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$476, $$509))
- -- HYBRID_HASH_JOIN [$$509][$$476] |PARTITIONED|
+ join (eq($$475, $$508))
+ -- HYBRID_HASH_JOIN [$$508][$$475] |PARTITIONED|
exchange
- -- HASH_PARTITION_EXCHANGE [$$509] |PARTITIONED|
- assign [$$456] <- [{ }]
- -- ASSIGN |PARTITIONED|
- project ([$$323, $$466, $$603, $$604, $$513, $$602, $$509, $$388])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (and(eq($$599, $$601), eq($$466, $$602)))
- -- HYBRID_HASH_JOIN [$$599, $$466][$$601, $$602] |PARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$599, $$466] |PARTITIONED|
- project ([$$323, $$466, $$599])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$uniqueID, $$489))
- -- HYBRID_HASH_JOIN [$$489][$$uniqueID] |PARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$489] |PARTITIONED|
- project ([$$323, $$599, $$489])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$323] <- [{"join_key2": $$599, "lov13_wsx1": substring($$465, 0, 7), "lcd": substring($$465, 0, 10), "lov13_name": $$467, "lov13_uunn": $$489, "lov13_t0_dd": $$600, "lov13_t0_ff": substring($$600, 0, 19)}]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$467, $$599, $$600])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$600] <- [to-string($$u_val.getField("t0"))]
- -- ASSIGN |PARTITIONED|
- select (not(is-null($$599)))
- -- STREAM_SELECT |PARTITIONED|
- project ([$$465, $$489, $$467, $$u_val, $$599])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$599] <- [substring(if-missing-or-null(if-missing-or-null($$461.getField("someId1"), $$461.getField("someId2")), $$461.getField("someId3")), 0, 38)]
+ -- HASH_PARTITION_EXCHANGE [$$508] |PARTITIONED|
+ project ([$$323, $$465, $$602, $$603, $$512, $$601, $$508, $$388])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (and(eq($$598, $$600), eq($$465, $$601)))
+ -- HYBRID_HASH_JOIN [$$598, $$465][$$600, $$601] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$598, $$465] |PARTITIONED|
+ project ([$$323, $$465, $$598])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$uniqueID, $$488))
+ -- HYBRID_HASH_JOIN [$$488][$$uniqueID] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$488] |PARTITIONED|
+ project ([$$323, $$598, $$488])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$323] <- [{"join_key2": $$598, "lov13_wsx1": substring($$464, 0, 7), "lcd": substring($$464, 0, 10), "lov13_name": $$466, "lov13_uunn": $$488, "lov13_t0_dd": $$599, "lov13_t0_ff": substring($$599, 0, 19)}]
+ -- ASSIGN |PARTITIONED|
+ project ([$$464, $$488, $$466, $$598, $$599])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$599] <- [to-string($$u_val.getField("t0"))]
+ -- ASSIGN |PARTITIONED|
+ select (not(is-null($$598)))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$464, $$488, $$466, $$u_val, $$598])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$598] <- [substring(if-missing-or-null(if-missing-or-null($$460.getField("someId1"), $$460.getField("someId2")), $$460.getField("someId3")), 0, 38)]
+ -- ASSIGN |PARTITIONED|
+ assign [$$460] <- [$$u_val.getField("marop")]
-- ASSIGN |PARTITIONED|
- assign [$$461] <- [$$u_val.getField("marop")]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$467, $$u_val])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$u_val <- scan-collection($$290)
- -- UNNEST |PARTITIONED|
- project ([$$465, $$489, $$467, $$290])
- -- STREAM_PROJECT |PARTITIONED|
- subplan {
- aggregate [$$290] <- [listify($$289)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- assign [$$289] <- [{"marop": $$468, "t0": $$488}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- select (and(eq($$487, "iap"), not(is-unknown($$468)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- STREAM_SELECT |LOCAL|
- assign [$$488, $$468, $$487] <- [$$item2.getField("t0"), $$item2.getField("marop"), $$item2.getField("ua")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- unnest $$item2 <- scan-collection($$486) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- UNNEST |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- SUBPLAN |PARTITIONED|
- select (not(or(eq($$467, "p1"), eq($$467, "p2"), eq($$467, "p3"))))
- -- STREAM_SELECT |PARTITIONED|
- project ([$$465, $$489, $$486, $$467])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$470, $$275))
- -- HYBRID_HASH_JOIN [$$275][$$470] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$275] <- [get-item(split($$467, " "), 0)]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$486, $$467])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$486, $$467] <- [$$item.getField("value"), $$item.getField("name")]
- -- ASSIGN |PARTITIONED|
- project ([$$465, $$489, $$item])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$item <- scan-collection(object-pairs($$477))
- -- UNNEST |PARTITIONED|
- project ([$$465, $$489, $$477])
- -- STREAM_PROJECT |PARTITIONED|
- select ($$262)
- -- STREAM_SELECT |PARTITIONED|
- project ([$$262, $$465, $$489, $$477])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- group by ([$$551 := $$457]) decor ([$$465; $$489; $$477]) {
- aggregate [$$262] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- select (not(is-missing($$550))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- STREAM_SELECT |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- PRE_CLUSTERED_GROUP_BY[$$457] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$489, $$465, $$477, $$550, $$457])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- left outer join (eq($$457, $$549))
- -- HYBRID_HASH_JOIN [$$457][$$549] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$489, $$465, $$477, $$457])
- -- STREAM_PROJECT |PARTITIONED|
- select (and(eq($$b.getField("t"), "lovStat"), not(and(eq(len($$597), 1), eq(get-item($$597, 0), "some_str")))))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$597] <- [object-names($$477)]
+ project ([$$464, $$488, $$466, $$u_val])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$u_val <- scan-collection($$290)
+ -- UNNEST |PARTITIONED|
+ project ([$$464, $$488, $$466, $$290])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$290] <- [listify($$289)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ assign [$$289] <- [{"marop": $$467, "t0": $$487}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ select (and(eq($$486, "iap"), not(is-unknown($$467)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |LOCAL|
+ assign [$$487, $$467, $$486] <- [$$item2.getField("t0"), $$item2.getField("marop"), $$item2.getField("ua")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ unnest $$item2 <- scan-collection($$485) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNNEST |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ select (not(or(eq($$466, "p1"), eq($$466, "p2"), eq($$466, "p3"))))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$464, $$488, $$485, $$466])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$469, $$275))
+ -- HYBRID_HASH_JOIN [$$275][$$469] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$275] <- [get-item(split($$466, " "), 0)]
+ -- ASSIGN |PARTITIONED|
+ project ([$$464, $$488, $$485, $$466])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$485, $$466] <- [$$item.getField("value"), $$item.getField("name")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$464, $$488, $$item])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$item <- scan-collection(object-pairs($$476))
+ -- UNNEST |PARTITIONED|
+ project ([$$464, $$488, $$476])
+ -- STREAM_PROJECT |PARTITIONED|
+ select ($$262)
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$262, $$464, $$488, $$476])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ group by ([$$550 := $$456]) decor ([$$464; $$488; $$476]) {
+ aggregate [$$262] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ select (not(is-missing($$549))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- PRE_CLUSTERED_GROUP_BY[$$456] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$488, $$464, $$476, $$549, $$456])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ left outer join (eq($$456, $$548))
+ -- HYBRID_HASH_JOIN [$$456][$$548] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$488, $$464, $$476, $$456])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (and(eq($$b.getField("t"), "lovStat"), not(and(eq(len($$596), 1), eq(get-item($$596, 0), "some_str")))))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$596] <- [object-names($$476)]
+ -- ASSIGN |PARTITIONED|
+ assign [$$488, $$464, $$476] <- [$$b.getField("uunn"), $$b.getField("qwer"), $$b.getField("ihsaa")]
-- ASSIGN |PARTITIONED|
- assign [$$489, $$465, $$477] <- [$$b.getField("uunn"), $$b.getField("qwer"), $$b.getField("ihsaa")]
- -- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$457, $$b] <- test.col2
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$550] <- [true]
- -- ASSIGN |PARTITIONED|
- project ([$$549])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$469, $$251))
- -- HYBRID_HASH_JOIN [$$251][$$469] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$549, $$251])
- -- STREAM_PROJECT |PARTITIONED|
- select ($$260)
- -- STREAM_SELECT |PARTITIONED|
- project ([$$549, $$251, $$260])
- -- STREAM_PROJECT |PARTITIONED|
- subplan {
- aggregate [$$260] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- select (and(eq($$484, "iap"), not(is-unknown($$483)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- STREAM_SELECT |LOCAL|
- assign [$$484, $$483] <- [$$item2.getField("ua"), $$item2.getField("marop")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- unnest $$item2 <- scan-collection($$482) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- UNNEST |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- SUBPLAN |PARTITIONED|
- project ([$$549, $$251, $$482])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$251, $$482] <- [get-item(split($$item.getField("name"), " "), 0), $$item.getField("value")]
- -- ASSIGN |PARTITIONED|
- project ([$$549, $$item])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$item <- scan-collection(object-pairs($$545))
- -- UNNEST |PARTITIONED|
- project ([$$549, $$545])
- -- STREAM_PROJECT |PARTITIONED|
- select (and(not(and(eq(len($$598), 1), eq(get-item($$598, 0), "some_str"))), eq($$546.getField("t"), "lovStat")))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$598] <- [object-names($$545)]
- -- ASSIGN |PARTITIONED|
- assign [$$545] <- [$$546.getField("ihsaa")]
- -- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$549, $$546] <- test.col2
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- BROADCAST_EXCHANGE |PARTITIONED|
- unnest $$469 <- scan-collection(array: [ "X", "Y" ])
- -- UNNEST |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$456, $$b] <- test.col2
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
- exchange
- -- BROADCAST_EXCHANGE |PARTITIONED|
- unnest $$470 <- scan-collection(array: [ "X", "Y" ])
- -- UNNEST |UNPARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$uniqueID] |PARTITIONED|
- project ([$$466, $$uniqueID])
- -- STREAM_PROJECT |PARTITIONED|
- unnest $$uniqueID <- scan-collection($$505)
- -- UNNEST |PARTITIONED|
- project ([$$466, $$505])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$505] <- [array-star($$p.getField("members")).getField("uniqueID")]
- -- ASSIGN |PARTITIONED|
- select (and(eq($$p.getField("r"), "uuu"), eq($$p.getField("t"), "ccc"), not(is-missing($$466))))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$466] <- [$$p.getField("cKey")]
- -- ASSIGN |PARTITIONED|
- project ([$$p])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$p] <- [$$p2]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$549] <- [true]
+ -- ASSIGN |PARTITIONED|
+ project ([$$548])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$468, $$251))
+ -- HYBRID_HASH_JOIN [$$251][$$468] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$548, $$251])
+ -- STREAM_PROJECT |PARTITIONED|
+ select ($$260)
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$548, $$251, $$260])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$260] <- [non-empty-stream()] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ select (and(eq($$483, "iap"), not(is-unknown($$482)))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |LOCAL|
+ assign [$$483, $$482] <- [$$item2.getField("ua"), $$item2.getField("marop")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ unnest $$item2 <- scan-collection($$481) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNNEST |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ project ([$$548, $$251, $$481])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$251, $$481] <- [get-item(split($$item.getField("name"), " "), 0), $$item.getField("value")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$548, $$item])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$item <- scan-collection(object-pairs($$544))
+ -- UNNEST |PARTITIONED|
+ project ([$$548, $$544])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (and(not(and(eq(len($$597), 1), eq(get-item($$597, 0), "some_str"))), eq($$545.getField("t"), "lovStat")))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$597] <- [object-names($$544)]
+ -- ASSIGN |PARTITIONED|
+ assign [$$544] <- [$$545.getField("ihsaa")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$548, $$545] <- test.col2
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ unnest $$468 <- scan-collection(array: [ "X", "Y" ])
+ -- UNNEST |UNPARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ exchange
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ unnest $$469 <- scan-collection(array: [ "X", "Y" ])
+ -- UNNEST |UNPARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$uniqueID] |PARTITIONED|
+ project ([$$465, $$uniqueID])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$uniqueID <- scan-collection($$504)
+ -- UNNEST |PARTITIONED|
+ project ([$$465, $$504])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$504] <- [array-star($$p.getField("members")).getField("uniqueID")]
+ -- ASSIGN |PARTITIONED|
+ select (and(eq($$p.getField("r"), "uuu"), eq($$p.getField("t"), "ccc"), not(is-missing($$465))))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$465] <- [$$p.getField("cKey")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$p])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$p] <- [$$p2]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ replicate
+ -- REPLICATE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$p2])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$458, $$p2] <- test.col1
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$600, $$601] |PARTITIONED|
+ project ([$$602, $$603, $$512, $$601, $$508, $$388, $$600])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$388] <- [{"join_key1": $$600, "wpdi": $$508, "wsc": $$512, "scKey": $$601, "wwlps": $$381, "wsd": $$602, "wsm": $$603}]
+ -- ASSIGN |PARTITIONED|
+ project ([$$512, $$508, $$381, $$603, $$602, $$601, $$600])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$603, $$602, $$601, $$600] <- [substring($$515, 0, 7), substring($$515, 0, 10), uppercase(substring($$512, 0, position($$512, "_"))), substring($$510, 0, 38)]
+ -- ASSIGN |PARTITIONED|
+ project ([$$512, $$508, $$515, $$510, $$381])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$381] <- [listify($$380)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |LOCAL|
+ assign [$$380] <- [$$item.getField("wkps")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |LOCAL|
+ unnest $$item <- scan-collection($$509) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNNEST |LOCAL|
+ nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ project ([$$512, $$508, $$515, $$510, $$509])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$472, $$353))
+ -- HYBRID_HASH_JOIN [$$353][$$472] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$512, $$508, $$515, $$510, $$509, $$353])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (and(eq($$p2.getField("t"), "ak1"), not(is-missing($$508)), not(is-missing($$540))))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$515, $$512, $$510, $$509, $$508] <- [$$540.getField("ssatt"), $$540.getField("stchk"), $$p2.getField("ed"), $$540.getField("intib"), $$540.getField("coaed")]
-- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- replicate
- -- REPLICATE |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$p2])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$459, $$p2] <- test.col1
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- HASH_PARTITION_EXCHANGE [$$601, $$602] |PARTITIONED|
- project ([$$603, $$604, $$513, $$602, $$509, $$388, $$601])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$388] <- [{"join_key1": $$601, "wpdi": $$509, "wsc": $$513, "scKey": $$602, "wwlps": $$381, "wsd": $$603, "wsm": $$604}]
- -- ASSIGN |PARTITIONED|
- project ([$$513, $$509, $$381, $$604, $$603, $$602, $$601])
- -- STREAM_PROJECT |PARTITIONED|
- assign [$$604, $$603, $$602, $$601] <- [substring($$516, 0, 7), substring($$516, 0, 10), uppercase(substring($$513, 0, position($$513, "_"))), substring($$511, 0, 38)]
- -- ASSIGN |PARTITIONED|
- project ([$$513, $$509, $$516, $$511, $$381])
- -- STREAM_PROJECT |PARTITIONED|
- subplan {
- aggregate [$$381] <- [listify($$380)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- AGGREGATE |LOCAL|
- assign [$$380] <- [$$item.getField("wkps")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ASSIGN |LOCAL|
- unnest $$item <- scan-collection($$510) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- UNNEST |LOCAL|
- nested tuple source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- NESTED_TUPLE_SOURCE |LOCAL|
- }
- -- SUBPLAN |PARTITIONED|
- project ([$$513, $$509, $$516, $$511, $$510])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$473, $$353))
- -- HYBRID_HASH_JOIN [$$353][$$473] |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$513, $$509, $$516, $$511, $$510, $$353])
- -- STREAM_PROJECT |PARTITIONED|
- select (and(eq($$p2.getField("t"), "ak1"), not(is-missing($$509)), not(is-missing($$541))))
- -- STREAM_SELECT |PARTITIONED|
- assign [$$516, $$513, $$511, $$510, $$509] <- [$$541.getField("ssatt"), $$541.getField("stchk"), $$p2.getField("ed"), $$541.getField("intib"), $$541.getField("coaed")]
+ assign [$$353, $$540] <- [$$p2.getField("ha"), $$p2.getField("kat")]
-- ASSIGN |PARTITIONED|
- assign [$$353, $$541] <- [$$p2.getField("ha"), $$p2.getField("kat")]
- -- ASSIGN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- replicate
- -- REPLICATE |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$p2])
- -- STREAM_PROJECT |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$459, $$p2] <- test.col1
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange
- -- BROADCAST_EXCHANGE |PARTITIONED|
- unnest $$473 <- scan-collection(array: [ "l1", "l2" ])
- -- UNNEST |UNPARTITIONED|
- empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ replicate
+ -- REPLICATE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$p2])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$458, $$p2] <- test.col1
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ unnest $$472 <- scan-collection(array: [ "l1", "l2" ])
+ -- UNNEST |UNPARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
exchange
- -- HASH_PARTITION_EXCHANGE [$$476] |PARTITIONED|
- project ([$$525, $$526, $$476])
+ -- HASH_PARTITION_EXCHANGE [$$475] |PARTITIONED|
+ project ([$$524, $$525, $$475])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$526, $$525] <- [$$454.getField("omaa"), $$454.getField("qaz")]
+ assign [$$525, $$524] <- [$$454.getField("omaa"), $$454.getField("qaz")]
-- ASSIGN |PARTITIONED|
select (not(is-missing($$454)))
-- STREAM_SELECT |PARTITIONED|
- project ([$$476, $$454])
+ project ([$$475, $$454])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$476, $$454] <- [$$sbr.getField("ed"), $$sbr.getField("tohac")]
+ assign [$$475, $$454] <- [$$sbr.getField("ed"), $$sbr.getField("tohac")]
-- ASSIGN |PARTITIONED|
project ([$$sbr])
-- STREAM_PROJECT |PARTITIONED|
@@ -340,9 +338,9 @@ distribute result [$$446]
-- STREAM_PROJECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$459, $$p2] <- test.col1
+ data-scan []<-[$$458, $$p2] <- test.col1
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.1.ddl.sqlpp
new file mode 100644
index 0000000000..36024d85d8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.1.ddl.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type prim as open {
+ id: bigint
+};
+
+create dataset A(prim)
+primary key id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.2.update.sqlpp
new file mode 100644
index 0000000000..fe77706e1d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.2.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+INSERT INTO A(
+ SELECT VALUE {"id": intVal, "a": intVal%2, "b": intVal%5 , "d": intVal%25}
+ FROM range(1, 2) intVal
+);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.3.query.sqlpp
new file mode 100644
index 0000000000..02ab319123
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/nested/nested.3.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+SELECT sub.* FROM (SELECT a.* FROM A a) sub;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/select-star/nested/nested.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-star/nested/nested.3.adm
new file mode 100644
index 0000000000..125daf12f9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/select-star/nested/nested.3.adm
@@ -0,0 +1,2 @@
+{ "id": 1, "a": 1, "b": 1, "d": 1 }
+{ "id": 2, "a": 0, "b": 2, "d": 2 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index af63f0b1e4..eb46490c2f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -10052,6 +10052,11 @@
<output-dir compare="Text">mixed</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="select-star">
+ <compilation-unit name="nested">
+ <output-dir compare="Text">nested</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="select-star">
<compilation-unit name="no_star">
<output-dir compare="Text">no_star</output-dir>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/DoNotExtractExpressionAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/DoNotExtractExpressionAnnotation.java
new file mode 100644
index 0000000000..fa19e22dbc
--- /dev/null
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/DoNotExtractExpressionAnnotation.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.algebricks.core.algebra.expressions;
+
+public class DoNotExtractExpressionAnnotation implements IExpressionAnnotation {
+ public static final DoNotExtractExpressionAnnotation INSTANCE = new DoNotExtractExpressionAnnotation();
+
+ private DoNotExtractExpressionAnnotation() {
+
+ }
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
index e2ba5571b7..7c4d62c56c 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
@@ -39,6 +39,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.DoNotExtractExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
@@ -260,6 +261,14 @@ public class ExtractCommonExpressionsRule implements IAlgebraicRewriteRule {
return modified;
}
+ private static boolean isExprExtractable(ILogicalExpression expr) {
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return true;
+ }
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ return !funcExpr.hasAnnotation(DoNotExtractExpressionAnnotation.class);
+ }
+
private class CommonExpressionSubstitutionVisitor implements ILogicalExpressionReferenceTransform {
private IOptimizationContext context;
@@ -302,7 +311,7 @@ public class ExtractCommonExpressionsRule implements IAlgebraicRewriteRule {
}
}
} else {
- if (expr.isFunctional() && assignCommonExpression(exprEqClass, expr)) {
+ if (expr.isFunctional() && isExprExtractable(expr) && assignCommonExpression(exprEqClass, expr)) {
modified = true;
//re-obtain the live vars after rewriting in the method called in the if condition
Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
(asterixdb) 08/18: [NO ISSUE][*DB][EXT] Update Apache Hadoop -> 3.4.2 for CVE
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit f42ec662c3741264fb46da525099122dd33fe621
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Feb 22 10:29:47 2026 -0500
[NO ISSUE][*DB][EXT] Update Apache Hadoop -> 3.4.2 for CVE
Ext-ref: MB-70644
Change-Id: I666444a2eec951ff4377dd70eead33dc0c17c0e5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20927
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
---
asterixdb/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index af2dd79636..8c503f54ba 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -84,7 +84,7 @@
<!-- Versions under dependencymanagement or used in many projects via properties -->
<algebricks.version>0.3.8.2-SNAPSHOT</algebricks.version>
<hyracks.version>0.3.8.2-SNAPSHOT</hyracks.version>
- <hadoop.version>3.4.1</hadoop.version>
+ <hadoop.version>3.4.2</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.19.0</log4j.version>
<awsjavasdk.version>2.29.27</awsjavasdk.version>
(asterixdb) 03/18: [ASTERIXDB-3634][EXT]: Validate snapshot on creation and query
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 466bb8b8b5764290d29454aec6a8971f764210e5
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Sat Feb 7 16:02:12 2026 +0300
[ASTERIXDB-3634][EXT]: Validate snapshot on creation and query
Ext-ref: MB-70437
Change-Id: Ib8bb436d966a5b801f677922c3809065167f7f87
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20886
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../asterix/app/translator/QueryTranslator.java | 26 ++---
...er.java => IcebergCatalogStatementHandler.java} | 4 +-
.../helpers/IcebergStatementValidationHelper.java | 74 +++++++++++++
.../src/main/resources/asx_errormsg/en.properties | 2 +-
.../aws/iceberg/IcebergFileRecordReader.java | 2 +-
.../iceberg/IcebergParquetRecordReaderFactory.java | 26 ++---
.../util/iceberg/IcebergSnapshotUtils.java | 115 +++++++++++++++++++++
.../external/util/iceberg/IcebergUtils.java | 28 +----
.../metadata/declared/MetadataProvider.java | 2 +-
9 files changed, 210 insertions(+), 69 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 0c5cd98582..de65cb03c5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -69,7 +69,8 @@ import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.app.result.fields.ResultHandlePrinter;
import org.apache.asterix.app.result.fields.ResultsPrinter;
import org.apache.asterix.app.result.fields.StatusPrinter;
-import org.apache.asterix.app.translator.handlers.CatalogStatementHandler;
+import org.apache.asterix.app.translator.handlers.IcebergCatalogStatementHandler;
+import org.apache.asterix.app.translator.helpers.IcebergStatementValidationHelper;
import org.apache.asterix.column.validation.ColumnPropertiesValidationUtil;
import org.apache.asterix.column.validation.ColumnSupportedTypesValidator;
import org.apache.asterix.common.api.IApplicationContext;
@@ -117,7 +118,6 @@ import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.WriterValidationUtil;
import org.apache.asterix.external.util.iceberg.IcebergConstants;
-import org.apache.asterix.external.util.iceberg.IcebergUtils;
import org.apache.asterix.external.writer.printer.parquet.SchemaConverterVisitor;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.IQueryRewriter;
@@ -199,7 +199,6 @@ import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
import org.apache.asterix.metadata.dataset.hints.DatasetHints;
import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetNodegroupCardinalityHint;
import org.apache.asterix.metadata.declared.MetadataProvider;
-import org.apache.asterix.metadata.entities.Catalog;
import org.apache.asterix.metadata.entities.CompactionPolicy;
import org.apache.asterix.metadata.entities.Database;
import org.apache.asterix.metadata.entities.Dataset;
@@ -1074,7 +1073,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
ExternalDataUtils.normalize(properties);
ExternalDataUtils.validate(properties);
ExternalDataUtils.validateType(properties, (ARecordType) itemType);
- validateIfIcebergTable(properties, mdTxnCtx, sourceLoc);
+ validateIfIcebergTable(metadataProvider, properties, mdTxnCtx, sourceLoc);
validateExternalDatasetProperties(externalDetails, properties, dd.getSourceLocation(), mdTxnCtx,
appCtx, metadataProvider);
datasetDetails = new ExternalDatasetDetails(externalDetails.getAdapter(), properties, new Date(),
@@ -1187,20 +1186,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
return Optional.of(dataset);
}
- protected void validateIfIcebergTable(Map<String, String> properties, MetadataTransactionContext mdTxnCtx,
- SourceLocation srcLoc) throws AlgebricksException {
- if (!IcebergUtils.isIcebergTable(properties)) {
- return;
- }
- IcebergUtils.setDefaultFormat(properties);
- IcebergUtils.validateIcebergTableProperties(properties);
-
- // ensure the specified catalog exists
- String catalogName = properties.get(IcebergConstants.ICEBERG_CATALOG_NAME);
- Catalog catalog = MetadataManager.INSTANCE.getCatalog(mdTxnCtx, catalogName);
- if (catalog == null) {
- throw new CompilationException(ErrorCode.UNKNOWN_CATALOG, srcLoc, catalogName);
- }
+ protected void validateIfIcebergTable(MetadataProvider metadataProvider, Map<String, String> properties,
+ MetadataTransactionContext mdTxnCtx, SourceLocation srcLoc) throws AlgebricksException {
+ IcebergStatementValidationHelper.validateIfIcebergTable(appCtx, metadataProvider, mdTxnCtx, properties, srcLoc);
}
protected boolean isDatasetWithoutTypeSpec(DatasetDecl datasetDecl, ARecordType aRecordType,
@@ -5897,7 +5885,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
protected void handleCatalogStatement(Statement.Kind kind, MetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception {
- CatalogStatementHandler statement = new CatalogStatementHandler(kind, metadataProvider, stmt,
+ IcebergCatalogStatementHandler statement = new IcebergCatalogStatementHandler(kind, metadataProvider, stmt,
Creator.DEFAULT_CREATOR, sessionConfig, lockUtil, lockManager);
statement.handle();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/IcebergCatalogStatementHandler.java
similarity index 98%
rename from asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java
rename to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/IcebergCatalogStatementHandler.java
index 3f88893404..e0e9d02a9f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/IcebergCatalogStatementHandler.java
@@ -50,7 +50,7 @@ import org.apache.asterix.object.base.AdmObjectNode;
import org.apache.asterix.translator.SessionConfig;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-public class CatalogStatementHandler {
+public class IcebergCatalogStatementHandler {
private final Statement.Kind kind;
private final MetadataProvider metadataProvider;
@@ -60,7 +60,7 @@ public class CatalogStatementHandler {
private final IMetadataLockUtil lockUtil;
private final IMetadataLockManager lockManager;
- public CatalogStatementHandler(Statement.Kind kind, MetadataProvider metadataProvider, Statement statement,
+ public IcebergCatalogStatementHandler(Statement.Kind kind, MetadataProvider metadataProvider, Statement statement,
Creator creator, SessionConfig sessionConfig, IMetadataLockUtil lockUtil,
IMetadataLockManager lockManager) {
this.kind = kind;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java
new file mode 100644
index 0000000000..106c93d3fc
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/helpers/IcebergStatementValidationHelper.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.translator.helpers;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.util.iceberg.IcebergConstants;
+import org.apache.asterix.external.util.iceberg.IcebergSnapshotUtils;
+import org.apache.asterix.external.util.iceberg.IcebergUtils;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Catalog;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+public class IcebergStatementValidationHelper {
+
+ public IcebergStatementValidationHelper() {
+ }
+
+ public static void validateIfIcebergTable(ICcApplicationContext appCtx, MetadataProvider metadataProvider,
+ MetadataTransactionContext mdTxnCtx, Map<String, String> collectionProperties, SourceLocation srcLoc)
+ throws AlgebricksException {
+ validateIfIcebergTable(appCtx, metadataProvider, mdTxnCtx, collectionProperties, Collections.emptyMap(),
+ srcLoc);
+ }
+
+ public static void validateIfIcebergTable(ICcApplicationContext appCtx, MetadataProvider metadataProvider,
+ MetadataTransactionContext mdTxnCtx, Map<String, String> collectionProperties,
+ Map<String, String> extraCollectionProperties, SourceLocation srcLoc) throws AlgebricksException {
+ if (!IcebergUtils.isIcebergTable(collectionProperties)) {
+ return;
+ }
+ IcebergUtils.setDefaultFormat(collectionProperties);
+ IcebergUtils.validateIcebergTableProperties(collectionProperties);
+
+ // work on a copy of the properties from now onward to avoid modifying the original collection properties
+ Map<String, String> propertiesCopy = new HashMap<>(collectionProperties);
+
+ // ensure the specified catalog exists
+ String catalogName = propertiesCopy.get(IcebergConstants.ICEBERG_CATALOG_NAME);
+ Catalog catalog = MetadataManager.INSTANCE.getCatalog(mdTxnCtx, catalogName);
+ if (catalog == null) {
+ throw new CompilationException(ErrorCode.UNKNOWN_CATALOG, srcLoc, catalogName);
+ }
+
+ // validate snapshot exists if provided
+ propertiesCopy.putAll(extraCollectionProperties);
+ metadataProvider.addIcebergCatalogPropertiesIfNeeded(appCtx, propertiesCopy);
+ IcebergSnapshotUtils.validateSnapshotExists(propertiesCopy);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index fe1e4ababe..23a66ffbb0 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -346,7 +346,7 @@
1236 = The provided Iceberg table '%1$s' does not exist
1237 = The provided data format '%1$s' is not supported for Iceberg tables
1238 = Iceberg snapshot '%1$s' not found
-1239 = Invalid Iceberg snapshot value: '%1$s'
+1239 = Snapshot '%1$s' not found for table '%2$s'.
1240 = No valid credentials provided to access Biglake Metastore catalog.
1241 = Invalid `%1$s` "%2$s" for frame size=%3$s. value should be >= %4$s * frame size: `%1$s` "%5$s" in %6$s
1242 = Collection '%1$s' is not an Iceberg table external collection.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergFileRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergFileRecordReader.java
index fddf692ea2..76309da25b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergFileRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergFileRecordReader.java
@@ -101,7 +101,7 @@ public class IcebergFileRecordReader implements IRecordReader<Record> {
Snapshot snapshot = table.snapshot(snapshotId);
if (snapshot == null) {
// Snapshot might have been expired/GC'd between compile and runtime
- throw CompilationException.create(ErrorCode.ICEBERG_SNAPSHOT_ID_NOT_FOUND, snapshotId);
+ throw CompilationException.create(ErrorCode.ICEBERG_SNAPSHOT_ID_NOT_FOUND, snapshotId, table.name());
}
this.schemaAtSnapshot = table.schemas().get(snapshot.schemaId());
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergParquetRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergParquetRecordReaderFactory.java
index aa1c52a8f5..705a906d40 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergParquetRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/iceberg/IcebergParquetRecordReaderFactory.java
@@ -21,10 +21,8 @@ package org.apache.asterix.external.input.record.reader.aws.iceberg;
import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_SCHEMA_ID_PROPERTY_KEY;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_SNAPSHOT_ID_PROPERTY_KEY;
-import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY;
+import static org.apache.asterix.external.util.iceberg.IcebergSnapshotUtils.snapshotIdExists;
import static org.apache.asterix.external.util.iceberg.IcebergUtils.getProjectedFields;
-import static org.apache.asterix.external.util.iceberg.IcebergUtils.snapshotIdExists;
-import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
import java.io.Serializable;
import java.util.ArrayList;
@@ -34,6 +32,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.PriorityQueue;
import java.util.Set;
@@ -48,6 +47,7 @@ import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.input.filter.IcebergTableFilterEvaluatorFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.iceberg.IcebergConstants;
+import org.apache.asterix.external.util.iceberg.IcebergSnapshotUtils;
import org.apache.asterix.external.util.iceberg.IcebergUtils;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -65,7 +65,6 @@ import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.io.CloseableIterable;
-import org.apache.iceberg.util.SnapshotUtil;
public class IcebergParquetRecordReaderFactory implements IIcebergRecordReaderFactory<Record> {
@@ -215,22 +214,11 @@ public class IcebergParquetRecordReaderFactory implements IIcebergRecordReaderFa
private TableScan setAndPinScanSnapshot(Map<String, String> configurationCopy, Table table, TableScan scan)
throws CompilationException {
long snapshot;
- String snapshotIdStr = configurationCopy.get(ICEBERG_SNAPSHOT_ID_PROPERTY_KEY);
- String asOfTimestampStr = configurationCopy.get(ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY);
-
- if (snapshotIdStr != null) {
- snapshot = Long.parseLong(snapshotIdStr);
+ Optional<Long> snapshotOptional = IcebergSnapshotUtils.getSnapshotId(configurationCopy, table);
+ if (snapshotOptional.isPresent()) {
+ snapshot = snapshotOptional.get();
if (!snapshotIdExists(table, snapshot)) {
- throw CompilationException.create(ErrorCode.ICEBERG_SNAPSHOT_ID_NOT_FOUND, snapshot);
- }
- } else if (asOfTimestampStr != null) {
- try {
- snapshot = SnapshotUtil.snapshotIdAsOfTime(table, Long.parseLong(asOfTimestampStr));
- if (!snapshotIdExists(table, snapshot)) {
- throw CompilationException.create(ErrorCode.ICEBERG_SNAPSHOT_ID_NOT_FOUND, snapshot);
- }
- } catch (IllegalArgumentException e) {
- throw CompilationException.create(EXTERNAL_SOURCE_ERROR, e, getMessageOrToString(e));
+ throw CompilationException.create(ErrorCode.ICEBERG_SNAPSHOT_ID_NOT_FOUND, snapshot, table.name());
}
} else {
if (table.currentSnapshot() == null) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergSnapshotUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergSnapshotUtils.java
new file mode 100644
index 0000000000..41bd25379b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergSnapshotUtils.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.iceberg;
+
+import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_SNAPSHOT_ID_PROPERTY_KEY;
+import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
+
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.util.SnapshotUtil;
+
+public class IcebergSnapshotUtils {
+
+ public static Optional<Long> validateAndGetSnapshot(Map<String, String> properties) throws CompilationException {
+ String snapshotId = properties.get(ICEBERG_SNAPSHOT_ID_PROPERTY_KEY);
+ String snapshotTimestamp = properties.get(ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY);
+ if (snapshotId != null && snapshotTimestamp != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
+ ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY, ICEBERG_SNAPSHOT_ID_PROPERTY_KEY);
+ }
+
+ try {
+ if (snapshotId != null) {
+ return Optional.of(Long.parseLong(snapshotId));
+ } else if (snapshotTimestamp != null) {
+ return Optional.of(Long.parseLong(snapshotTimestamp));
+ } else {
+ return Optional.empty();
+ }
+ } catch (NumberFormatException e) {
+ throw new CompilationException(ErrorCode.INVALID_ICEBERG_SNAPSHOT_VALUE,
+ snapshotId != null ? snapshotId : snapshotTimestamp);
+ }
+ }
+
+ /**
+ * Returns the snapshot ID from the configuration. If a snapshot timestamp is provided instead, the snapshot ID
+ * for that timestamp is returned instead, otherwise, an error is thrown
+ *
+ * @param properties properties
+ * @param table table the snapshot belongs to
+ * @return snapshot id if exists
+ * @throws CompilationException CompilationException
+ */
+ public static Optional<Long> getSnapshotId(Map<String, String> properties, Table table)
+ throws CompilationException {
+ Optional<Long> snapshotOptional = validateAndGetSnapshot(properties);
+ if (snapshotOptional.isEmpty()) {
+ return snapshotOptional;
+ }
+
+ // if we have a snapshot timestamp, get the snapshot id for that timestamp and validate it instead
+ if (properties.containsKey(IcebergConstants.ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY)) {
+ try {
+ return Optional.of(SnapshotUtil.snapshotIdAsOfTime(table, snapshotOptional.get()));
+ } catch (IllegalArgumentException e) {
+ throw CompilationException.create(EXTERNAL_SOURCE_ERROR, e, getMessageOrToString(e));
+ }
+ }
+ return snapshotOptional;
+ }
+
+ public static void validateSnapshotExists(Map<String, String> catalogAndCollectionProperties)
+ throws AlgebricksException {
+ Optional<Long> snapshotOptional = validateAndGetSnapshot(catalogAndCollectionProperties);
+ if (snapshotOptional.isPresent()) {
+ String namespace = catalogAndCollectionProperties.get(IcebergConstants.ICEBERG_NAMESPACE_PROPERTY_KEY);
+ String tableName = catalogAndCollectionProperties.get(IcebergConstants.ICEBERG_TABLE_NAME_PROPERTY_KEY);
+ Map<String, String> catalogProperties =
+ IcebergUtils.filterCatalogProperties(catalogAndCollectionProperties);
+ Catalog icebergCatalog = IcebergUtils.initializeCatalog(catalogProperties, namespace, false);
+
+ Namespace actualNamespace = Namespace.of(namespace);
+ TableIdentifier tableIdentifier = TableIdentifier.of(actualNamespace, tableName);
+ Table table = icebergCatalog.loadTable(tableIdentifier);
+
+ Optional<Long> snapshotIdOptional = getSnapshotId(catalogAndCollectionProperties, table);
+ if (snapshotIdOptional.isPresent() && !snapshotIdExists(table, snapshotIdOptional.get())) {
+ throw CompilationException.create(ErrorCode.ICEBERG_SNAPSHOT_ID_NOT_FOUND, snapshotIdOptional.get(),
+ tableIdentifier.toString());
+ }
+ }
+ }
+
+ public static boolean snapshotIdExists(Table table, long snapshot) {
+ return table.snapshot(snapshot) != null;
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java
index c9a4e5bd0b..20b5781fc1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/iceberg/IcebergUtils.java
@@ -18,15 +18,13 @@
*/
package org.apache.asterix.external.util.iceberg;
-import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
import static org.apache.asterix.common.exceptions.ErrorCode.UNSUPPORTED_ICEBERG_DATA_FORMAT;
import static org.apache.asterix.external.util.aws.EnsureCloseClientsFactoryRegistry.FACTORY_INSTANCE_ID_KEY;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_AVRO_FORMAT;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_CATALOG_PROPERTY_PREFIX_INTERNAL;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_COLLECTION_PROPERTY_PREFIX_INTERNAL;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_PARQUET_FORMAT;
-import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_SNAPSHOT_ID_PROPERTY_KEY;
-import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_TABLE_FORMAT;
import static org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_WAREHOUSE_PROPERTY_KEY;
@@ -53,7 +51,6 @@ import org.apache.asterix.external.util.iceberg.nessie.NessieUtils;
import org.apache.asterix.external.util.iceberg.rest.RestUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.iceberg.CatalogProperties;
-import org.apache.iceberg.Table;
import org.apache.iceberg.aws.AwsProperties;
import org.apache.iceberg.aws.glue.GlueCatalog;
import org.apache.iceberg.catalog.Catalog;
@@ -163,24 +160,7 @@ public class IcebergUtils {
IcebergConstants.ICEBERG_NAMESPACE_PROPERTY_KEY);
}
- // validate snapshot id and timestamp
- String snapshotId = properties.get(ICEBERG_SNAPSHOT_ID_PROPERTY_KEY);
- String snapshotTimestamp = properties.get(ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY);
- if (snapshotId != null && snapshotTimestamp != null) {
- throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
- ICEBERG_SNAPSHOT_TIMESTAMP_PROPERTY_KEY, ICEBERG_SNAPSHOT_ID_PROPERTY_KEY);
- }
-
- try {
- if (snapshotId != null) {
- Long.parseLong(snapshotId);
- } else if (snapshotTimestamp != null) {
- Long.parseLong(snapshotTimestamp);
- }
- } catch (NumberFormatException e) {
- throw new CompilationException(ErrorCode.INVALID_ICEBERG_SNAPSHOT_VALUE,
- snapshotId != null ? snapshotId : snapshotTimestamp);
- }
+ IcebergSnapshotUtils.validateAndGetSnapshot(properties);
}
/**
@@ -355,10 +335,6 @@ public class IcebergUtils {
}
}
- public static boolean snapshotIdExists(Table table, long snapshot) {
- return table.snapshot(snapshot) != null;
- }
-
public static String[] getProjectedFields(Map<String, String> configuration) throws IOException {
String encoded = configuration.get(ExternalDataConstants.KEY_REQUESTED_FIELDS);
ARecordType projectedRecordType = ExternalDataUtils.getExpectedType(encoded);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 1d6d1a28ce..8963ed1c94 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -1023,7 +1023,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
}
}
- protected void addIcebergCatalogPropertiesIfNeeded(ICcApplicationContext appCtx, Map<String, String> configuration)
+ public void addIcebergCatalogPropertiesIfNeeded(ICcApplicationContext appCtx, Map<String, String> configuration)
throws AlgebricksException {
if (IcebergUtils.isIcebergTable(configuration)) {
String catalogName = configuration.get(IcebergConstants.ICEBERG_CATALOG_NAME);
(asterixdb) 02/18: [ASTERIXDB-3706][COMP] Support IF NOT EXISTS Before and After identifier in DDL Statements
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7f7af27cfd08d75345d33c5e1c8b40ed5380a4ca
Author: Shahrzad Shirazi <Sh...@couchbase.com>
AuthorDate: Tue Feb 17 10:58:06 2026 -0800
[ASTERIXDB-3706][COMP] Support IF NOT EXISTS Before and After identifier in DDL Statements
- user model changes: yes
- storage format changes: no
- interface changes: no
Details: This change will support IF NOT EXISTS both before and after the identifier.
Ext-ref: MB-70013
Change-Id: Ib731eb88db4dfbd9c312cbcb4700029821a85717
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20905
Reviewed-by: Shahrzad Shirazi <Sh...@couchbase.com>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../src/test/resources/runtimets/sqlpp_queries.xml | 2 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 245 ++++++++++++++++++---
2 files changed, 211 insertions(+), 36 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index bc1834fefe..9db0281b6d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -4333,7 +4333,7 @@
<test-case FilePath="ddl/create-index">
<compilation-unit name="create-index-4">
<output-dir compare="Text">create-index-4</output-dir>
- <expected-error>Syntax error: In line 53 >>create primary index if not exists sec_primary_idx1 if not exists on LineItem;<< Encountered <IDENTIFIER> "sec_primary_idx1" at column 37.</expected-error>
+ <expected-error>Syntax error: In line 53 >>create primary index if not exists sec_primary_idx1 if not exists on LineItem;<< Encountered "if" at column 55.</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="ddl/create-index">
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index e04afb68ea..23105fde09 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -1157,7 +1157,13 @@ TypeDecl TypeSpecification(Token startStmtToken) throws ParseException:
TypeExpression typeExpr = null;
}
{
- nameComponents = TypeName() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists()
+ nameComponents = TypeName()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<AS> typeExpr = RecordTypeDef()
{
boolean dgen = false;
@@ -1200,7 +1206,12 @@ NodegroupDecl NodegroupSpecification(Token startStmtToken) throws ParseException
List<Identifier> ncNames = new ArrayList<Identifier>();
}
{
- name = Identifier() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists() name = Identifier()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<ON> tmp = Identifier()
{
ncNames.add(new Identifier(tmp));
@@ -1253,8 +1264,14 @@ CatalogCreateStatement CatalogSpecification(Token startStmtToken) throws ParseEx
{
// this identifier is the soft keyword CATALOG, consume it then continue
<IDENTIFIER>
- catalogName = Identifier()
ifNotExists = IfNotExists()
+ catalogName = Identifier()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
+
<TYPE> catalogType = Identifier()
(LOOKAHEAD({laIdentifier(SOURCE)}) <IDENTIFIER> catalogSource = Identifier())?
<WITH> withRecord = RecordConstructor()
@@ -1308,6 +1325,7 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
Query query = null;
}
{
+ ifNotExists = IfNotExists()
nameComponents = QualifiedName()
(typeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN))?
(
@@ -1322,7 +1340,11 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
}
metaTypeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN)
)?
- ifNotExists = IfNotExists()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
(LOOKAHEAD(3) primaryKeyFieldsWithTypes = PrimaryKeyWithType()
| primaryKeyFields = PrimaryKey())
(<AUTOGENERATED> { autogenerated = true; } )?
@@ -1381,9 +1403,14 @@ DatasetDecl ExternalDatasetSpecification(Token startStmtToken) throws ParseExcep
RecordConstructor withRecord = null;
}
{
+ ifNotExists = IfNotExists()
nameComponents = QualifiedName()
typeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN)
- ifNotExists = IfNotExists()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<USING> adapterName = AdapterName() properties = Configuration()
( <HINTS> hints = Properties() )?
( <WITH> withRecord = RecordConstructor() )?
@@ -1504,7 +1531,13 @@ CreateIndexStatement IndexSpecification(Token startStmtToken) throws ParseExcept
}
{
(
- indexName = Identifier() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists()
+ indexName = Identifier()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<ON> nameComponents = QualifiedName()
<LEFTPAREN> { startElementToken = token; }
indexedElement = IndexedElement(startElementToken) {
@@ -1674,7 +1707,12 @@ CreateIndexStatement PrimaryIndexSpecification(Token startStmtToken) throws Pars
boolean ifNotExists = false;
}
{
- (indexName = Identifier())? ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists() (indexName = Identifier())?
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<ON> nameComponents = QualifiedName() (<TYPE> <BTREE>)?
{
if (indexName == null) {
@@ -1757,7 +1795,12 @@ CreateDatabaseStatement DatabaseSpecification(Token startStmtToken) throws Parse
boolean ifNotExists = false;
}
{
- dbName = Identifier() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists()
+ dbName = Identifier()
+ { if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
{
CreateDatabaseStatement stmt = new CreateDatabaseStatement(new Identifier(dbName), ifNotExists);
return addSourceLocation(stmt, startStmtToken);
@@ -1781,7 +1824,13 @@ CreateDataverseStatement DataverseSpecification(Token startStmtToken) throws Par
boolean ifNotExists = false;
}
{
- ns = Namespace() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists()
+ ns = Namespace()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
{
CreateDataverseStatement stmt = new CreateDataverseStatement(ns, null, ifNotExists);
return addSourceLocation(stmt, startStmtToken);
@@ -1807,8 +1856,13 @@ CreateAdapterStatement AdapterSpecification(Token startStmtToken) throws ParseEx
boolean ifNotExists = false;
}
{
- adapterName = QualifiedName()
ifNotExists = IfNotExists()
+ adapterName = QualifiedName()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<AS> externalIdentifier = FunctionExternalIdentifier()
<AT> libraryName = QualifiedName()
{
@@ -1848,11 +1902,16 @@ CreateViewStatement ViewSpecification(Token startStmtToken, boolean orReplace) t
Namespace currentNamespace = defaultNamespace;
}
{
+ ifNotExists = IfNotExists()
nameComponents = QualifiedName()
(
(
typeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.CLOSED)
- ifNotExists = IfNotExists()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
viewConfigDefaultNull = CastDefaultNull()
{
viewConfig = viewConfigDefaultNull.first;
@@ -1874,9 +1933,14 @@ CreateViewStatement ViewSpecification(Token startStmtToken, boolean orReplace) t
foreignKeyFields.first, refNameComponents.first, refNameComponents.second));
}
)*
- )
+ )
|
- ( ifNotExists = IfNotExists() )
+ ( {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
+ )
)
{
if (orReplace && ifNotExists) {
@@ -1986,6 +2050,7 @@ CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orRe
Namespace currentNamespace = defaultNamespace;
}
{
+ ifNotExists = IfNotExists()
fctName = FunctionName()
{
defaultDataverse = fctName.dataverse;
@@ -1998,7 +2063,11 @@ CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orRe
params = paramsWithArity.second;
signature = new FunctionSignature(fctName.database, fctName.dataverse, fctName.function, arity);
}
- ifNotExists = IfNotExists()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
{
if (orReplace && ifNotExists) {
throw new SqlppParseException(getSourceLocation(token), "Unexpected IF NOT EXISTS");
@@ -2169,7 +2238,11 @@ CreateFeedStatement FeedSpecification(Token startStmtToken) throws ParseExceptio
RecordConstructor withRecord = null;
}
{
- nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists() nameComponents = QualifiedName()
+ { if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<WITH> withRecord = RecordConstructor()
{
try {
@@ -2203,7 +2276,12 @@ CreateFeedPolicyStatement FeedPolicySpecification(Token startStmtToken) throws P
CreateFeedPolicyStatement stmt = null;
}
{
- policyName = Identifier() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists() policyName = Identifier()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<FROM>
(<POLICY> basePolicyName = Identifier() properties = Configuration() (<DEFINITION> definition = ConstantString())?
{
@@ -2245,7 +2323,12 @@ CreateFullTextFilterStatement CreateFullTextFilterSpec(Token startStmtToken) thr
}
{
(
- nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists() nameComponents = QualifiedName()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<AS>
expr = RecordConstructor()
)
@@ -2268,7 +2351,12 @@ CreateFullTextConfigStatement CreateFullTextConfigSpec(Token startStmtToken) thr
}
{
(
- nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists() nameComponents = QualifiedName()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<AS>
expr = RecordConstructor()
)
@@ -2300,7 +2388,12 @@ CreateSynonymStatement SynonymSpecification(Token startStmtToken) throws ParseEx
boolean ifNotExists = false;
}
{
- nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ ifNotExists = IfNotExists() nameComponents = QualifiedName()
+ {
+ if (ifNotExists == false) {
+ ifNotExists = IfNotExists();
+ }
+ }
<FOR> objectNameComponents = QualifiedName()
{
CreateSynonymStatement stmt = new CreateSynonymStatement(nameComponents.first, nameComponents.second.getValue(),
@@ -2495,7 +2588,12 @@ TruncateDatasetStatement TruncateDatasetSpecification(Token startStmtToken) thro
boolean ifExists = false;
}
{
- pairId = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() pairId = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
TruncateDatasetStatement stmt = new TruncateDatasetStatement(pairId.first, pairId.second.getValue(), ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2547,7 +2645,12 @@ DropDatasetStatement DropDatasetSpecification(Token startStmtToken) throws Parse
boolean ifExists = false;
}
{
- pairId = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() pairId = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
DropDatasetStatement stmt = new DropDatasetStatement(pairId.first, pairId.second, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2571,7 +2674,12 @@ ViewDropStatement DropViewSpecification(Token startStmtToken) throws ParseExcept
boolean ifExists = false;
}
{
- pairId = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() pairId = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
ViewDropStatement stmt = new ViewDropStatement(pairId.first, pairId.second, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2595,7 +2703,13 @@ IndexDropStatement DropIndexSpecification(Token startStmtToken) throws ParseExce
boolean ifExists = false;
}
{
- tripleId = DoubleQualifiedName() ifExists = IfExists()
+ ifExists = IfExists() tripleId = DoubleQualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
+
{
IndexDropStatement stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2624,7 +2738,12 @@ FullTextFilterDropStatement DropFullTextFilterSpec(Token startStmtToken) throws
boolean ifExists = false;
}
{
- nameComponents = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() nameComponents = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
stmt = new FullTextFilterDropStatement(nameComponents.first, nameComponents.second.getValue(), ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2638,7 +2757,12 @@ FullTextConfigDropStatement DropFullTextConfigSpec(Token startStmtToken) throws
boolean ifExists = false;
}
{
- nameComponents = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() nameComponents = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
stmt = new FullTextConfigDropStatement(nameComponents.first, nameComponents.second.getValue(), ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2663,7 +2787,13 @@ NodeGroupDropStatement DropNodeGroupSpecification(Token startStmtToken) throws P
boolean ifExists = false;
}
{
- id = Identifier() ifExists = IfExists()
+ ifExists = IfExists() id = Identifier()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
+
{
NodeGroupDropStatement stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2687,7 +2817,12 @@ TypeDropStatement DropTypeSpecification(Token startStmtToken) throws ParseExcept
boolean ifExists = false;
}
{
- pairId = TypeName() ifExists = IfExists()
+ ifExists = IfExists() pairId = TypeName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
TypeDropStatement stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2711,7 +2846,12 @@ DatabaseDropStatement DropDatabaseSpecification(Token startStmtToken) throws Par
boolean ifExists = false;
}
{
- dbName = Identifier() ifExists = IfExists()
+ ifExists = IfExists() dbName = Identifier()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
DatabaseDropStatement stmt = new DatabaseDropStatement(new Identifier(dbName), ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2735,7 +2875,12 @@ DataverseDropStatement DropDataverseSpecification(Token startStmtToken) throws P
boolean ifExists = false;
}
{
- ns = Namespace() ifExists = IfExists()
+ ifExists = IfExists() ns = Namespace()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
DataverseDropStatement stmt = new DataverseDropStatement(ns, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2759,7 +2904,12 @@ AdapterDropStatement DropAdapterSpecification(Token startStmtToken) throws Parse
boolean ifExists = false;
}
{
- adapterName = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() adapterName = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
AdapterDropStatement stmt = new AdapterDropStatement(adapterName.first, adapterName.second.getValue(), ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2783,7 +2933,12 @@ FunctionDropStatement DropFunctionSpecification(Token startStmtToken) throws Par
boolean ifExists = false;
}
{
- funcSig = FunctionSignature() ifExists = IfExists()
+ ifExists = IfExists() funcSig = FunctionSignature()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
FunctionDropStatement stmt = new FunctionDropStatement(funcSig, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2807,7 +2962,12 @@ FeedDropStatement DropFeedSpecification(Token startStmtToken) throws ParseExcept
boolean ifExists = false;
}
{
- pairId = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() pairId = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
FeedDropStatement stmt = new FeedDropStatement(pairId.first, pairId.second, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2831,7 +2991,12 @@ FeedPolicyDropStatement DropFeedPolicySpecification(Token startStmtToken) throws
boolean ifExists = false;
}
{
- pairId = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() pairId = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
FeedPolicyDropStatement stmt = new FeedPolicyDropStatement(pairId.first, pairId.second, ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2855,7 +3020,12 @@ SynonymDropStatement DropSynonymSpecification(Token startStmtToken) throws Parse
boolean ifExists = false;
}
{
- pairId = QualifiedName() ifExists = IfExists()
+ ifExists = IfExists() pairId = QualifiedName()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
{
SynonymDropStatement stmt = new SynonymDropStatement(pairId.first, pairId.second.getValue(), ifExists);
return addSourceLocation(stmt, startStmtToken);
@@ -2882,8 +3052,13 @@ CatalogDropStatement DropCatalogSpecification(Token startStmtToken) throws Parse
boolean isCascade = false;
}
{
- catalogName = Identifier()
ifExists = IfExists()
+ catalogName = Identifier()
+ {
+ if (ifExists == false) {
+ ifExists = IfExists();
+ }
+ }
(LOOKAHEAD({laIdentifier(CASCADE)}) <IDENTIFIER> { isCascade = true; })?
{
CatalogDropStatement stmt = new CatalogDropStatement(catalogName, ifExists, isCascade);
(asterixdb) 15/18: Merge branch 'gerrit/morpheus' into 'master'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 287a9a64165b729fe72058376532d1ce95318827
Merge: 27562debb3 b4f3098763
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Wed Feb 25 00:29:06 2026 -0500
Merge branch 'gerrit/morpheus' into 'master'
* MB-70678: re-resolve cluster address on initial connect retry
Change-Id: Icd363bb50dca7f442c9e3e7c9dd9063767ae1c26
.../src/main/java/org/apache/hyracks/ipc/impl/IPCConnectionManager.java | 1 +
hyracks-fullstack/pom.xml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
(asterixdb) 11/18: Null merge branch 'gerrit/morpheus' into 'master'
Posted by im...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit bee318fb0a53829f22f06d5f40470264615ae8ea
Merge: 466bb8b8b5 1933dfca4b
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Mon Feb 23 15:42:52 2026 -0500
Null merge branch 'gerrit/morpheus' into 'master'
* [ASTERIXDB-3677][COMP] Fix nested star operators failure (MB-70076)
Change-Id: Ic481f8e6fc4f61cf1fbfd6f9db210765dcf58c5b