You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/11/06 17:28:57 UTC
(asterixdb) 04/08: [ASTERIXDB-3661][COMP] Eliminate shared ops when translating COPY TO
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b4ac82cbee7c315b6c8f0f5acd487187c41d0854
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu Oct 23 10:46:35 2025 -0700
[ASTERIXDB-3661][COMP] Eliminate shared ops when translating COPY TO
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Similar to queries and other expressions translated into the logical
plan, COPY TO expression translation should also eliminate shared
operators after the initial translation into the plan.
Ext-ref: MB-69082
Change-Id: Ie07cd81d857fb194f1e4d2fff73eabc2bb401db6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20515
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Peeyush Gupta <pe...@couchbase.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../translator/LangExpressionToPlanTranslator.java | 8 ++-
.../copy-to/union-all/union-all.01.ddl.sqlpp | 35 ++++++++++++
.../copy-to/union-all/union-all.02.update.sqlpp | 66 ++++++++++++++++++++++
.../copy-to/union-all/union-all.03.query.sqlpp | 24 ++++++++
.../copy-to/union-all/union-all.99.ddl.sqlpp | 19 +++++++
.../results/copy-to/union-all/union-all.03.adm | 3 +
.../runtimets/testsuite_external_dataset_s3.xml | 10 ++++
7 files changed, 163 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 98c832a41b..2c61f8c751 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -345,7 +345,9 @@ abstract class LangExpressionToPlanTranslator
DelegateOperator delegateOperator = new DelegateOperator(new CommitOperator(true));
delegateOperator.getInputs().add(new MutableObject<>(upsertOp));
delegateOperator.setSourceLocation(sourceLoc);
- return new ALogicalPlanImpl(new MutableObject<>(delegateOperator));
+ ALogicalPlanImpl plan = new ALogicalPlanImpl(new MutableObject<>(delegateOperator));
+ eliminateSharedOperatorReferenceForPlan(plan);
+ return plan;
} else {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Unrecognized Statement Type",
stmt.getKind());
@@ -481,7 +483,9 @@ abstract class LangExpressionToPlanTranslator
newTop.getInputs().add(new MutableObject<>(writeOperator));
globalPlanRoots.add(new MutableObject<>(newTop));
- return new ALogicalPlanImpl(globalPlanRoots);
+ ALogicalPlanImpl plan = new ALogicalPlanImpl(globalPlanRoots);
+ eliminateSharedOperatorReferenceForPlan(plan);
+ return plan;
}
private String getSeparator(String adapter, boolean isFileStore) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.01.ddl.sqlpp
new file mode 100644
index 0000000000..192d05811c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.01.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+CREATE TYPE OpenType AS {
+};
+
+CREATE COLLECTION col1 PRIMARY KEY (id: int);
+CREATE COLLECTION col2 PRIMARY KEY (id: int);
+
+CREATE EXTERNAL DATASET colCopy(OpenType) USING %adapter% (
+ %template%,
+ %additional_Properties%,
+ ("definition"="%path_prefix%copy-to-result/union-all"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.02.update.sqlpp
new file mode 100644
index 0000000000..2327027a6f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.02.update.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+UPSERT INTO col1 ([
+{"id": 1, "city": "c1"},
+{"id": 2, "city": "c2"},
+{"id": 3, "city": "c2"},
+{"id": 4, "city": "c1"},
+{"id": 6, "city": "c1"},
+{"id": 7, "city": "c3"},
+{"id": 8, "city": "c4"}
+]
+);
+
+UPSERT INTO col2 ([
+{"id": 1, "city": "c1"},
+{"id": 2, "city": "c2"},
+{"id": 3, "city": "c2"},
+{"id": 4, "city": "c1"},
+{"id": 5, "city": "c3"},
+{"id": 6, "city": "c3"},
+{"id": 7, "city": "c5"}
+]
+);
+
+COPY (
+SELECT
+ city,
+ COUNT(col1_id) as nombre_col1,
+ COUNT(col2_id) as nombre_col2
+FROM (
+ SELECT col1.id as col1_id, col1.city as city, NULL as col2_id
+ FROM col1
+ UNION ALL
+ SELECT col2.id as col2_id, col2.city as city, NULL as col1_id
+ FROM col2
+) as combined_data
+GROUP BY city
+HAVING COUNT(col1_id) > 0
+AND COUNT(col2_id) > 0)
+AS b
+TO %adapter%
+PATH (%pathprefix% "copy-to-result", "union-all")
+WITH {
+ %template_colons%,
+ %additionalProperties%
+ "format":"json"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.03.query.sqlpp
new file mode 100644
index 0000000000..d8e022635c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.03.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM colCopy c
+ORDER BY c.city;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.99.ddl.sqlpp
new file mode 100644
index 0000000000..6c301395ab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/union-all/union-all.99.ddl.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/union-all/union-all.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/union-all/union-all.03.adm
new file mode 100644
index 0000000000..37fc8a677e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/union-all/union-all.03.adm
@@ -0,0 +1,3 @@
+{ "nombre_col1": 3, "nombre_col2": 2, "city": "c1" }
+{ "nombre_col1": 2, "nombre_col2": 2, "city": "c2" }
+{ "nombre_col1": 1, "nombre_col2": 2, "city": "c3" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index a2677833b9..fa9ffa158f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -39,6 +39,16 @@
<output-dir compare="Text">simple-write</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="union-all">
+ <placeholder name="adapter" value="S3" />
+ <placeholder name="pathprefix" value="" />
+ <placeholder name="path_prefix" value="" />
+ <placeholder name="additionalProperties" value='"container":"playground",' />
+ <placeholder name="additional_Properties" value='("container"="playground")' />
+ <output-dir compare="Text">union-all</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="copy-to">
<compilation-unit name="query">
<placeholder name="adapter" value="S3" />