You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2021/03/06 16:57:47 UTC
[asterixdb] branch master updated (67fd1f3 -> b99ca94)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 67fd1f3 [NO-ISSUE][IDX] Adding support for array-indexes.
new e052611 [NO ISSUE] Update netty
new f9f467a Merge branch 'gerrit/stabilization-f69489'
new 604a57e Merge branch 'gerrit/stabilization-5949a1cb71'
new 40c99b0 Merge branch 'gerrit/mad-hatter'
new 1b282ef [NO ISSUE][MTD] Add error codes in MetadataNode
new 265e3ce [NO ISSUE][COMP] CREATE FUNCTION IF NOT EXISTS improvements
new 0820015 [NO ISSUE][OTH] Introduce DatasetFullyQualifiedName
new ce0141a [NO ISSUE][MD] Limit metadata identifiers to 255 bytes
new 2ee9720 [NO ISSUE][DOC] Facilitate content reuse
new 1cb3a92 [ASTERIXDB-2841][*DB][STO] Metadata object name restrictions
new 4b35832 [ASTERIXDB-2841][*DB][STO] Encode multiple-dataverse parts as subdirs on disk
new 038a627 [NO ISSUE][HYR] Exception map utility method, fix getBootClassPath() on java >= 9
new 1d95047 Merge branch 'gerrit/mad-hatter'
new 32b72e6 Merge "Merge branch 'gerrit/mad-hatter'" into cheshire-cat
new a4bf145 [NO ISSUE][COMP] Support batch assign for external functions
new e3e4220 [ASTERIXDB-2841][*DB][STO] Prevent ^ as first character in dv first part
new b99ca94 Merge branch 'gerrit/cheshire-cat'
The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../operators/physical/AssignBatchPOperator.java | 103 +++++
.../jobgen/QueryLogicalExpressionJobGen.java | 8 +-
.../asterix/optimizer/base/RuleCollections.java | 6 +-
.../ExtractBatchableExternalFunctionCallsRule.java | 220 +++++++++
.../rules/SetAsterixPhysicalOperatorsRule.java | 61 +++
.../apache/asterix/api/common/APIFramework.java | 4 +-
.../asterix/app/translator/QueryTranslator.java | 8 +-
.../asterix/test/metadata/MetadataManagerTest.java | 88 +++-
.../metadata/MetadataManagerWindowsOsTest.java | 92 ++++
.../bad-ext-function-ddl-1.4.ddl.sqlpp} | 12 +-
.../external-library/mysum/mysum.2.ddl.sqlpp | 2 +-
.../bad-function-ddl-11.4.ddl.sqlpp} | 15 +-
.../resources/runtimets/testsuite_it_sqlpp.xml | 3 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 23 +-
.../asterix/common/exceptions/ErrorCode.java | 26 +-
.../common/metadata/DatasetFullyQualifiedName.java | 64 +++
.../common/storage/DatasetCopyIdentifier.java | 4 +-
.../asterix/common/storage/ResourceReference.java | 61 ++-
.../asterix/common/utils/StoragePathUtil.java | 18 +-
.../src/main/resources/asx_errormsg/en.properties | 32 +-
asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf | 2 +-
.../src/main/markdown/builtins/14_window.md | 22 +-
.../src/main/markdown/builtins/15_bitwise.md | 1 +
.../src/main/markdown/builtins/9_aggregate_sql.md | 4 +-
.../main/markdown/datamodel/datamodel_composite.md | 1 +
.../datamodel/datamodel_primitive_common.md | 2 +-
.../asterix-doc/src/main/markdown/sqlpp/1_intro.md | 3 +-
.../asterix-doc/src/main/markdown/sqlpp/2_expr.md | 289 +++++-------
.../asterix-doc/src/main/markdown/sqlpp/3_query.md | 515 +++++++++------------
.../src/main/markdown/sqlpp/3_query_title.md | 20 -
.../src/main/markdown/sqlpp/4_windowfunctions.md | 136 ++----
.../src/main/markdown/sqlpp/7_ddl_dml.md | 386 +++++++--------
.../src/main/markdown/sqlpp/7_ddl_head.md | 12 +-
.../main/markdown/sqlpp/appendix_2_parameters.md | 7 +-
.../main/markdown/sqlpp/appendix_3_resolution.md | 10 +-
.../main/markdown/sqlpp/appendix_4_manual_data.md | 1 +
.../ExternalFunctionDescriptorProvider.java | 19 +-
.../library/ExternalScalarFunctionDescriptor.java | 14 +-
.../ExternalAssignBatchRuntimeFactory.java | 52 +++
.../apache/asterix/external/util/FeedUtils.java | 4 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 16 +-
.../apache/asterix/metadata/MetadataManager.java | 3 +-
.../org/apache/asterix/metadata/MetadataNode.java | 283 +++++------
.../metadata/declared/MetadataProvider.java | 13 +-
.../apache/asterix/metadata/entities/Dataset.java | 8 +
.../functions/ExternalFunctionCompilerUtil.java | 20 +
.../apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../asterix/metadata/utils/MetadataConstants.java | 9 +-
.../asterix/metadata/utils/MetadataUtil.java | 6 +
.../metadata/utils/SplitsAndConstraintsUtil.java | 9 +-
.../apache/asterix/metadata/utils/TypeUtil.java | 2 +-
.../om/functions/IExternalFunctionDescriptor.java} | 13 +-
asterixdb/asterix-server/pom.xml | 4 +-
.../appended-resources/supplemental-models.xml | 6 +-
....com_netty_netty_netty-4.1.59.Final_NOTICE.txt} | 16 +-
.../core/algebra/base/PhysicalOperatorTag.java | 2 +-
...POperator.java => AbstractAssignPOperator.java} | 36 +-
.../operators/physical/AssignPOperator.java | 84 +---
.../core/algebra/plan/PlanStabilityVerifier.java | 16 +-
.../algebra/util/OperatorManipulationUtil.java | 16 +-
.../rewriter/rules/ConsolidateAssignsRule.java | 9 +
.../rules/SetAlgebricksPhysicalOperatorsRule.java | 2 +-
hyracks-fullstack/hyracks/hyracks-api/pom.xml | 4 +
.../apache/hyracks/api/util/ExceptionUtils.java | 25 +
.../java/org/apache/hyracks/util/MXHelper.java | 3 +-
hyracks-fullstack/pom.xml | 2 +-
66 files changed, 1736 insertions(+), 1223 deletions(-)
create mode 100644 asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/AssignBatchPOperator.java
create mode 100644 asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
create mode 100644 asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerWindowsOsTest.java
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/{mysum/mysum.2.ddl.sqlpp => bad-ext-function-ddl-1/bad-ext-function-ddl-1.4.ddl.sqlpp} (69%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{external-library/mysum/mysum.2.ddl.sqlpp => user-defined-functions/bad-function-ddl-11/bad-function-ddl-11.4.ddl.sqlpp} (69%)
create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DatasetFullyQualifiedName.java
create mode 100644 asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
copy asterixdb/{asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp => asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionDescriptor.java} (69%)
rename asterixdb/src/main/licenses/content/{raw.githubusercontent.com_netty_netty_netty-4.1.48.Final_NOTICE.txt => raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt} (95%)
copy hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/{physical/AssignPOperator.java => AbstractAssignPOperator.java} (75%)
[asterixdb] 14/17: Merge "Merge branch 'gerrit/mad-hatter'" into
cheshire-cat
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 32b72e65f80f8915ce05fb82ee4c377e7e3448b7
Merge: 4b35832 1d95047
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Thu Mar 4 21:38:30 2021 +0000
Merge "Merge branch 'gerrit/mad-hatter'" into cheshire-cat
hyracks-fullstack/hyracks/hyracks-api/pom.xml | 4 ++++
.../apache/hyracks/api/util/ExceptionUtils.java | 25 ++++++++++++++++++++++
.../java/org/apache/hyracks/util/MXHelper.java | 3 +--
3 files changed, 30 insertions(+), 2 deletions(-)
[asterixdb] 04/17: Merge branch 'gerrit/mad-hatter'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 40c99b0ea28c250072538700770465799f4aef0c
Merge: dc8efea 604a57e
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun Feb 28 09:48:14 2021 -0500
Merge branch 'gerrit/mad-hatter'
Change-Id: Ife2f01464d5c6dfaa2977a3384875286d638ffee
asterixdb/asterix-server/pom.xml | 4 ++--
.../src/main/appended-resources/supplemental-models.xml | 6 +++---
...ontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt} | 16 ++++++++--------
hyracks-fullstack/pom.xml | 2 +-
4 files changed, 14 insertions(+), 14 deletions(-)
[asterixdb] 16/17: [ASTERIXDB-2841][*DB][STO] Prevent ^ as first
character in dv first part
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e3e4220c18b5ddccecd2f9697360ae4011df2a1e
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Mar 4 17:41:07 2021 -0500
[ASTERIXDB-2841][*DB][STO] Prevent ^ as first character in dv first part
Change-Id: I5e5aeea9ec38320f7f14877b1ee3bf904519ee66
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10365
Reviewed-by: Ian Maxon <im...@uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../asterix/test/metadata/MetadataManagerTest.java | 19 ++++++++++++++++---
.../asterix/common/storage/ResourceReference.java | 2 +-
.../org/apache/asterix/external/util/FeedUtils.java | 2 +-
.../asterix/metadata/declared/MetadataProvider.java | 3 +++
4 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
index 946d116..eed9ae1 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
@@ -27,6 +27,7 @@ import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
@@ -100,7 +101,14 @@ public class MetadataManagerTest {
StringUtils.repeat(euml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
StringUtils.repeat(auml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
StringUtils.repeat(euml,
- MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2))));
+ MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2))),
+ // #4. single-part name containing continuation char
+ DataverseName
+ .createSinglePartName("abc" + StoragePathUtil.DATAVERSE_CONTINUATION_MARKER + "def"),
+ // #5. multi-part name containing continuation chars
+ DataverseName
+ .create(Arrays.asList("abc" + StoragePathUtil.DATAVERSE_CONTINUATION_MARKER + "def",
+ StoragePathUtil.DATAVERSE_CONTINUATION_MARKER + "def")));
for (DataverseName dvNameOk : dvNameOkList) {
String sql = String.format("create dataverse %s;", dvNameOk);
@@ -172,8 +180,13 @@ public class MetadataManagerTest {
DataverseName.createSinglePartName("abc\u0000def"),
// #2. leading whitespace
DataverseName.createSinglePartName(" abcdef"),
- // #2. file separator
- DataverseName.createSinglePartName("abc" + File.separatorChar + "def"));
+ // #3. file separator
+ DataverseName.createSinglePartName("abc" + File.separatorChar + "def"),
+ // #4. single-part starting with ^
+ DataverseName.createSinglePartName(StoragePathUtil.DATAVERSE_CONTINUATION_MARKER + "abcdef"),
+ // #5. multi-part w/ first part starting with ^
+ DataverseName
+ .create(Arrays.asList(StoragePathUtil.DATAVERSE_CONTINUATION_MARKER + "abcdef", "abcdef")));
ErrorCode invalidNameErrCode = ErrorCode.INVALID_DATABASE_OBJECT_NAME;
for (DataverseName dvNameOk : dvNameBadCharsList) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
index 0e78152..94a5f2a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
@@ -60,7 +60,7 @@ public class ResourceReference {
dataset = tokens[--offset];
List<String> dvParts = new ArrayList<>();
String dvPart = tokens[--offset];
- while (dvPart.charAt(0) == StoragePathUtil.DATAVERSE_CONTINUATION_MARKER) {
+ while (dvPart.codePointAt(0) == StoragePathUtil.DATAVERSE_CONTINUATION_MARKER) {
dvParts.add(dvPart.substring(1));
dvPart = tokens[--offset];
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index 7f3d911..0a91ae8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -89,7 +89,7 @@ public class FeedUtils {
String relPathFile = StoragePathUtil.prepareDataverseComponentName(dataverseName, feedName);
String storagePartitionPath = StoragePathUtil.prepareStoragePartitionPath(partition.getPartitionId());
// Note: feed adapter instances in a single node share the feed logger
- // format: 'storage dir name'/partition_#/dataverse_part1[/ dataverse_part2[...]]/feed/node
+ // format: 'storage dir name'/partition_#/dataverse_part1[^dataverse_part2[...]]/feed/node
File f = new File(storagePartitionPath + File.separator + relPathFile + File.separator + nodeName);
return StoragePathUtil.getFileSplitForClusterPartition(partition, f.getPath());
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index a534954..5f7fdea 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -1751,6 +1751,9 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
int totalLengthUTF8 = 0;
for (String dvNamePart : dataverseName.getParts()) {
validateDatabaseObjectNameImpl(dvNamePart, sourceLoc);
+ if (totalLengthUTF8 == 0 && StoragePathUtil.DATAVERSE_CONTINUATION_MARKER == dvNamePart.codePointAt(0)) {
+ throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, dvNamePart);
+ }
totalLengthUTF8 += dvNamePart.getBytes(StandardCharsets.UTF_8).length;
}
if (totalLengthUTF8 > MetadataConstants.DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8) {
[asterixdb] 09/17: [NO ISSUE][DOC] Facilitate content reuse
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2ee9720eaeac9555800f02fa872d64ac48c4082d
Author: Simon Dew <Si...@couchbase.com>
AuthorDate: Wed Mar 3 15:30:54 2021 +0000
[NO ISSUE][DOC] Facilitate content reuse
- Fix formatting of syntax diagram titles
- Fix line spacing, get rid of horizontal rules
- Fix paragraph indents
- Ensure unique anchor IDs
- Move query intro from query title to query document
- Move SQL++ book link to intro
- Standardize Example headings
- Standardize tabs to spaces and add numeric callouts
- Rewrite paragraphs to avoid markup errors
- Fix links
- Fix admonitions
- Fix punctuation and spelling errors
- Fix entity markup
- Add extra line space at EOF after indented text
- Other miscellaneous fixes
Change-Id: If8e995d9085e85160bbdc02b1d28c331a683f5c4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10305
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../src/main/markdown/builtins/14_window.md | 22 +-
.../src/main/markdown/builtins/15_bitwise.md | 1 +
.../src/main/markdown/builtins/9_aggregate_sql.md | 4 +-
.../main/markdown/datamodel/datamodel_composite.md | 1 +
.../datamodel/datamodel_primitive_common.md | 2 +-
.../asterix-doc/src/main/markdown/sqlpp/1_intro.md | 3 +-
.../asterix-doc/src/main/markdown/sqlpp/2_expr.md | 289 +++++-------
.../asterix-doc/src/main/markdown/sqlpp/3_query.md | 515 +++++++++------------
.../src/main/markdown/sqlpp/3_query_title.md | 20 -
.../src/main/markdown/sqlpp/4_windowfunctions.md | 136 ++----
.../src/main/markdown/sqlpp/7_ddl_dml.md | 384 +++++++--------
.../src/main/markdown/sqlpp/7_ddl_head.md | 12 +-
.../main/markdown/sqlpp/appendix_2_parameters.md | 7 +-
.../main/markdown/sqlpp/appendix_3_resolution.md | 10 +-
.../main/markdown/sqlpp/appendix_4_manual_data.md | 1 +
15 files changed, 575 insertions(+), 832 deletions(-)
diff --git a/asterixdb/asterix-doc/src/main/markdown/builtins/14_window.md b/asterixdb/asterix-doc/src/main/markdown/builtins/14_window.md
index e0e5948..67ead24 100644
--- a/asterixdb/asterix-doc/src/main/markdown/builtins/14_window.md
+++ b/asterixdb/asterix-doc/src/main/markdown/builtins/14_window.md
@@ -209,7 +209,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Syntax:
- FIRST_VALUE(expr) [nulls-treatment] OVER (window-definition)
+ FIRST_VALUE(expr) [nulls-modifier] OVER (window-definition)
* Returns the requested value from the first tuple in the current window
frame, where the window frame is specified by the window definition.
@@ -221,7 +221,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Modifiers:
- * [Nulls Treatment](manual.html#Nulls_treatment): (Optional) Determines how
+ * [NULLS Modifier](manual.html#Window_function_options): (Optional) Determines how
NULL or MISSING values are treated when finding the first value in the
window frame.
@@ -330,7 +330,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Syntax:
- LAG(expr[, offset[, default]]) [nulls-treatment] OVER ([window-partition-clause] [window-order-clause])
+ LAG(expr[, offset[, default]]) [nulls-modifier] OVER ([window-partition-clause] [window-order-clause])
* Returns the value from a tuple at a given offset prior to the current tuple
position.
@@ -353,7 +353,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Modifiers:
- * [Nulls Treatment](manual.html#Nulls_treatment): (Optional) Determines how
+ * [NULLS Modifier](manual.html#Window_function_options): (Optional) Determines how
NULL or MISSING values are treated when finding the offset tuple in the
window partition.
@@ -442,7 +442,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Syntax:
- LAST_VALUE(expr) [nulls-treatment] OVER (window-definition)
+ LAST_VALUE(expr) [nulls-modifier] OVER (window-definition)
* Returns the requested value from the last tuple in the current window frame,
where the window frame is specified by the window definition.
@@ -454,7 +454,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Modifiers:
- * [Nulls Treatment](manual.html#Nulls_treatment): (Optional) Determines how
+ * [NULLS Modifier](manual.html#Window_function_options): (Optional) Determines how
NULL or MISSING values are treated when finding the last tuple in the
window frame.
@@ -573,7 +573,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Syntax:
- LEAD(expr[, offset[, default]]) [nulls-treatment] OVER ([window-partition-clause] [window-order-clause])
+ LEAD(expr[, offset[, default]]) [nulls-modifier] OVER ([window-partition-clause] [window-order-clause])
* Returns the value from a tuple at a given offset ahead of the current tuple
position.
@@ -596,7 +596,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Modifiers:
- * [Nulls Treatment](manual.html#Nulls_treatment): (Optional) Determines how
+ * [NULLS Modifier](manual.html#Window_function_options): (Optional) Determines how
NULL or MISSING values are treated when finding the offset tuple in the
window partition.
@@ -685,7 +685,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Syntax:
- NTH_VALUE(expr, offset) [nthval-from] [nulls-treatment] OVER (window-definition)
+ NTH_VALUE(expr, offset) [from-modifier] [nulls-modifier] OVER (window-definition)
* Returns the requested value from a tuple in the current window frame, where
the window frame is specified by the window definition.
@@ -700,7 +700,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
* Modifiers:
- * [Nth Val From](manual.html#Nth_val_from): (Optional) Determines where the
+ * [FROM Modifier](manual.html#Window_function_options): (Optional) Determines where the
function starts counting the offset.
- `FROM FIRST`: Counting starts at the first tuple in the window frame.
@@ -714,7 +714,7 @@ Window functions cannot appear in the FROM clause clause or LIMIT clause.
The order of the tuples is determined by the window order clause.
If this modifier is omitted, the default is `FROM FIRST`.
- * [Nulls Treatment](manual.html#Nulls_treatment): (Optional) Determines how
+ * [NULLS Modifier](manual.html#Window_function_options): (Optional) Determines how
NULL or MISSING values are treated when finding the offset tuple in the
window frame.
diff --git a/asterixdb/asterix-doc/src/main/markdown/builtins/15_bitwise.md b/asterixdb/asterix-doc/src/main/markdown/builtins/15_bitwise.md
index e86f679..f3b5ba5 100644
--- a/asterixdb/asterix-doc/src/main/markdown/builtins/15_bitwise.md
+++ b/asterixdb/asterix-doc/src/main/markdown/builtins/15_bitwise.md
@@ -651,3 +651,4 @@ The function has an alias `isbitset`.
This returns 10 (1010 in binary) because 3 XOR 6 equals 5 (0101 in binary),
and then 5 XOR 15 equals 10 (1010 in binary).
+
diff --git a/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md b/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md
index 93f6d9a..0657fb0 100644
--- a/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md
+++ b/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md
@@ -25,12 +25,12 @@ The query language also supports standard SQL aggregate functions (e.g., `MIN`,
Note that these are not real functions in the query language, but just syntactic sugars over corresponding
builtin aggregate functions (e.g., `ARRAY_MIN`, `ARRAY_MAX`,
`ARRAY_SUM`, `ARRAY_COUNT`, and `ARRAY_AVG`).
-Refer to [SQL-92 Aggregation Functions](manual.html#SQL-92_aggregation_functions) for details.
+Refer to [Aggregation Pseudo-Functions](manual.html#Aggregation_PseudoFunctions) for details.
The `DISTINCT` keyword may be used with built-in aggregate functions and standard SQL aggregate functions.
It may also be used with aggregate functions used as window functions.
It determines whether the function aggregates all values in the group, or distinct values only.
-Refer to [Aggregation Functions](manual.html#Aggregation_functions) for details.
+Refer to [Function Calls](manual.html#Function_call_expressions) for details.
Aggregate functions may be used as window functions when they are used with an OVER clause.
Refer to [OVER Clauses](manual.html#Over_clauses) for details.
diff --git a/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_composite.md b/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_composite.md
index 92b0374..d8b1742 100644
--- a/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_composite.md
+++ b/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_composite.md
@@ -55,3 +55,4 @@ An example would be
{{"hello", 9328, "world", [1, 2, null]}}
+
diff --git a/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_primitive_common.md b/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_primitive_common.md
index 4c0b2e0..1f77a53 100644
--- a/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_primitive_common.md
+++ b/asterixdb/asterix-doc/src/main/markdown/datamodel/datamodel_primitive_common.md
@@ -20,7 +20,7 @@
## <a id="PrimitiveTypes">Primitive Types</a> ##
### <a id="PrimitiveTypesBoolean">Boolean</a> ###
-`boolean` data type can have one of the two values: _*true*_ or _*false*_.
+`boolean` data type can have one of the two values: *true* or *false*.
* Example:
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/1_intro.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/1_intro.md
index 7b56e12..fda5516 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/1_intro.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/1_intro.md
@@ -17,7 +17,7 @@
! under the License.
!-->
-# <a id="Introduction">1. Introduction</a><font size="3"/>
+# <a id="Introduction">1. Introduction</a>
This document is intended as a reference guide to the full syntax and semantics of
AsterixDB's query language, a SQL-based language for working with semistructured data.
@@ -43,3 +43,4 @@ We list and briefly explain each of the productions in the query grammar, offeri
using two datasets named `customers` and `orders`. Each dataset is a collection of objects.
The contents of the example datasets can be found at the end of this manual in [Appendix 4](#Manual_data).
+For additional reading on SQL++ and more examples, refer to [SQL++ for SQL Users: A Tutorial](https://asterixdb.apache.org/files/SQL_Book.pdf).
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md
index 4453099..de7f442 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md
@@ -23,25 +23,16 @@ SQL++ is an orthogonal language, which means that expressions can serve as opera
In this section, we'll discuss the various kinds of SQL++ expressions.
----
-
-### Expr
-****
-
-
----
+##### Expr
+
## <a id="Operator_expressions">Operator Expressions</a>
Operators perform a specific operation on the input values or expressions.
The syntax of an operator expression is as follows:
----
-
-### OperatorExpr
-****
-
----
+##### OperatorExpr
+
The language provides a full set of operators that you can use within its statements.
Here are the categories of operators:
@@ -89,6 +80,7 @@ values.
| || | String concatenation | SELECT VALUE "ab"||"c"||"d"; |
### <a id="Collection_operators">Collection Operators</a>
+
Collection operators are used for membership tests (IN, NOT IN) or empty collection tests (EXISTS, NOT EXISTS).
| Operator | Purpose | Example |
@@ -99,20 +91,22 @@ Collection operators are used for membership tests (IN, NOT IN) or empty collect
| NOT EXISTS | Check whether a collection is empty | FROM orders AS o <br/>WHERE NOT EXISTS o.items <br/> SELECT *; |
### <a id="Comparison_operators">Comparison Operators</a>
+
Comparison operators are used to compare values.
+
The comparison operators fall into one of two sub-categories: missing value comparisons and regular value comparisons.
-SQL++ (and JSON) has two ways of representing missing information in an object - the presence of the field
+SQL++ (and JSON) has two ways of representing missing information in an object — the presence of the field
with a NULL for its value (as in SQL), and the absence of the field (which JSON permits).
For example, the first of the following objects represents Jack, whose friend is Jill.
-In the other examples, Jake is friendless a la SQL, with a friend field that is NULL, while Joe is friendless in a more
+In the other examples, Jake is friendless à la SQL, with a friend field that is NULL, while Joe is friendless in a more
natural (for JSON) way, i.e., by not having a friend field.
##### Examples
-{"name": "Jack", "friend": "Jill"}
+ {"name": "Jack", "friend": "Jill"}
-{"name": "Jake", "friend": NULL}
+ {"name": "Jake", "friend": NULL}
-{"name": "Joe"}
+ {"name": "Joe"}
The following table enumerates all of the comparison operators available in SQL++.
@@ -151,6 +145,7 @@ The following table summarizes how the missing value comparison operators work.
| IS NOT KNOWN (IS NOT VALUED) | FALSE | TRUE | TRUE |
### <a id="Logical_operators">Logical Operators</a>
+
Logical operators perform logical `NOT`, `AND`, and `OR` operations over Boolean values (`TRUE` and `FALSE`) plus `NULL` and `MISSING`.
| Operator | Purpose | Example |
@@ -186,13 +181,10 @@ The following table demonstrates the results of `NOT` on all possible inputs.
## <a id="Quantified_expressions">Quantified Expressions</a>
----
+##### QuantifiedExpr
+
-### QuantifiedExpr
-****
-##### Synonym for `SOME`: `ANY`
-
----
+Synonym for `SOME`: `ANY`
Quantified expressions are used for expressing existential or universal predicates involving the elements of a collection.
@@ -206,23 +198,21 @@ Otherwise, a type error will be raised if the first expression in a quantified e
##### Examples
- EVERY x IN [ 1, 2, 3 ] SATISFIES x < 3 Returns FALSE
- SOME x IN [ 1, 2, 3 ] SATISFIES x < 3 Returns TRUE
+ EVERY x IN [ 1, 2, 3 ] SATISFIES x < 3 -- ➊
+ SOME x IN [ 1, 2, 3 ] SATISFIES x < 3 -- ➋
+➀ Returns `FALSE`
+➁ Returns `TRUE`
## <a id="Path_expressions">Path Expressions</a>
----
-
-### PathExpr
-****
-
----
+##### PathExpr
+
Components of complex types in the data model are accessed via path expressions. Path access can be applied to the
result of a query expression that yields an instance of a complex type, for example, an object or an array instance.
-For objects, path access is based on field names, and it accesses the field whose name was specified.<br/>
+For objects, path access is based on field names, and it accesses the field whose name was specified.
For arrays, path access is based on (zero-based) array-style indexing. Array indices can be used to retrieve either a single element from an array, or a whole subset of an array. Accessing a single element is achieved by providing a single index argument (zero-based element position), while obtaining a subset of an array is achieved by
providing the `start` and `end` (zero-based) index positions; the returned subset is from position `start` to position
@@ -239,71 +229,56 @@ and also a composition thereof.
##### Examples
- ({"name": "MyABCs", "array": [ "a", "b", "c"]}).array Returns [["a", "b", "c"]]
-
- (["a", "b", "c"])[2] Returns ["c"]
-
- (["a", "b", "c"])[-1] Returns ["c"]
-
- ({"name": "MyABCs", "array": [ "a", "b", "c"]}).array[2] Returns ["c"]
-
- (["a", "b", "c"])[0:2] Returns [["a", "b"]]
-
- (["a", "b", "c"])[0:] Returns [["a", "b", "c"]]
-
- (["a", "b", "c"])[-2:-1] Returns [["b"]]
-
+ ({"name": "MyABCs", "array": [ "a", "b", "c"]}).array -- ➊
+ (["a", "b", "c"])[2] -- ➋
+ (["a", "b", "c"])[-1] -- ➌
+ ({"name": "MyABCs", "array": [ "a", "b", "c"]}).array[2] -- ➍
+ (["a", "b", "c"])[0:2] -- ➎
+ (["a", "b", "c"])[0:] -- ➏
+ (["a", "b", "c"])[-2:-1] -- ➐
+
+➀ Returns `[["a", "b", "c"]]`
+➁ Returns `["c"]`
+➂ Returns `["c"]`
+➃ Returns `["c"]`
+➄ Returns `[["a", "b"]]`
+➅ Returns `[["a", "b", "c"]]`
+➆ Returns `[["b"]]`
## <a id="Primary_expressions">Primary Expressions</a>
----
-
-### PrimaryExpr
-****
-
----
+##### PrimaryExpr
+
-The most basic building block for any expression in SQL++ is PrimaryExpression.
+The most basic building block for any expression in SQL++ is Primary Expression.
This can be a simple literal (constant) value, a reference to a query variable that is in scope, a parenthesized
expression, a function call, or a newly constructed instance of the data model (such as a newly constructed object,
array, or multiset of data model instances).
### <a id="Literals">Literals</a>
----
-
-### Literal
-****
-
----
+##### Literal
+
The simplest kind of expression is a literal that directly represents a value in JSON format. Here are some examples:
-
-
- -42
- "Hello"
- true
- false
- null
-
+ -42
+ "Hello"
+ true
+ false
+ null
Numeric literals may include a sign and an optional decimal point. They may also be written in exponential notation, like this:
-
- 5e2
- -4.73E-2
-
-
+ 5e2
+ -4.73E-2
String literals may be enclosed in either single quotes or double quotes. Inside a string literal, the delimiter character for that string must be "escaped" by a backward slash, as in these examples:
+ "I read \"War and Peace\" today."
+ 'I don\'t believe everything I read.'
-
- "I read \"War and Peace\" today."
- 'I don\'t believe everything I read.'
-
-The table below shows how to escape characters in SQL++
+The table below shows how to escape characters in SQL++.
|Character Name |Escape Method
|----------|----------------|
@@ -317,19 +292,15 @@ The table below shows how to escape characters in SQL++
|CarriageReturn|`\r`|
|EscapeTab|`\t`|
-
-
### <a id="Variable_references">Identifiers and Variable References</a>
-
Like SQL, SQL++ makes use of a language construct called an *identifier*. An identifier starts with an alphabetic character or the underscore character _ , and contains only case-sensitive alphabetic characters, numeric digits, or the special characters _ and $. It is also possible for an identifier to include other special characters, or to be the same as a reserved word, by enclosing the identifier in back-ticks (it's then called a *delimited identifier*). Identifiers are used in varia [...]
- X
- customer_name
- `SELECT`
- `spaces in here`
- `@&#`
-
+ X
+ customer_name
+ `SELECT`
+ `spaces in here`
+ `@&#`
A very simple kind of SQL++ expression is a variable, which is simply an identifier. As in SQL, a variable can be bound to a value, which may be an input dataset, some intermediate result during processing of a query, or the final result of a query. We'll learn more about variables when we discuss queries.
@@ -339,9 +310,9 @@ Note that the SQL++ rules for delimiting strings and identifiers are different f
A parameter reference is an external variable. Its value is provided using the [statement execution API](../api.html#queryservice).
-Parameter references come in two forms, *Named Parameter References* and *Positional Parameter References.*
+Parameter references come in two forms, *Named Parameter References* and *Positional Parameter References*.
-Named paramater references consist of the "$" symbol followed by an identifier or delimited identifier.
+Named parameter references consist of the "$" symbol followed by an identifier or delimited identifier.
Positional parameter references can be either a "$" symbol followed by one or more digits or a "?" symbol. If numbered, positional parameters start at 1. "?" parameters are interpreted as $1 to $N based on the order in which they appear in the statement.
@@ -357,20 +328,15 @@ An error will be raised in the parameter is not bound at query execution time.
### <a id="Parenthesized_expressions">Parenthesized Expressions</a>
----
-
-### ParenthesizedExpr
-****
-
-### Subquery
-****
+##### ParenthesizedExpr
+
----
+##### Subquery
+
An expression can be parenthesized to control the precedence order or otherwise clarify a query.
A [subquery](#Subqueries) (nested [selection](#Union_all)) may also be enclosed in parentheses. For more on these topics please see their respective sections.
-
The following expression evaluates to the value 2.
##### Example
@@ -379,21 +345,17 @@ The following expression evaluates to the value 2.
### <a id="Function_call_expressions">Function Calls</a>
----
-
-### FunctionCall
-****
-
-### OrdinaryFunctionCall
-****
+##### FunctionCall
+
-### AggregateFunctionCall
-****
+##### OrdinaryFunctionCall
+
-### DataverseName
-****
+##### AggregateFunctionCall
+
----
+##### DataverseName
+
Functions are included in SQL++, like most languages, as a way to package useful functionality or to
componentize complicated or reusable computations.
@@ -411,20 +373,16 @@ The following example is a function call expression whose value is 8.
length('a string')
-## <a id="Case_expressions">Case Expressions</a>
+### <a id="Case_expressions">Case Expressions</a>
----
+##### CaseExpr
+
-### CaseExpr
-****
+##### SimpleCaseExpr
+
-### SimpleCaseExpr
-****
-
-### SearchedCaseExpr
-****
-
----
+##### SearchedCaseExpr
+
In a simple `CASE` expression, the query evaluator searches for the first `WHEN` ... `THEN` pair in which the `WHEN` expression is equal to the expression following `CASE` and returns the expression following `THEN`. If none of the `WHEN` ... `THEN` pairs meet this condition, and an `ELSE` branch exists, it returns the `ELSE` expression. Otherwise, `NULL` is returned.
@@ -439,63 +397,54 @@ The following example illustrates the form of a case expression.
### <a id="Constructors">Constructors</a>
----
-
-### Constructor
-****
-
-### ObjectConstructor
-****
+##### Constructor
+
-### ArrayConstructor
-****
+##### ObjectConstructor
+
-### MultisetConstructor
-****
+##### ArrayConstructor
+
----
+##### MultisetConstructor
+
Structured JSON values can be represented by constructors, as in these examples:
- An object: { "name": "Bill", "age": 42 }
- An array: [ 1, 2, "Hello", null ]
-
-In a constructed object, the names of the fields must be strings (either literal strings or computed strings), and an object may not contain any duplicate names. Of course, structured literals can be nested, as in this example:
-
+ { "name": "Bill", "age": 42 } -- ➊
+ [ 1, 2, "Hello", null ] -- ➋
+➀ An object
+➁ An array
- [ {"name": "Bill",
- "address":
- {"street": "25 Main St.",
- "city": "Cincinnati, OH"
- }
- },
- {"name": "Mary",
- "address":
- {"street": "107 Market St.",
- "city": "St. Louis, MO"
- }
- }
- ]
-
+In a constructed object, the names of the fields must be strings (either literal strings or computed strings), and an object may not contain any duplicate names. Of course, structured literals can be nested, as in this example:
+ [ {"name": "Bill",
+ "address":
+ {"street": "25 Main St.",
+ "city": "Cincinnati, OH"
+ }
+ },
+ {"name": "Mary",
+ "address":
+ {"street": "107 Market St.",
+ "city": "St. Louis, MO"
+ }
+ }
+ ]
The array items in an array constructor, and the field-names and field-values in an object constructor, may be represented by expressions. For example, suppose that the variables firstname, lastname, salary, and bonus are bound to appropriate values. Then structured values might be constructed by the following expressions:
-
-
An object:
- {
- "name": firstname || " " || lastname,
- "income": salary + bonus
- }
-
+ {
+ "name": firstname || " " || lastname,
+ "income": salary + bonus
+ }
An array:
- ["1984", lastname, salary + bonus, null]
-
+ ["1984", lastname, salary + bonus, null]
If only one expression is specified instead of the field-name/field-value pair in an object constructor then this
expression is supposed to provide the field value. The field name is then automatically generated based on the
@@ -509,16 +458,16 @@ kind of the value expression as in Q2.1:
##### Example
(Q2.1)
- FROM customers AS c
- WHERE c.custid = "C47"
- SELECT VALUE {c.name, c.rating}
+ FROM customers AS c
+ WHERE c.custid = "C47"
+ SELECT VALUE {c.name, c.rating}
This query outputs:
+ [
+ {
+ "name": "S. Logan",
+ "rating": 625
+ }
+ ]
- [
- {
- "name": "S. Logan",
- "rating": 625
- }
- ]
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
index f4bbc4d..3e70922 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
@@ -17,27 +17,42 @@
! under the License.
!-->
+A *query* can be an expression, or it can be constructed from blocks of code called *query blocks*. A query block may contain several clauses, including `SELECT`, `FROM`, `LET`, `WHERE`, `GROUP BY`, and `HAVING`.
+
+##### Query
+
+
+##### Selection
+
+
+##### QueryBlock
+
+
+##### StreamGenerator
+
+
+Note that, unlike SQL, SQL++ allows the `SELECT` clause to appear either at the beginning or at the end of a query block. For some queries, placing the `SELECT` clause at the end may make a query block easier to understand, because the `SELECT` clause refers to variables defined in the other clauses.
+
## <a id="Select_clauses">SELECT Clause</a>
----
-### SelectClause
-****
+##### SelectClause
+
-### Projection
-****
+##### Projection
+
-##### Synonyms for `VALUE`: `ELEMENT`, `RAW`
----
+Synonyms for `VALUE`: `ELEMENT`, `RAW`
In a query block, the `FROM`, `WHERE`, `GROUP BY`, and `HAVING` clauses (if present) are collectively called the Stream Generator. All these clauses, taken together, generate a stream of tuples of bound variables. The `SELECT` clause then uses these bound variables to generate the output of the query block.
-For example, the clause `FROM customers AS c` scans over the `customers` collection, binding the variable `c` to each customer` object in turn, producing a stream of bindings.
+For example, the clause `FROM customers AS c` scans over the `customers` collection, binding the variable `c` to each `customer` object in turn, producing a stream of bindings.
Here's a slightly more complex example of a stream generator:
##### Example
- FROM customers AS c, orders AS o
- WHERE c.custid = o.custid
+
+ FROM customers AS c, orders AS o
+ WHERE c.custid = o.custid
In this example, the `FROM` clause scans over the customers and orders collections, producing a stream of variable pairs (`c`, `o`) in which `c` is bound to a `customer` object and `o` is bound to an `order` object. The `WHERE` clause then retains only those pairs in which the custid values of the two objects match.
@@ -49,12 +64,12 @@ In SQL++, the `SELECT` clause may appear either at the beginning or at the end o
### <a id="Select_element">SELECT VALUE</a>
-
The `SELECT VALUE` clause returns an array or multiset that contains the results of evaluating the `VALUE` expression, with one evaluation being performed per "binding tuple" (i.e., per `FROM` clause item) satisfying the statement's selection criteria.
If there is no `FROM` clause, the expression after `VALUE` is evaluated once with no binding tuples
(except those inherited from an outer environment).
##### Example
+
(Q3.1)
SELECT VALUE 1;
@@ -66,6 +81,7 @@ Result:
]
##### Example
+
(Q3.2) The following query returns the names of all customers whose rating is above 650.
FROM customers AS c
@@ -74,7 +90,6 @@ Result:
Result:
- RESULT:
[
"T. Cody",
"M. Sinclair",
@@ -87,13 +102,14 @@ Result:
Traditional SQL-style `SELECT` syntax is also supported in SQL++, however the result of a query is not guaranteed to preserve the order of expressions in the `SELECT` clause.
##### Example
+
(Q3.3) The following query returns the names and customers ids of any customers whose rating is 750.
FROM customers AS c
WHERE c.rating = 750
SELECT c.name AS customer_name, c.custid AS customer_id;
-Returns:
+Result:
[
{
@@ -110,25 +126,23 @@ Returns:
As in SQL, the phrase `SELECT *` suggests, "select everything."
-
For each binding tuple in the stream, `SELECT *` produces an output object. For each variable in the binding tuple, the output object contains a field: the name of the field is the name of the variable, and the value of the field is the value of the variable. Essentially, `SELECT *` means, "return all the bound variables, with their names and values."
-
The effect of `SELECT *` can be illustrated by an example based on two small collections named `ages` and `eyes`. The contents of the two collections are as follows:
`ages`:
- [
- { "name": "Bill", "age": 21 },
- { "name": "Sue", "age": 32 }
- ]
+ [
+ { "name": "Bill", "age": 21 },
+ { "name": "Sue", "age": 32 }
+ ]
`eyes`:
- [
- { "name": "Bill", "eyecolor": "brown" },
- { "name": "Sue", "eyecolor": "blue" }
- ]
+ [
+ { "name": "Bill", "eyecolor": "brown" },
+ { "name": "Sue", "eyecolor": "blue" }
+ ]
The following example applies `SELECT *` to a single collection.
@@ -136,17 +150,17 @@ The following example applies `SELECT *` to a single collection.
(Q3.4a) Return all the information in the `ages` collection.
- FROM ages AS a
- SELECT * ;
+ FROM ages AS a
+ SELECT * ;
Result:
- [
- { "a": { "name": "Bill", "age": 21 },
- },
- { "a": { "name": "Sue", "age": 32}
- }
- ]
+ [
+ { "a": { "name": "Bill", "age": 21 },
+ },
+ { "a": { "name": "Sue", "age": 32}
+ }
+ ]
Note that the variable-name `a` appears in the query result. If the `FROM` clause had been simply `FROM ages` (omitting `AS a`), the variable-name in the query result would have been `ages`.
@@ -156,26 +170,28 @@ The next example applies `SELECT *` to a join of two collections.
(Q3.4b) Return all the information in a join of `ages` and `eyes` on matching name fields.
- FROM ages AS a, eyes AS e
- WHERE a.name = e.name
- SELECT * ;
+ FROM ages AS a, eyes AS e
+ WHERE a.name = e.name
+ SELECT * ;
Result:
- [
- { "a": { "name": "Bill", "age": 21 },
- "e": { "name": "Bill", "eyecolor": "Brown" }
- },
- { "a": { "name": "Sue", "age": 32 },
- "e": { "name": "Sue", "eyecolor": "Blue" }
- }
- ]
+ [
+ { "a": { "name": "Bill", "age": 21 },
+ "e": { "name": "Bill", "eyecolor": "Brown" }
+ },
+ { "a": { "name": "Sue", "age": 32 },
+ "e": { "name": "Sue", "eyecolor": "Blue" }
+ }
+ ]
Note that the result of `SELECT *` in SQL++ is more complex than the result of `SELECT *` in SQL.
### <a id="Select_variable_star">SELECT _variable_.*</a>
-SQL++ has an alternative version of `SELECT *` in which the star is preceded by a variable. Whereas `SELECT *` means, "return all the bound variables, with their names and values," `SELECT` *variable* `.*` means "return only the named variable, and return only its value, not its name."
+SQL++ has an alternative version of `SELECT *` in which the star is preceded by a variable.
+
+Whereas the version without a named variable means, "return all the bound variables, with their names and values," `SELECT` *variable* `.*` means "return only the named variable, and return only its value, not its name."
The following example can be compared with (Q3.4a) to see the difference between the two versions of `SELECT *`:
@@ -183,37 +199,39 @@ The following example can be compared with (Q3.4a) to see the difference between
(Q3.4c) Return all information in the `ages` collection.
- FROM ages AS a
- SELECT a.*
+ FROM ages AS a
+ SELECT a.*
Result:
- [
- { "name": "Bill", "age": 21 },
- { "name": "Sue", "age": 32 }
- ]
+ [
+ { "name": "Bill", "age": 21 },
+ { "name": "Sue", "age": 32 }
+ ]
-Note that, for queries over a single collection, `SELECT` *variable* `.*` returns a simpler result and therefore may be preferable to `SELECT *`. In fact, `SELECT` *variable* `.*`, like `SELECT *` in SQL, is equivalent to a `SELECT` clause that enumerates all the fields of the collection, as in (Q3.4d):
+Note that, for queries over a single collection, `SELECT` *variable* `.*` returns a simpler result and therefore may be preferable to `SELECT *`.
+
+In fact, `SELECT` *variable* `.*`, like `SELECT *` in SQL, is equivalent to a `SELECT` clause that enumerates all the fields of the collection, as in (Q3.4d):
##### Example
(Q3.4d) Return all the information in the `ages` collection.
- FROM ages AS a
- SELECT a.name, a.age
+ FROM ages AS a
+ SELECT a.name, a.age
(same result as (Q3.4c))
- `SELECT` *variable* `.*` has an additional application. It can be used to return all the fields of a nested object. To illustrate this use, we will use the `customers` dataset in the example database (see Appendix 4).
+`SELECT` *variable* `.*` has an additional application. It can be used to return all the fields of a nested object. To illustrate this use, we will use the `customers` dataset in the example database — see [Appendix 4](#Manual_data).
##### Example
+
(Q3.4e) In the `customers` dataset, return all the fields of the `address` objects that have zipcode "02340".
FROM customers AS c
WHERE c.address.zipcode = "02340"
SELECT address.* ;
-
Result:
[
@@ -224,8 +242,8 @@ Result:
}
]
-
### <a id="Select_distinct">SELECT DISTINCT</a>
+
The `DISTINCT` keyword is used to eliminate duplicate items from the results of a query block.
##### Example
@@ -253,6 +271,7 @@ Result:
]
### <a id="Unnamed_projections">Unnamed Projections</a>
+
Similar to standard SQL, the query language supports unnamed projections (a.k.a, unnamed `SELECT` clause items), for which names are generated rather than user-provided.
Name generation has three cases:
@@ -284,7 +303,8 @@ Result:
In the result, `$1` is the generated name for `o.orderno % 1000`, while `order_date` is the generated name for `o.order_date`. It is good practice, however, to not rely on the randomly generated names which can be confusing and irrelevant. Instead, practice good naming conventions by providing a meaningful and concise name which properly describes the selected item.
### <a id="Abbreviated_field_access_expressions">Abbreviated Field Access Expressions</a>
-As in standard SQL, field access expressions can be abbreviated when there is no ambiguity. In the next example, the variable `o` is the only possible variable reference for fields `orderno` and `order_date` and thus could be omitted in the query. This practice is not recommended, however, as queries may have fields (such as `custid`) which can be present in multiple datasets. More information on abbbreviated field access can be found in the appendix section on Variable Resolution.
+
+As in standard SQL, field access expressions can be abbreviated when there is no ambiguity. In the next example, the variable `o` is the only possible variable reference for fields `orderno` and `order_date` and thus could be omitted in the query. This practice is not recommended, however, as queries may have fields (such as `custid`) which can be present in multiple datasets. More information on abbreviated field access can be found in the appendix section on Variable Resolution.
##### Example
@@ -307,28 +327,24 @@ Result:
}
]
+## <a id="From_clauses">FROM Clause</a>
-## <a id="From_clauses">FROM clause</a>
-
----
+##### FromClause
+
-### FromClause
-****
+##### FromTerm
+
-### FromTerm
-****
+##### NamedExpr
+
-### NamedExpr
-****
+##### JoinStep
+
-### JoinStep
-****
+##### UnnestStep
+
-### UnnestStep
-****
-
-##### Synonyms for `UNNEST`: `CORRELATE`, `FLATTEN`
----
+Synonyms for `UNNEST`: `CORRELATE`, `FLATTEN`
The purpose of a `FROM` clause is to iterate over a collection, binding a variable to each item in turn. Here's a query that iterates over the `customers` dataset, choosing certain customers and returning some of their attributes.
@@ -336,19 +352,13 @@ The purpose of a `FROM` clause is to iterate over a collection, binding a variab
(Q3.8) List the customer ids and names of the customers in zipcode 63101, in order by their customer IDs.
-
-
FROM customers
WHERE address.zipcode = "63101"
SELECT custid AS customer_id, name
ORDER BY customer_id;
-
-
Result:
-
-
[
{
"customer_id": "C13",
@@ -373,35 +383,26 @@ You may also provide an explicit iteration variable, as in this version of the s
(Q3.9) Alternative version of Q3.8 (same result).
-
-
FROM customers AS c
WHERE c.address.zipcode = "63101"
SELECT c.custid AS customer_id, c.name
ORDER BY customer_id;
-
In Q3.9, the variable `c` is bound to each `customer` object in turn as the query iterates over the `customers` dataset. An explicit iteration variable can be used to identify the fields of the referenced object, as in `c.name` in the `SELECT` clause of Q3.9. When referencing a field of an object, the iteration variable can be omitted when there is no ambiguity. For example, `c.name` could be replaced by `name` in the `SELECT` clause of Q3.9. That's why field-names like `name` and `custi [...]
-
-
In the examples above, the `FROM` clause iterates over the objects in a dataset. But in general, a `FROM` clause can iterate over any collection. For example, the objects in the `orders` dataset each contain a field called `items`, which is an array of nested objects. In some cases, you will write a `FROM` clause that iterates over a nested array like `items`.
-
The stream of objects (more accurately, variable bindings) that is produced by the `FROM` clause does not have any particular order. The system will choose the most efficient order for the iteration. If you want your query result to have a specific order, you must use an `ORDER BY` clause.
-
It's good practice to specify an explicit iteration variable for each collection in the `FROM` clause, and to use these variables to qualify the field-names in other clauses. Here are some reasons for this convention:
+ - It's nice to have different names for the collection as a whole and an object in the collection. For example, in the clause `FROM customers AS c`, the name `customers` represents the dataset and the name `c` represents one object in the dataset.
-- It's nice to have different names for the collection as a whole and an object in the collection. For example, in the clause `FROM customers AS c`, the name `customers` represents the dataset and the name `c` represents one object in the dataset.
-
-- In some cases, iteration variables are required. For example, when joining a dataset to itself, distinct iteration variables are required to distinguish the left side of the join from the right side.
-
-- In a subquery it's sometimes necessary to refer to an object in an outer query block (this is called a *correlated subquery*). To avoid confusion in correlated subqueries, it's best to use explicit variables.
+ - In some cases, iteration variables are required. For example, when joining a dataset to itself, distinct iteration variables are required to distinguish the left side of the join from the right side.
+ - In a subquery it's sometimes necessary to refer to an object in an outer query block (this is called a *correlated subquery*). To avoid confusion in correlated subqueries, it's best to use explicit variables.
-### <a id="Left_outer_unnests">Joins</a>
+### <a id="Join_clauses">Joins</a>
A `FROM` clause gets more interesting when there is more than one collection involved. The following query iterates over two collections: `customers` and `orders`. The `FROM` clause produces a stream of binding tuples, each containing two variables, `c` and `o`. In each binding tuple, `c` is bound to an object from `customers`, and `o` is bound to an object from `orders`. Conceptually, at this point, the binding tuple stream contains all possible pairs of a customer and an order (this is [...]
@@ -409,7 +410,6 @@ A `FROM` clause gets more interesting when there is more than one collection inv
(Q3.10) Create a packing list for order number 1001, showing the customer name and address and all the items in the order.
-
FROM customers AS c, orders AS o
WHERE c.custid = o.custid
AND o.orderno = 1001
@@ -418,7 +418,6 @@ A `FROM` clause gets more interesting when there is more than one collection inv
c.address,
o.items AS items_ordered;
-
Result:
[
@@ -445,16 +444,12 @@ Result:
}
]
-
-
Q3.10 is called a *join query* because it joins the `customers` collection and the `orders` collection, using the join condition `c.custid = o.custid`. In SQL++, as in SQL, you can express this query more explicitly by a `JOIN` clause that includes the join condition, as follows:
-
##### Example
(Q3.11) Alternative statement of Q3.10 (same result).
-
FROM customers AS c JOIN orders AS o
ON c.custid = o.custid
WHERE o.orderno = 1001
@@ -463,10 +458,8 @@ Q3.10 is called a *join query* because it joins the `customers` collection and t
c.address,
o.items AS items_ordered;
-
Whether you express the join condition in a `JOIN` clause or in a `WHERE` clause is a matter of taste; the result is the same. This manual will generally use a comma-separated list of collection-names in the `FROM` clause, leaving the join condition to be expressed elsewhere. As we'll soon see, in some query blocks the join condition can be omitted entirely.
-
There is, however, one case in which an explicit `JOIN` clause is necessary. That is when you need to join collection A to collection B, and you want to make sure that every item in collection A is present in the query result, even if it doesn't match any item in collection B. This kind of query is called a *left outer join*, and it is illustrated by the following example.
##### Example
@@ -479,8 +472,6 @@ There is, however, one case in which an explicit `JOIN` clause is necessary. Tha
SELECT c.custid, c.name, o.orderno, o.order_date
ORDER BY c.custid, o.order_date;
-
-
Result:
[
@@ -514,7 +505,6 @@ Result:
}
]
-
As you can see from the result of this left outer join, our data includes four orders from customer T. Cody, but no orders from customer M. Sinclair. The behavior of left outer join in SQL++ is different from that of SQL. SQL would have provided M. Sinclair with an order in which all the fields were `null`. SQL++, on the other hand, deals with schemaless data, which permits it to simply omit the order fields from the outer join.
Now we're ready to look at a new kind of join that was not provided (or needed) in original SQL. Consider this query:
@@ -523,8 +513,6 @@ Now we're ready to look at a new kind of join that was not provided (or needed)
(Q3.13) For every case in which an item is ordered in a quantity greater than 100, show the order number, date, item number, and quantity.
-
-
FROM orders AS o, o.items AS i
WHERE i.qty > 100
SELECT o.orderno, o.order_date, i.itemno AS item_number,
@@ -554,11 +542,8 @@ Result:
}
]
-
Q3.13 illustrates a feature called *left-correlation* in the `FROM` clause. Notice that we are joining `orders`, which is a dataset, to `items`, which is an array nested inside each order. In effect, for each order, we are unnesting the `items` array and joining it to the `order` as though it were a separate collection. For this reason, this kind of query is sometimes called an *unnesting query*. The keyword `UNNEST` may be used whenever left-correlation is used in a `FROM` clause, as sh [...]
-
-
##### Example
(Q3.14) Alternative statement of Q3.13 (same result).
@@ -569,38 +554,27 @@ Q3.13 illustrates a feature called *left-correlation* in the `FROM` clause. Noti
i.qty AS quantity
ORDER BY o.orderno, item_number;
-
The results of Q3.13 and Q3.14 are exactly the same. `UNNEST` serves as a reminder that left-correlation is being used to join an object with its nested items. The join condition in Q3.14 is expressed by the left-correlation: each order `o` is joined to its own items, referenced as `o.items`. The result of the `FROM` clause is a stream of binding tuples, each containing two variables, `o` and `i`. The variable `o` is bound to an order and the variable `i` is bound to one item inside that order.
Like `JOIN`, `UNNEST` has a `LEFT OUTER` option. Q3.14 could have specified:
-
-
- FROM orders AS o LEFT OUTER UNNEST o.items AS i
-
-
+ FROM orders AS o LEFT OUTER UNNEST o.items AS i
In this case, orders that have no nested items would appear in the query result.
## <a id="Let_clauses">LET Clause</a>
----
-
-### LetClause
-****
+##### LetClause
+
-##### Synonyms for `LET`: `LETTING`
----
+Synonyms for `LET`: `LETTING`
-
- `LET` clauses can be useful when a (complex) expression is used several times within a query, allowing it to be written once to make the query more concise. The word `LETTING` can also be used, although this is not as common. The next query shows an example.
+`LET` clauses can be useful when a (complex) expression is used several times within a query, allowing it to be written once to make the query more concise. The word `LETTING` can also be used, although this is not as common. The next query shows an example.
##### Example
(Q3.15) For each item in an order, the revenue is defined as the quantity times the price of that item. Find individual items for which the revenue is greater than 5000. For each of these, list the order number, item number, and revenue, in descending order by revenue.
-
-
FROM orders AS o, o.items AS i
LET revenue = i.qty * i.price
WHERE revenue > 5000
@@ -627,58 +601,47 @@ Result:
}
]
-
The expression for computing revenue is defined once in the `LET` clause and then used three times in the remainder of the query. Avoiding repetition of the revenue expression makes the query shorter and less prone to errors.
## <a id="Where_having_clauses">WHERE Clause</a>
----
-
-### WhereClause
-****
-
-
----
+##### WhereClause
+
The purpose of a `WHERE` clause is to operate on the stream of binding tuples generated by the `FROM` clause, filtering out the tuples that do not satisfy a certain condition. The condition is specified by an expression based on the variable names in the binding tuples. If the expression evaluates to true, the tuple remains in the stream; if it evaluates to anything else, including `null` or `missing`, it is filtered out. The surviving tuples are then passed along to the next clause to b [...]
Often, the expression in a `WHERE` clause is some kind of comparison like `quantity > 100`. However, any kind of expression is allowed in a `WHERE` clause. The only thing that matters is whether the expression returns `true` or not.
-
-## <a id="Group_By_clauses">Grouping</a>
+## <a id="Grouping">Grouping</a>
Grouping is especially important when manipulating hierarchies like the ones that are often found in JSON data. Often you will want to generate output data that includes both summary data and line items within the summaries. For this purpose, SQL++ supports several important extensions to the traditional grouping features of SQL. The familiar `GROUP BY` and `HAVING` clauses are still there, and they are joined by a new clause called `GROUP AS`. We'll illustrate these clauses by a series [...]
-### <a id="Left_outer_unnests">GROUP BY Clause</a>
-
----
-
-### GroupByClause
-****
+### <a id="Group_By_clauses">GROUP BY Clause</a>
-### GroupingElement
-****
+##### GroupByClause
+
-### OrdinaryGroupingSet
-****
+##### GroupingElement
+
-### NamedExpr
-****
+##### OrdinaryGroupingSet
+
----
+##### NamedExpr
+
We'll begin our discussion of grouping with an example from ordinary SQL.
##### Example
- (Q3.16) List the number of orders placed by each customer who has placed an order.
+(Q3.16) List the number of orders placed by each customer who has placed an order.
SELECT o.custid, COUNT(o.orderno) AS `order count`
FROM orders AS o
GROUP BY o.custid
ORDER BY o.custid;
- Result:
+Result:
[
{
@@ -702,13 +665,14 @@ We'll begin our discussion of grouping with an example from ordinary SQL.
"custid": "C41"
}
]
- The input to a `GROUP BY` clause is the stream of binding tuples generated by the `FROM` and `WHERE`clauses. In this query, before grouping, the variable `o` is bound to each object in the `orders` collection in turn.
- SQL++ evaluates the expression in the `GROUP BY` clause, called the grouping expression, once for each of the binding tuples. It then organizes the results into groups in which the grouping expression has a common value (as defined by the `=` operator). In this example, the grouping expression is `o.custid`, and each of the resulting groups is a set of `orders` that have the same `custid`. If necessary, a group is formed for `orders` in which `custid` is `null`, and another group is for [...]
+The input to a `GROUP BY` clause is the stream of binding tuples generated by the `FROM` and `WHERE`clauses. In this query, before grouping, the variable `o` is bound to each object in the `orders` collection in turn.
+
+SQL++ evaluates the expression in the `GROUP BY` clause, called the grouping expression, once for each of the binding tuples. It then organizes the results into groups in which the grouping expression has a common value (as defined by the `=` operator). In this example, the grouping expression is `o.custid`, and each of the resulting groups is a set of `orders` that have the same `custid`. If necessary, a group is formed for `orders` in which `custid` is `null`, and another group is form [...]
In the `GROUP BY`clause, you may optionally define an alias for the grouping expression. For example, in Q3.16, you could have written `GROUP BY o.custid AS cid`. The alias `cid` could then be used in place of the grouping expression in later clauses. In cases where the grouping expression contains an operator, it is especially helpful to define an alias (for example, `GROUP BY salary + bonus AS pay)`.
- Q3.16 had a single grouping expression, `o.custid`. If a query has multiple grouping expressions, the combination of grouping expressions is evaluated for every binding tuple, and the stream of binding tuples is partitioned into groups that have values in common for all of the grouping expressions. We'll see an example of such a query in Q3.18.
+Q3.16 had a single grouping expression, `o.custid`. If a query has multiple grouping expressions, the combination of grouping expressions is evaluated for every binding tuple, and the stream of binding tuples is partitioned into groups that have values in common for all of the grouping expressions. We'll see an example of such a query in Q3.18.
After grouping, the number of binding tuples is reduced: instead of a binding tuple for each of the input objects, there is a binding tuple for each group. The grouping expressions (identified by their aliases, if any) are bound to the results of their evaluations. However, all the non-grouping fields (that is, fields that were not named in the grouping expressions), are accessible only in a special way: as an argument of one of the special aggregation pseudo-functions such as: `SUM`, `A [...]
@@ -717,14 +681,14 @@ You may notice that the results of Q3.16 do not include customers who have no `o
##### Example
- (Q3.17) List the number of orders placed by each customer including those customers who have placed no orders.
+(Q3.17) List the number of orders placed by each customer including those customers who have placed no orders.
SELECT c.custid, c.name, COUNT(o.orderno) AS `order count`
FROM customers AS c LEFT OUTER JOIN orders AS o ON c.custid = o.custid
GROUP BY c.custid, c.name
ORDER BY c.custid;
- Result:
+Result:
[
{
@@ -764,7 +728,6 @@ You may notice that the results of Q3.16 do not include customers who have no `o
}
]
-
Notice in Q3.17 what happens when the special aggregation function `COUNT` is applied to a collection that does not exist, such as the orders of M. Sinclair: it returns zero. This behavior is unlike that of the other special aggregation functions `SUM`, `AVG`, `MAX`, and `MIN`, which return `null` if their operand does not exist. This should make you cautious about the `COUNT` function: If it returns zero, that may mean that the collection you are counting has zero members, or that it do [...]
Q3.17 also shows how a query block can have more than one grouping expression. In general, the `GROUP BY`clause produces a binding tuple for each different combination of values for the grouping expressions. In Q3.17, the `c.custid` field uniquely identifies a customer, so adding `c.name` as a grouping expression does not result in any more groups. Nevertheless, `c.name` must be included as a grouping expression if it is to be referenced outside (after) the `GROUP BY` clause. If `c.name` [...]
@@ -807,7 +770,6 @@ Q3.19 also shows how a `LET` clause can be used after a `GROUP BY` clause to def
(Q3.19) For each order, define the total revenue of the order as the sum of quantity times price for all the items in that order. List the total revenue for all the orders placed by the customer with id "C13", in descending order by total revenue.
-
FROM orders as o, o.items as i
WHERE o.custid = "C13"
GROUP BY o.orderno
@@ -832,21 +794,15 @@ Result:
}
]
-### <a id="Left_outer_unnests">HAVING Clause</a>
-
----
+### <a id="Having_clauses">HAVING Clause</a>
-### HavingClause
-****
-
-
----
+##### HavingClause
+
The `HAVING` clause is very similar to the `WHERE` clause, except that it comes after `GROUP BY` and applies a filter to groups rather than to individual objects. Here's an example of a `HAVING` clause that filters orders by applying a condition to their nested arrays of `items`.
By adding a `HAVING` clause to Q3.19, we can filter the results to include only those orders whose total revenue is greater than 1000, as shown in Q3.22.
-
##### Example
(Q3.20) Modify Q3.19 to include only orders whose total revenue is greater than 5000.
@@ -872,27 +828,30 @@ Result:
SQL provides several special functions for performing aggregations on groups including: `SUM`, `AVG`, `MAX`, `MIN`, and `COUNT` (some implementations provide more). These same functions are supported in SQL++. However, it's worth spending some time on these special functions because they don't behave like ordinary functions. They are called "pseudo-functions" here because they don't evaluate their operands in the same way as ordinary functions. To see the difference, consider these two e [...]
-##### Example 1:
+##### Example 1
SELECT LENGTH(name) FROM customers
- In Example 1, `LENGTH` is an ordinary function. It simply evaluates its operand (name) and then returns a result computed from the operand.
+In Example 1, `LENGTH` is an ordinary function. It simply evaluates its operand (name) and then returns a result computed from the operand.
+
+##### Example 2
-##### Example 2:
SELECT AVG(rating) FROM customers
The effect of `AVG` in Example 2 is quite different. Rather than performing a computation on an individual rating value, `AVG` has a global effect: it effectively restructures the query. As a pseudo-function, `AVG` requires its operand to be a group; therefore, it automatically collects all the rating values from the query block and forms them into a group.
The aggregation pseudo-functions always require their operand to be a group. In some queries, the group is explicitly generated by a `GROUP BY` clause, as in Q3.21:
+
##### Example
- (Q3.21) List the average credit rating of customers by zipcode.
+
+(Q3.21) List the average credit rating of customers by zipcode.
FROM customers AS c
GROUP BY c.address.zipcode AS zip
SELECT zip, AVG(c.rating) AS `avg credit rating`
ORDER BY zip;
- Result:
+Result:
[
{
@@ -915,10 +874,10 @@ The aggregation pseudo-functions always require their operand to be a group. In
Note in the result of Q3.21 that one or more customers had no zipcode. These customers were formed into a group for which the value of the grouping key is missing. When the query results were returned in JSON format, the `missing` key simply does not appear. Also note that the group whose key is `missing` appears first because `missing` is considered to be smaller than any other value. If some customers had had `null` as a zipcode, they would have been included in another group, appearin [...]
When an aggregation pseudo-function is used without an explicit `GROUP BY` clause, it implicitly forms the entire query block into a single group, as in Q3.22:
-##### Example
-(Q3.22) Find the average credit rating among all customers.
+##### Example
+(Q3.22) Find the average credit rating among all customers.
FROM customers AS c
SELECT AVG(c.rating) AS `avg credit rating`;
@@ -931,50 +890,42 @@ Result:
}
]
+The aggregation pseudo-function `COUNT` has a special form in which its operand is `*` instead of an expression.
+For example, `SELECT COUNT(*) FROM customers` simply returns the total number of customers, whereas `SELECT COUNT(rating) FROM customers` returns the number of customers who have known ratings (that is, their ratings are not `null` or `missing`).
-The aggregation pseudo-function `COUNT` has a special form in which its operand is `*` instead of an expression. For example, `SELECT COUNT(*) FROM customers` simply returns the total number of customers, whereas `SELECT COUNT(rating) FROM customers` returns the number of customers who have known ratings (that is, their ratings are not `null` or `missing`).
+Because the aggregation pseudo-functions sometimes restructure their operands, they can be used only in query blocks where (explicit or implicit) grouping is being done. Therefore the pseudo-functions cannot operate directly on arrays or multisets. For operating directly on JSON collections, SQL++ provides a set of ordinary functions for computing aggregations. Each ordinary aggregation function (except the ones corresponding to `COUNT` and `ARRAY_AGG`) has two versions: one that ignores [...]
-
-
- Because the aggregation pseudo-functions sometimes restructure their operands, they can be used only in query blocks where (explicit or implicit) grouping is being done. Therefore the pseudo-functions cannot operate directly on arrays or multisets. For operating directly on JSON collections, SQL++ provides a set of ordinary functions for computing aggregations. Each ordinary aggregation function (except the ones corresponding to `COUNT` and `ARRAY_AGG`) has two versions: one that ignore [...]
-
-| Aggregation pseudo-function; operates on groups only | ordinary functions: Ignores NULL or MISSING values | ordinary functions: Returns NULL if NULL or MISSING are encountered|
+| Aggregation pseudo-function; operates on groups only | Ordinary function: Ignores NULL or MISSING values | Ordinary function: Returns NULL if NULL or MISSING are encountered|
|----------|----------|--------|
|SUM| ARRAY_SUM| STRICT_SUM |
| AVG |ARRAY_MAX| STRICT_MAX |
| MAX | ARRAY_MIN| STRICT_MIN |
| MIN | ARRAY_AVG| STRICT_AVG |
-| COUNT |ARRAY_COUNT|STRICT_COUNT (see exception below) |
+| COUNT |ARRAY_COUNT|STRICT_COUNT (see exception below) |
|STDDEV_SAMP|ARRAY_STDDEV_SAMP| STRICT_STDDEV_SAMP |
|STDDEV_POP|ARRAY_STDDEV_POP| STRICT_STDDEV_POP |
|VAR_SAMP|ARRAY_VAR_SAMP| STRICT_VAR_SAMP |
|VAR_POP|ARRAY_VAR_POP| STRICT_VAR_POP |
|SKEWENESS|ARRAY_SKEWNESS| STRICT_SKEWNESS |
|KURTOSIS|ARRAY_KURTOSIS| STRICT_KURTOSIS |
-||ARRAY_AGG||||
-
-
-##### Exception: the ordinary aggregation function STRICT_COUNT operates on any collection, and returns a count of its items, including null values in the count. In this respect, STRICT_COUNT is more similar to COUNT(*) than to COUNT(expression).
+| |ARRAY_AGG| |
- Note that the ordinary aggregation functions that ignore `null` have names beginning with "ARRAY." This naming convention has historical roots. Despite their names, the functions operate on both arrays and multisets.
+Exception: the ordinary aggregation function STRICT_COUNT operates on any collection, and returns a count of its items, including null values in the count. In this respect, STRICT_COUNT is more similar to COUNT(*) than to COUNT(expression).
+Note that the ordinary aggregation functions that ignore `null` have names beginning with "ARRAY". This naming convention has historical roots. Despite their names, the functions operate on both arrays and multisets.
-
-Because of the special properties of the aggregation pseudo-functions, SQL (and therefore SQL++) is not a pure functional language. But every query that uses a pseudo-function can be expressed as an equivalent query that uses an ordinary function. Q3.23 is an example of how queries can be expressed without pseudo-functions. A more detailed explanation of all of the functions is also available [here](builtins.html#AggregateFunctions) .
+Because of the special properties of the aggregation pseudo-functions, SQL (and therefore SQL++) is not a pure functional language. But every query that uses a pseudo-function can be expressed as an equivalent query that uses an ordinary function. Q3.23 is an example of how queries can be expressed without pseudo-functions. A more detailed explanation of all of the functions is also available in the section on [Aggregate Functions](builtins.html#AggregateFunctions).
##### Example
- (Q3.23) Alternative form of Q3.22, using the ordinary function `ARRAY_AVG` rather than the aggregating pseudo-function `AVG`.
-
-
+(Q3.23) Alternative form of Q3.22, using the ordinary function `ARRAY_AVG` rather than the aggregating pseudo-function `AVG`.
SELECT ARRAY_AVG(
(SELECT VALUE c.rating
FROM customers AS c) ) AS `avg credit rating`;
- Result (same as Q3.22):
-
+Result (same as Q3.22):
[
{
@@ -984,31 +935,18 @@ Because of the special properties of the aggregation pseudo-functions, SQL (and
If the function `STRICT_AVG` had been used in Q3.23 in place of `ARRAY_AVG`, the average credit rating returned by the query would have been `null`, because at least one customer has no credit rating.
+### <a id="Group_As_clauses">GROUP AS Clause</a>
-
-### <a id="Left_outer_unnests">GROUP AS Clause</a>
-
----
-
-### GroupAsClause
-****
-
-
----
+##### GroupAsClause
+
JSON is a hierarchical format, and a fully featured JSON query language needs to be able to produce hierarchies of its own, with computed data at every level of the hierarchy. The key feature of SQL++ that makes this possible is the `GROUP AS` clause.
-
-
A query may have a `GROUP AS` clause only if it has a `GROUP BY` clause. The `GROUP BY` clause "hides" the original objects in each group, exposing only the grouping expressions and special aggregation functions on the non-grouping fields. The purpose of the `GROUP AS` clause is to make the original objects in the group visible to subsequent clauses. Thus the query can generate output data both for the group as a whole and for the individual objects inside the group.
-
-
For each group, the `GROUP AS` clause preserves all the objects in the group, just as they were before grouping, and gives a name to this preserved group. The group name can then be used in the `FROM` clause of a subquery to process and return the individual objects in the group.
-
-
-To see how this works, we'll write some queries that investigate the customers in each zipcode and their credit ratings. This would be a good time to review the sample database in Appendix 4. A part of the data is summarized below.
+To see how this works, we'll write some queries that investigate the customers in each zipcode and their credit ratings. This would be a good time to review the sample database in [Appendix 4](#Manual_data). A part of the data is summarized below.
Customers in zipcode 02115:
C35, J. Roberts, rating 565
@@ -1025,8 +963,6 @@ To see how this works, we'll write some queries that investigate the customers i
Customers with no zipcode:
C47, S. Logan, rating 625
-
-
Now let's consider the effect of the following clauses:
FROM customers AS c
@@ -1037,7 +973,6 @@ This query fragment iterates over the `customers` objects, using the iteration v
The clause `GROUP AS g` now makes the original objects visible again. For each group in turn, the variable `g` is bound to a multiset of objects, each of which has a field named `c`, which in turn contains one of the original objects. Thus after `GROUP AS g`, for the group with zipcode 02115, `g` is bound to the following multiset:
-
[
{ "c":
{ "custid": "C35",
@@ -1063,8 +998,6 @@ The clause `GROUP AS g` now makes the original objects visible again. For each g
}
]
-
-
Thus, the clauses following `GROUP AS` can see the original objects by writing subqueries that iterate over the multiset `g`.
The extra level named `c` was introduced into this multiset because the groups might have been formed from a join of two or more collections. Suppose that the `FROM` clause looked like `FROM customers AS c, orders AS o`. Then each item in the group would contain both a `customers` object and an `orders` object, and these two objects might both have a field with the same name. To avoid ambiguity, each of the original objects is wrapped in an "outer" object that gives it the name of its it [...]
@@ -1086,11 +1019,12 @@ In this case, following `GROUP AS g`, the variable `g` would be bound to the fol
...
]
-After using `GROUP AS` to make the content of a group accessible, you will probably want to write a subquery to access that content. A subquery for this purpose is written in exactly the same way as any other subquery. The name specified in the `GROUP AS` clause (`g` in the above example) is the name of a collection of objects. You can write a `FROM` clause to iterate over the objects in the collection, and you can specify an iteration variable to represent each object in turn. For `GRO [...]
+After using `GROUP AS` to make the content of a group accessible, you will probably want to write a subquery to access that content. A subquery for this purpose is written in exactly the same way as any other subquery. The name specified in the `GROUP AS` clause (`g` in the above example) is the name of a collection of objects. You can write a `FROM` clause to iterate over the objects in the collection, and you can specify an iteration variable to represent each object in turn. For `GROU [...]
Now we are ready to take a look at how `GROUP AS` might be used in a query. Suppose that we want to group customers by zipcode, and for each group we want to see the average credit rating and a list of the individual customers in the group. Here's a query that does that:
##### Example
+
(Q3.24) For each zipcode, list the average credit rating in that zipcode, followed by the customer numbers and names in numeric order.
FROM customers AS c
@@ -1157,18 +1091,16 @@ Result:
"zip": "63101"
}
]
-Note that this query contains two `ORDER BY` clauses: one in the outer query and one in the subquery. These two clauses govern the ordering of the outer-level list of zipcodes and the inner-level lists of customers, respectively. Also note that the group of customers with no zipcode comes first in the output list. For additional reading on SQL++ and more examples using `GROUP AS` as well as other clauses discussed in this manual see the [SQL++ Tutorial](https://asterixdb.apache.org/file [...]
-## <a id="Union_all">Selection and UNION ALL</a>
+Note that this query contains two `ORDER BY` clauses: one in the outer query and one in the subquery. These two clauses govern the ordering of the outer-level list of zipcodes and the inner-level lists of customers, respectively. Also note that the group of customers with no zipcode comes first in the output list.
----
-### Selection
-****
+## <a id="Union_all">Selection and UNION ALL</a>
-### UnionOption
-****
+##### Selection
+
----
+##### UnionOption
+
In a SQL++ query, two or more query blocks can be connected by the operator `UNION ALL`. The result of a `UNION ALL` between two query blocks contains all the items returned by the first query block, and all the items returned by the second query block. Duplicate items are not eliminated from the query result.
@@ -1178,87 +1110,71 @@ When two or more query blocks are connected by `UNION ALL`, they can be followed
In this example, a customer might be selected because he has ordered more than two different items (first query block) or because he has a high credit rating (second query block). By adding an explanatory string to each query block, the query writer can cause the output objects to be labeled to distinguish these two cases.
-
-
##### Example
(Q3.25a) Find customer ids for customers who have placed orders for more than two different items or who have a credit rating greater than 700, with labels to distinguish these cases.
+ FROM orders AS o, o.items AS i
+ GROUP BY o.orderno, o.custid
+ HAVING COUNT(*) > 2
+ SELECT DISTINCT o.custid AS customer_id, "Big order" AS reason
+ UNION ALL
- FROM orders AS o, o.items AS i
- GROUP BY o.orderno, o.custid
- HAVING COUNT(*) > 2
- SELECT DISTINCT o.custid AS customer_id, "Big order" AS reason
-
- UNION ALL
-
- FROM customers AS c
- WHERE rating > 700
- SELECT c.custid AS customer_id, "High rating" AS reason
- ORDER BY customer_id;
+ FROM customers AS c
+ WHERE rating > 700
+ SELECT c.custid AS customer_id, "High rating" AS reason
+ ORDER BY customer_id;
Result:
-
- [
- {
- "reason": "High rating",
- "customer_id": "C13"
- },
- {
- "reason": "Big order",
- "customer_id": "C37"
- },
- {
- "reason": "High rating",
- "customer_id": "C37"
- },
- {
- "reason": "Big order",
- "customer_id": "C41"
- }
- ]
-
-
+ [
+ {
+ "reason": "High rating",
+ "customer_id": "C13"
+ },
+ {
+ "reason": "Big order",
+ "customer_id": "C37"
+ },
+ {
+ "reason": "High rating",
+ "customer_id": "C37"
+ },
+ {
+ "reason": "Big order",
+ "customer_id": "C41"
+ }
+ ]
If, on the other hand, you simply want a list of the customer ids and you don't care to preserve the reasons, you can simplify your output by using `SELECT VALUE`, as follows:
-
-
(Q3.25b) Simplify Q3.25a to return a simple list of unlabeled customer ids.
+ FROM orders AS o, o.items AS i
+ GROUP BY o.orderno, o.custid
+ HAVING COUNT(*) > 2
+ SELECT VALUE o.custid
+ UNION ALL
- FROM orders AS o, o.items AS i
- GROUP BY o.orderno, o.custid
- HAVING COUNT(*) > 2
- SELECT VALUE o.custid
-
- UNION ALL
-
- FROM customers AS c
- WHERE rating > 700
- SELECT VALUE c.custid;
+ FROM customers AS c
+ WHERE rating > 700
+ SELECT VALUE c.custid;
Result:
- [
- "C37",
- "C41",
- "C13",
- "C37"
- ]
+ [
+ "C37",
+ "C41",
+ "C13",
+ "C37"
+ ]
## <a id="With_clauses">WITH Clause</a>
----
-
-### WithClause
-****
-
-
----
+##### WithClause
+
As in standard SQL, a `WITH` clause can be used to improve the modularity of a query. A `WITH` clause often contains a subquery that is needed to compute some result that is used later in the main query. In cases like this, you can think of the `WITH` clause as computing a “temporary view" of the input data. The next example uses a `WITH` clause to compute the total revenue of each order in 2020; then the main part of the query finds the minimum, maximum, and average revenue for orders i [...]
@@ -1274,7 +1190,7 @@ As in standard SQL, a `WITH` clause can be used to improve the modularity of a q
)
FROM order_revenue
SELECT AVG(revenue) AS average,
- MIN(revenue) AS minimum,
+ MIN(revenue) AS minimum,
MAX(revenue) AS maximum;
@@ -1292,16 +1208,14 @@ Result:
## <a id="Order_By_clauses">ORDER BY, LIMIT, and OFFSET Clauses</a>
----
-### OrderbyClause
-****
+##### OrderbyClause
+
-### LimitClause
-****
+##### LimitClause
+
-### OffsetClause
-****
----
+##### OffsetClause
+
The last three (optional) clauses to be processed in a query are `ORDER BY`, `LIMIT`, and `OFFSET`.
@@ -1317,6 +1231,7 @@ The `OFFSET` can also be used as a standalone clause, without the `LIMIT`.
The following example illustrates use of the `ORDER BY` and `LIMIT` clauses.
##### Example
+
(Q3.27) Return the top three customers by rating.
FROM customers AS c
@@ -1354,6 +1269,7 @@ The following example illustrates the use of `OFFSET`:
SELECT c.custid, c.name, c.rating
ORDER BY c.rating DESC
LIMIT 1 OFFSET 2;
+
Result:
[
@@ -1364,23 +1280,16 @@ Result:
}
]
-
-
## <a id="Subqueries">Subqueries</a>
----
-
-### Subquery
-****
-
-
----
+##### Subquery
+
A subquery is simply a query surrounded by parentheses. In SQL++, a subquery can appear anywhere that an expression can appear. Like any query, a subquery always returns a collection, even if the collection contains only a single value or is empty. If the subquery has a SELECT clause, it returns a collection of objects. If the subquery has a SELECT VALUE clause, it returns a collection of scalar values. If a single scalar value is expected, the indexing operator [0] can be used to extrac [...]
##### Example
-(Q3.29)(Subquery in SELECT clause)
+(Q3.29) (Subquery in SELECT clause)
For every order that includes item no. 120, find the order number, customer id, and customer name.
Here, the subquery is used to find a customer name, given a customer id. Since the outer query expects a scalar result, the subquery uses SELECT VALUE and is followed by the indexing operator [0].
@@ -1420,6 +1329,7 @@ Here, the subquery is used to find the average rating among all customers. Once
(FROM customers AS c2
SELECT VALUE AVG(c2.rating))[0]
SELECT c1.custid, c1.name, c1.rating;
+
Result:
[
@@ -1440,7 +1350,6 @@ Result:
}
]
-
##### Example
(Q3.31) (Subquery in FROM clause)
@@ -1454,15 +1363,15 @@ Here, the FROM clause expects to iterate over a collection of objects, so the su
SELECT o.orderno, SUM(i.qty * i.price) AS revenue
) AS r
SELECT AVG(r.revenue) AS average,
- MIN(r.revenue) AS minimum,
- MAX(r.revenue) AS maximum;
+ MIN(r.revenue) AS minimum,
+ MAX(r.revenue) AS maximum;
Result:
[
{
"average": 4669.99,
- "minimum": 130.45,
+ "minimum": 130.45,
"maximum": 18847.58
}
]
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query_title.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query_title.md
index 5a73096..d5d988a 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query_title.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query_title.md
@@ -18,23 +18,3 @@
!-->
# <a id="Queries">3. Queries</a>
-
-A *query* can be an expression, or it can be constructed from blocks of code called *query blocks*. A query block may contain several clauses, including `SELECT`, `FROM`, `LET`, `WHERE`, `GROUP BY`, and `HAVING`.
-
----
-### Query
-****
-
-### Selection
-****
-
-### QueryBlock
-****
-
-### StreamGenerator
-****
-
-
----
-
-Note that, unlike SQL, SQL++ allows the `SELECT` clause to appear either at the beginning or at the end of a query block. For some queries, placing the `SELECT` clause at the end may make a query block easier to understand, because the `SELECT` clause refers to variables defined in the other clauses.
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md
index 7a74900..d6e40e7 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md
@@ -21,39 +21,27 @@ Window functions are special functions that compute aggregate values over a "win
A window function call is identified by an `OVER` clause, which can specify three things: partitioning, ordering, and framing. The partitioning specification is like a `GROUP BY`: it splits the input data into partitions. For example, a set of employees might be partitioned by department. The window function, when applied to a given object, is influenced only by other objects in the same partition. The ordering specification is like an `ORDER BY`: it determines the ordering of the object [...]
-
Here is an example of a window function call:
-
-
- SELECT deptno, purchase_date, item, cost,
- SUM(cost) OVER (
- PARTITION BY deptno
- ORDER BY purchase_date
- ROWS UNBOUNDED PRECEDING) AS running_total_cost
- FROM purchases
- ORDER BY deptno, purchase_date
-
-
+ SELECT deptno, purchase_date, item, cost,
+ SUM(cost) OVER (
+ PARTITION BY deptno
+ ORDER BY purchase_date
+ ROWS UNBOUNDED PRECEDING) AS running_total_cost
+ FROM purchases
+ ORDER BY deptno, purchase_date
This example partitions the `purchases` dataset by department number. Within each department, it orders the `purchases` by date and computes a running total cost for each item, using the frame specification `ROWS UNBOUNDED PRECEDING`. Note that the `ORDER BY` clause in the window function is separate and independent from the `ORDER BY` clause of the query as a whole.
-
-
-The general syntax of a window function call is specified in this section. SQL++ has a set of builtin window functions, which are listed and explained in their respective [section](builtins.html#WindowFunctions) of the builtin functions page. In addition, standard SQL aggregate functions such as `SUM` and `AVG` can be used as window functions if they are used with an `OVER` clause.
-The query language has a dedicated set of window functions.
-Aggregate functions can also be used as window functions, when they are used with an `OVER` clause.
+The general syntax of a window function call is specified in this section. SQL++ has a set of builtin window functions, which are listed and explained in the [Window Functions](builtins.html#WindowFunctions) section of the builtin functions page. In addition, standard SQL aggregate functions such as `SUM` and `AVG` can be used as window functions if they are used with an `OVER` clause.
## <a id="Window_function_call">Window Function Call</a> ##
----
-### WindowFunctionCall
-****
-
-### WindowFunctionType
-****
+##### WindowFunctionCall
+
-----
+##### WindowFunctionType
+
Refer to the [Aggregate Functions](builtins.html#AggregateFunctions) section
for a list of aggregate functions.
@@ -63,25 +51,15 @@ list of window functions.
### <a id="Window_function_arguments">Window Function Arguments</a> ###
----
-
-### WindowFunctionArguments
-****
-
-
----
+##### WindowFunctionArguments
+
Refer to the [Aggregate Functions](builtins.html#AggregateFunctions) section or the [Window Functions](builtins.html#WindowFunctions) section for details of the arguments for individual functions.
### <a id="Window_function_options">Window Function Options</a> ###
----
-
-### WindowFunctionOptions
-****
-
-
----
+##### WindowFunctionOptions
+
Window function options cannot be used with [aggregate functions](builtins.html#AggregateFunctions).
@@ -89,7 +67,7 @@ Window function options can only be used with some [window functions](builtins.h
The *FROM modifier* determines whether the computation begins at the first or last tuple in the window. It is optional and can only be used with the `nth_value()` function. If it is omitted, the default setting is `FROM FIRST`.
-The *NULLS modifier* determines whether NULL values are included in the computation, or ignored. MISSING values are treated the same way as NULL values. It is also optional and can only be used with the `first_value()`, `last_value()`, `nth_value()`, `lag()`, and `lead()` functions. If omitted, the default setting is `RESPECT NULLS`.
+The *NULLS modifier* determines whether NULL values are included in the computation, or ignored. MISSING values are treated the same way as NULL values. It is also optional and can only be used with the `first_value()`, `last_value()`, `nth_value()`, `lag()`, and `lead()` functions. If omitted, the default setting is `RESPECT NULLS`.
### <a id="Window_frame_variable">Window Frame Variable</a> ###
@@ -101,27 +79,17 @@ The `AS` keyword enables you to specify an alias for the window frame contents.
The alias is not necessary when using a [window function](builtins.html#WindowFunctions), or when using a standard SQL aggregate function with the `OVER` clause.
-
### <a id="Window_definition">Window Definition</a> ###
----
-
-### WindowDefinition
-****
-
----
+##### WindowDefinition
+
The *window definition* specifies the partitioning, ordering, and framing for window functions.
#### <a id="Window_partition_clause">Window Partition Clause</a> ####
----
-
-### WindowPartitionClause
-****
-
-
----
+##### WindowPartitionClause
+
The *window partition clause* divides the tuples into logical partitions
using one or more expressions.
@@ -134,13 +102,9 @@ This clause is optional.
If omitted, all tuples are united in a single partition.
#### <a id="Window_order_clause">Window Order Clause</a> ####
----
-### WindowOrderClause
-****
-
-
----
+##### WindowOrderClause
+
The *window order clause* determines how tuples are ordered within each partition. The window function works on tuples in the order specified by this clause.
@@ -159,23 +123,19 @@ This clause is optional. If omitted, all tuples are considered peers, i.e. their
If the window frame is defined by `RANGE` or `GROUPS`, the results are same
for each tuple.
-##### Note #####
-
-This clause does not guarantee the overall order of the query results. To guarantee the order of the final results, use the query `ORDER BY` clause.
-
+**Note:** This clause does not guarantee the overall order of the query results. To guarantee the order of the final results, use the query `ORDER BY` clause.
#### <a id="Window_frame_clause">Window Frame Clause</a> ####
-### WindowFrameClause
-****
+##### WindowFrameClause
+
-
-The *window frame clause* defines the window frame. It can be used with all [aggregate functions](builtins.html#AggregateFunctions) and some [window functions](builtins.html#WindowFunctions) - refer to the descriptions of individual functions for more details. It is optional and allowed only when the [window order clause](#Window_order_clause) is present.
+The *window frame clause* defines the window frame. It can be used with all [aggregate functions](builtins.html#AggregateFunctions) and some [window functions](builtins.html#WindowFunctions) — refer to the descriptions of individual functions for more details. It is optional and allowed only when the [window order clause](#Window_order_clause) is present.
* If this clause is omitted and there is no [window order clause](#Window_order_clause), the window frame is the entire partition.
* If this clause is omitted but there is a [window order clause](#Window_order_clause), the window frame becomes all tuples
- in the partition preceding the current tuple and its peers - the same as `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`.
+ in the partition preceding the current tuple and its peers — the same as `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`.
The window frame can be defined in the following ways:
@@ -185,29 +145,18 @@ The window frame can be defined in the following ways:
* `GROUPS`: Counts all groups of tied rows within the frame. The function produces deterministic results.
-##### Note #####
-
-If this clause uses `RANGE` with either *Expr* `PRECEDING` or *Expr* ` FOLLOWING`, the [window order clause](#Window_order_clause) must have only a single ordering term.
-
+**Note:** If this clause uses `RANGE` with either *Expr* `PRECEDING` or *Expr* `FOLLOWING`, the [window order clause](#Window_order_clause) must have only a single ordering term.
The ordering term expression must evaluate to a number.
-
If these conditions are not met, the window frame will be empty, which means the window function will return its default value: in most cases this is `null`, except for `strict_count()` or `array_count()`, whose default value is 0. This restriction does not apply when the window frame uses `ROWS` or `GROUPS`.
-##### Tip #####
-
-The `RANGE` window frame is commonly used to define window frames based
+**Tip:** The `RANGE` window frame is commonly used to define window frames based
on date or time.
-
If you want to use `RANGE` with either *Expr* `PRECEDING` or *Expr* `FOLLOWING`, and you want to use an ordering expression based on date or time, the expression in *Expr* `PRECEDING` or *Expr* `FOLLOWING` must use a data type that can be added to the ordering expression.
#### <a id="Window_frame_extent">Window Frame Extent</a> ####
----
-
-### WindowFrameExtent
-****
-
----
+##### WindowFrameExtent
+
The *window frame extent clause* specifies the start point and end point of the window frame.
The expression before `AND` is the start point and the expression after `AND` is the end point.
@@ -226,13 +175,13 @@ Window frame extents that result in an explicit violation are:
Window frame extents that result in an implicit violation are:
-* `BETWEEN UNBOUNDED PRECEDING AND` *Expr* `PRECEDING` - if *Expr* is too high, some tuples may generate an empty window frame.
+* `BETWEEN UNBOUNDED PRECEDING AND` *Expr* `PRECEDING` — if *Expr* is too high, some tuples may generate an empty window frame.
-* `BETWEEN` *Expr* `PRECEDING AND` *Expr* `PRECEDING` - if the second *Expr* is greater than or equal to the first *Expr*, all result sets will generate an empty window frame.
+* `BETWEEN` *Expr* `PRECEDING AND` *Expr* `PRECEDING` — if the second *Expr* is greater than or equal to the first *Expr*, all result sets will generate an empty window frame.
-* `BETWEEN` *Expr* `FOLLOWING AND` *Expr* `FOLLOWING` - if the first *Expr* is greater than or equal to the second *Expr*, all result sets will generate an empty window frame.
+* `BETWEEN` *Expr* `FOLLOWING AND` *Expr* `FOLLOWING` — if the first *Expr* is greater than or equal to the second *Expr*, all result sets will generate an empty window frame.
-* `BETWEEN` *Expr* `FOLLOWING AND UNBOUNDED FOLLOWING` - if *Expr* is too high, some tuples may generate an empty window frame.
+* `BETWEEN` *Expr* `FOLLOWING AND UNBOUNDED FOLLOWING` — if *Expr* is too high, some tuples may generate an empty window frame.
* If the [window frame exclusion clause](#Window_frame_exclusion) is present, any window frame specification may result in empty window frame.
@@ -240,21 +189,16 @@ The *Expr* must be a positive constant or an expression that evaluates as a posi
#### <a id="Window_frame_exclusion">Window Frame Exclusion</a> ####
----
-
-### WindowFrameExclusion
-****
-
-
----
+##### WindowFrameExclusion
+
The *window frame exclusion clause* enables you to exclude specified tuples from the window frame.
-This clause can be used with all [aggregate functions](builtins.html#AggregateFunctions) and some [window functions](builtins.html#WindowFunctions) - refer to the descriptions of individual functions for more details.
+This clause can be used with all [aggregate functions](builtins.html#AggregateFunctions) and some [window functions](builtins.html#WindowFunctions) — refer to the descriptions of individual functions for more details.
This clause is allowed only when the [window frame clause](#Window_frame_clause) is present.
-This clause is optional. If this clause is omitted, the default is no exclusion - the same as `EXCLUDE NO OTHERS`.
+This clause is optional. If this clause is omitted, the default is no exclusion — the same as `EXCLUDE NO OTHERS`.
* `EXCLUDE CURRENT ROW`: If the current tuple is still part of the window frame, it is removed from the window frame.
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
index 222c7a4..e064f06 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
@@ -20,22 +20,19 @@
## <a id="Lifecycle_management_statements">Lifecycle Management Statements</a>
### <a id="Use">Use Statement</a>
----
-### UseStmnt
-****
+##### UseStmnt
+
-### DataverseName
-****
-
----
+##### DataverseName
+
At the uppermost level, the world of data is organized into data namespaces called **dataverses**.
To set the default dataverse for statements, the `USE` statement is provided.
As an example, the following statement sets the default dataverse to be `Commerce`.
- USE Commerce;
+ USE Commerce;
### <a id="Sets"> Set Statement</a>
@@ -50,24 +47,21 @@ In general, the function body (expression) can be any legal query expression.
The function named in the `DECLARE FUNCTION` statement is accessible only in the current query. To create a persistent function for use in multiple queries, use the `CREATE FUNCTION` statement.
----
-### FunctionDeclaration
-****
-
----
+##### FunctionDeclaration
+
The following is a simple example of a temporary function definition and its use.
##### Example
DECLARE FUNCTION nameSearch(customerId){
- (SELECT c.custid, c.name
- FROM customers AS c
- WHERE c.custid = customerId)[0]
+ (SELECT c.custid, c.name
+ FROM customers AS c
+ WHERE c.custid = customerId)[0]
};
- SELECT VALUE nameSearch("C25");
+ SELECT VALUE nameSearch("C25");
For our sample data set, this returns:
@@ -76,31 +70,26 @@ For our sample data set, this returns:
]
### <a id="Create"> Create Statement</a>
----
-### CreateStmnt
-****
-### DataverseName
-****
+##### CreateStmnt
+
-### QualifiedName
-****
+##### DataverseName
+
-### DoubleQualifiedName
-****
+##### QualifiedName
+
----
+##### DoubleQualifiedName
+
The `CREATE` statement is used for creating dataverses as well as other persistent artifacts in a dataverse.
It can be used to create new dataverses, datatypes, datasets, indexes, and user-defined query functions.
#### <a id="Dataverses"> Create Dataverse</a>
----
-### CreateDataverse
-****
-
----
+##### CreateDataverse
+
The `CREATE DATAVERSE` statement is used to create new dataverses.
To ease the authoring of reusable query scripts, an optional `IF NOT EXISTS` clause is included to allow
@@ -115,29 +104,26 @@ The following example creates a new dataverse named `Commerce` if one does not a
#### <a id="Types"> Create Type </a>
----
-### CreateType
-****
-
-### ObjectTypeDef
-****
+##### CreateType
+
-### ObjectField
-****
+##### ObjectTypeDef
+
-### TypeExpr
-****
+##### ObjectField
+
-### ArrayTypeDef
-****
+##### TypeExpr
+
-### MultisetTypeDef
-****
+##### ArrayTypeDef
+
-### TypeReference
-****
+##### MultisetTypeDef
+
----
+##### TypeReference
+
The `CREATE TYPE` statement is used to create a new named datatype.
This type can then be used to create stored collections or utilized when defining one or more other datatypes.
@@ -147,38 +133,38 @@ A object type can be defined as being either open or closed.
Instances of a closed object type are not permitted to contain fields other than those specified in the create type statement.
Instances of an open object type may carry additional fields, and open is the default for new types if neither option is specified.
-The following example creates three new object type called `addressType` , `customerType` and `itemType`.
+The following example creates three new object types called `addressType`, `customerType`, and `itemType`.
Their fields are essentially traditional typed name/value pairs (much like SQL fields).
Since it is defined as (defaulting to) being an open type, instances will be permitted to contain more than what is specified in the type definition. Indeed many of the customer objects contain a rating as well, however this is not necessary for the customer object to be created. As can be seen in the sample data, customers can exist without ratings or with part (or all) of the address missing.
##### Example
- CREATE TYPE addressType AS {
- street: string,
- city: string,
- zipcode: string?
- };
+ CREATE TYPE addressType AS {
+ street: string,
+ city: string,
+ zipcode: string?
+ };
CREATE TYPE customerType AS {
- custid: string,
- name: string,
- address: addressType?
+ custid: string,
+ name: string,
+ address: addressType?
};
- CREATE TYPE itemType AS {
- itemno: int,
- qty: int,
- price: int
- };
+ CREATE TYPE itemType AS {
+ itemno: int,
+ qty: int,
+ price: int
+ };
Optionally, you may wish to create a type that has an automatically generated primary key field. The example below shows an alternate form of `itemType` which achieves this by setting its primary key, `itemno`, to UUID. (Refer to the Datasets section later for more details on such fields.)
##### Example
- CREATE TYPE itemType AS {
- itemno: uuid,
- qty: int,
- price: int
- };
+ CREATE TYPE itemType AS {
+ itemno: uuid,
+ qty: int,
+ price: int
+ };
Note that the type of the `itemno` in this example is UUID. This field type can be used if you want to have an autogenerated-PK field. (Refer to the Datasets section later for more details on such fields.)
@@ -188,54 +174,51 @@ although the `ship_date` field is marked as optional and may thus be `NULL` or `
##### Example
- CREATE TYPE orderType AS CLOSED {
- orderno: int,
- custid: string,
- order_date: string,
- ship_date: string?,
- items: [ itemType ]
- };
+ CREATE TYPE orderType AS CLOSED {
+ orderno: int,
+ custid: string,
+ order_date: string,
+ ship_date: string?,
+ items: [ itemType ]
+ };
#### <a id="Datasets"> Create Dataset</a>
----
-### CreateDataset
-****
+##### CreateDataset
+
-### CreateInternalDataset
-****
+##### CreateInternalDataset
+
-### CreateExternalDataset
-****
+##### CreateExternalDataset
+
-### DatasetTypeDef
-****
+##### DatasetTypeDef
+
-### DatasetFieldDef
-****
+##### DatasetFieldDef
+
-### TypeReference
-****
+##### TypeReference
+
-### PrimaryKey
-****
+##### PrimaryKey
+
-### NestedField
-****
+##### NestedField
+
-### AdapterName
-****
+##### AdapterName
+
-### Configuration
-****
+##### Configuration
+
-### KeyValuePair
-****
+##### KeyValuePair
+
-### Properties
-****
-
----
+##### Properties
+
The `CREATE DATASET` statement is used to create a new dataset.
Datasets are named, multisets of object type instances;
@@ -277,7 +260,7 @@ When defining an External dataset, an appropriate adapter type must be selected
The following example creates an Internal dataset for storing `customerType` objects.
It specifies that their `custid` field is their primary key.
-#### Example
+##### Example
CREATE INTERNAL DATASET customers(customerType) PRIMARY KEY custid;
@@ -286,13 +269,13 @@ It also specifies that the `itemno` field is an auto-generated field, meaning th
Note that the `itemno` field's declared type must be UUID in this case.
-#### Example
+##### Example
CREATE DATASET MyItems(itemType) PRIMARY KEY itemno AUTOGENERATED;
Alternatively the dataset object type can be specified using inline type definition syntax.
-#### Example
+##### Example
CREATE DATASET MyItems(itemno INT NOT UNKNOWN, qty INT NOT UNKNOWN, price INT NOT UNKNOWN) PRIMARY KEY itemno AUTOGENERATED;
@@ -301,7 +284,7 @@ The choice of the `hdfs` adapter means that this dataset's data actually resides
The example `CREATE` statement also provides parameters used by the hdfs adapter:
the URL and path needed to locate the data in HDFS and a description of the data format.
-#### Example
+##### Example
CREATE EXTERNAL DATASET LineItem(LineItemType) USING hdfs (
("hdfs"="hdfs://HOST:PORT"),
@@ -312,26 +295,23 @@ the URL and path needed to locate the data in HDFS and a description of the data
#### <a id="Indices">Create Index</a>
----
-### CreateIndex
-****
-
-### CreateSecondaryIndex
-****
+##### CreateIndex
+
-### CreatePrimaryKeyIndex
-****
+##### CreateSecondaryIndex
+
-### IndexField
-****
+##### CreatePrimaryKeyIndex
+
-### NestedField
-****
+##### IndexField
+
-### IndexType
-****
+##### NestedField
+
----
+##### IndexType
+
The `CREATE INDEX` statement creates a secondary index on one or more fields of a specified dataset.
Supported index types include `BTREE` for totally ordered datatypes, `RTREE` for spatial data,
@@ -345,9 +325,9 @@ specified at the end of the index definition.
(if the optional field exists in the object) always matches this specified (open) field type.
The following example creates a btree index called `cCustIdx` on the `custid` field of the orders dataset.
-This index can be useful for accelerating exact-match queries, range search queries, and joins involving the `custid`field.
+This index can be useful for accelerating exact-match queries, range search queries, and joins involving the `custid` field.
-#### Example
+##### Example
CREATE INDEX cCustIdx ON orders(custid) TYPE BTREE;
@@ -356,7 +336,7 @@ This index can be useful for accelerating exact-match queries, range search quer
The index is enforced so that records that do not have the `createdTime` field or have a mismatched type on the field
cannot be inserted into the dataset.
-#### Example
+##### Example
CREATE INDEX oCreatedTimeIdx ON orders(createdTime: datetime?) TYPE BTREE ENFORCED;
@@ -367,7 +347,7 @@ and joins involving the `addedTime` field.
The index is not enforced so that records that do not have the `addedTime` field or have a mismatched type on the field
can still be inserted into the dataset.
-#### Example
+##### Example
CREATE INDEX cAddedTimeIdx ON customers(addedTime: datetime?);
@@ -377,25 +357,25 @@ This index can be useful for accelerating exact-match queries, range search quer
and joins involving the nested `orderUserName` field.
Such nested fields must be singular, i.e., one cannot index through (or on) an array-valued field.
-#### Example
+##### Example
CREATE INDEX oOrderUserNameIdx ON orders(order.orderUserName) TYPE BTREE;
The following example creates an open rtree index called `oOrderLocIdx` on the order-location field of the `orders` dataset. This index can be useful for accelerating queries that use the [`spatial-intersect` function](builtins.html#spatial_intersect) in a predicate involving the sender-location field.
-#### Example
+##### Example
CREATE INDEX oOrderLocIDx ON orders(`order-location` : point?) TYPE RTREE ENFORCED;
The following example creates a 3-gram index called `cUserIdx` on the name field of the `customers` dataset. This index can be used to accelerate some similarity or substring maching queries on the name field. For details refer to the document on [similarity queries](similarity.html#NGram_Index).
-#### Example
+##### Example
CREATE INDEX cUserIdx ON customers(name) TYPE NGRAM(3);
The following example creates a keyword index called `oCityIdx` on the `city` within the `address` field of the `customers` dataset. This keyword index can be used to optimize queries with token-based similarity predicates on the `address` field. For details refer to the document on [similarity queries](similarity.html#Keyword_Index).
-#### Example
+##### Example
CREATE INDEX oCityIdx ON customers(address.city) TYPE KEYWORD;
@@ -404,7 +384,7 @@ This index is useful for speeding up aggregation queries which involve only prim
The name of the index is optional. If the name is not specified, the system will generate
one. When the user would like to drop this index, the metadata can be queried to find the system-generated name.
-#### Example
+##### Example
CREATE PRIMARY INDEX cus_pk_idx ON customers;
@@ -420,32 +400,32 @@ To look up the the above primary-key index, issue the following query:
The query returns:
- [
- {
- "DataverseName": "Commerce",
- "DatasetName": "customers",
- "IndexName": "cus_pk_idx",
- "IndexStructure": "BTREE",
- "SearchKey": [],
- "IsPrimary": false,
- "Timestamp": "Fri Sep 18 14:15:51 PDT 2020",
- "PendingOp": 0
- },
- {
- "DataverseName": "Commerce",
- "DatasetName": "customers",
- "IndexName": "customers",
- "IndexStructure": "BTREE",
- "SearchKey": [
- [
- "custid"
- ]
- ],
- "IsPrimary": true,
- "Timestamp": "Thu Jul 16 13:07:37 PDT 2020",
- "PendingOp": 0
- }
- ]
+ [
+ {
+ "DataverseName": "Commerce",
+ "DatasetName": "customers",
+ "IndexName": "cus_pk_idx",
+ "IndexStructure": "BTREE",
+ "SearchKey": [],
+ "IsPrimary": false,
+ "Timestamp": "Fri Sep 18 14:15:51 PDT 2020",
+ "PendingOp": 0
+ },
+ {
+ "DataverseName": "Commerce",
+ "DatasetName": "customers",
+ "IndexName": "customers",
+ "IndexStructure": "BTREE",
+ "SearchKey": [
+ [
+ "custid"
+ ]
+ ],
+ "IsPrimary": true,
+ "Timestamp": "Thu Jul 16 13:07:37 PDT 2020",
+ "PendingOp": 0
+ }
+ ]
Remember that `CREATE PRIMARY INDEX` creates a secondary index.
That is the reason the `IsPrimary` field is false.
@@ -453,11 +433,8 @@ The primary-key index can be identified by the fact that the `SearchKey` field i
#### <a id="Synonyms"> Create Synonym</a>
----
-### CreateSynonym
-****
-
----
+##### CreateSynonym
+
The `CREATE SYNONYM` statement creates a synonym for a given dataset.
This synonym may be used used instead of the dataset name in `SELECT`, `INSERT`, `UPSERT`, `DELETE`, and `LOAD` statements.
@@ -471,24 +448,21 @@ The target dataset does not need to exist when the synonym is created.
SELECT * FROM customersSynonym;
-More information on how synonyms are resolved can be found in the [Appendix 3. Variable Bindings and Name Resolution](#Variable_bindings_and_name_resolution).
+More information on how synonyms are resolved can be found in [Appendix 3. Variable Bindings and Name Resolution](#Variable_bindings_and_name_resolution).
#### <a id="Create_function">Create Function</a>
The `CREATE FUNCTION` statement creates a **named** function that can then be used and reused in queries.
The body of a function can be any query expression involving the function's parameters.
----
-### CreateFunction
-****
-
-### FunctionParameters
-****
+##### CreateFunction
+
-### ExternalFunctionDef
-****
+##### FunctionParameters
+
----
+##### ExternalFunctionDef
+
The following is an example of a `CREATE FUNCTION` statement which is similar to our earlier `DECLARE FUNCTION` example.
@@ -531,26 +505,23 @@ would be as follows
### <a id="Removal">Drop Statement</a>
----
-### DropStmnt
-****
+##### DropStmnt
+
-### DataverseName
-****
+##### DataverseName
+
-### QualifiedName
-****
+##### QualifiedName
+
-### DoubleQualifiedName
-****
+##### DoubleQualifiedName
+
-### FunctionSignature
-****
+##### FunctionSignature
+
-### FunctionParameters
-****
-
----
+##### FunctionParameters
+
The `DROP` statement is the inverse of the `CREATE` statement. It can be used to drop dataverses, datatypes, datasets, indexes, functions, and synonyms.
@@ -580,20 +551,17 @@ the identifying name of the function to be dropped must explicitly include that
### <a id="Load_statement">Load Statement</a>
----
-### LoadStmnt
-****
-
-### AdapterName
-****
+##### LoadStmnt
+
-### Configuration
-****
+##### AdapterName
+
-### KeyValuePair
-****
+##### Configuration
+
----
+##### KeyValuePair
+
The `LOAD` statement is used to initially populate a dataset via bulk loading of data from an external file.
An appropriate adapter must be selected to handle the nature of the desired external data.
@@ -614,11 +582,8 @@ The following example shows how to bulk load the `customers` dataset from an ext
### <a id="Inserts">Insert Statement</a>
----
-### InsertStmnt
-****
-
----
+##### InsertStmnt
+
The `INSERT` statement is used to insert new data into a dataset.
The data to be inserted comes from a query expression.
@@ -644,11 +609,8 @@ The following example illustrates a query-based insertion.
### <a id="Upserts">Upsert Statement</a>
----
-### UpsertStmnt
-****
-
----
+##### UpsertStmnt
+
The `UPSERT` statement syntactically mirrors the `INSERT `statement discussed above.
The difference lies in its semantics, which for `UPSERT` are "add or replace" instead of the `INSERT` "add if not present, else error" semantics.
@@ -667,11 +629,8 @@ The following example illustrates a query-based upsert operation.
### <a id="Deletes">Delete Statement</a>
----
-### DeleteStmnt
-****
-
----
+##### DeleteStmnt
+
The `DELETE` statement is used to delete data from a target dataset.
The data to be deleted is identified by a boolean expression involving the variable bound to the target dataset in the `DELETE` statement.
@@ -692,3 +651,4 @@ The following examples illustrate single-object deletions.
##### Example
DELETE FROM customers WHERE custid = "C47";
+
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_head.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_head.md
index 39a5439..b73d82d 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_head.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_head.md
@@ -19,15 +19,11 @@
# <a id="DDL_and_DML_statements">7. DDL and DML statements</a>
----
+##### Stmnt
+
-### Stmnt
-****
-
-### SingleStmnt
-****
-
----
+##### SingleStmnt
+
In addition to queries, an implementation of SQL++ needs to support statements for data definition
and manipulation purposes as well as controlling the context to be used in evaluating query expressions.
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_2_parameters.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_2_parameters.md
index 86510cf..2799acc 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_2_parameters.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_2_parameters.md
@@ -19,8 +19,8 @@
The `SET` statement can be used to override some cluster-wide configuration parameters for a specific request:
-### SetStmnt
-****
+##### SetStmnt
+
As parameter identifiers are qualified names (containing a '.') they have to be escaped using backticks (\`\`).
Note that changing query parameters will not affect query correctness but only impact performance
@@ -32,7 +32,7 @@ in a cluster. A user can manually specify the maximum execution parallelism for
using the following parameter:
* **compiler.parallelism**: the maximum number of CPU cores can be used to process a query.
-There are three cases of the value *p* for compiler.parallelism:
+ There are three cases of the value *p* for compiler.parallelism:
- *p* \< 0 or *p* \> the total number of cores in a cluster: the system will use all available cores in the
cluster;
@@ -51,6 +51,7 @@ There are three cases of the value *p* for compiler.parallelism:
FROM customers c JOIN orders o ON c.custid = o.custid;
## <a id="Memory_parameters">Memory Parameters</a>
+
In the system, each blocking runtime operator such as join, group-by and order-by
works within a fixed memory budget, and can gracefully spill to disks if
the memory budget is smaller than the amount of data they have to hold.
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_3_resolution.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_3_resolution.md
index c3f0450..a4acbb0 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_3_resolution.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_3_resolution.md
@@ -197,16 +197,15 @@ The likely intent of the query above can be accomplished as follows:
SELECT e.name, pay
ORDER BY pay
-Note:
-In the phrase *expr1* `JOIN` *expr2* `ON` *expr3*, variables defined in *expr1* are visible in *expr3* but not in *expr2*. Here's an example that will not work:
+Note that in the phrase *expr1* `JOIN` *expr2* `ON` *expr3*, variables defined in *expr1* are visible in *expr3* but not in *expr2*. Here's an example that will not work:
- FROM orders AS o JOIN o.items AS i ON 1 = 1
+ FROM orders AS o JOIN o.items AS i ON 1 = 1
The variable `o`, defined in the phrase before `JOIN`, cannot be used in the phrase immediately following `JOIN`. The probable intent of this example could be accomplished in either of the following ways:
- FROM orders AS o UNNEST o.items AS i
+ FROM orders AS o UNNEST o.items AS i
- FROM orders AS o, o.items AS i
+ FROM orders AS o, o.items AS i
To summarize this rule: You may not use left-correlation in an explicit `JOIN` clause.
@@ -281,3 +280,4 @@ The rules for resolving the leftmost identifier are:
5. Once the leftmost identifier has been resolved, the following dots and identifiers in the name (if any) are treated as a path expression that navigates to a field nested inside that object.
The name resolves to the field at the end of the path.
If this field does not exist, the value `missing` is returned.
+
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_4_manual_data.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_4_manual_data.md
index 7516e3f..a07c67b 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_4_manual_data.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_4_manual_data.md
@@ -292,3 +292,4 @@ This appendix lists the data definitions and the datasets used for the examples
"items": []
}
]
+
[asterixdb] 08/17: [NO ISSUE][MD] Limit metadata identifiers to 255
bytes
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ce0141ac79fee0d9684b6aaee5fd3be8b90c066a
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Tue Mar 2 01:16:08 2021 +0300
[NO ISSUE][MD] Limit metadata identifiers to 255 bytes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Ensure the length of metadata identifiers doesn't exceed
255 bytes.
Change-Id: Icf92ad741d166f35d8f132ed4ba94d35a2d1aee1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10304
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
Contrib: Murtadha Hubail <mh...@apache.org>
---
.../asterix/test/metadata/MetadataManagerTest.java | 49 +++++++++++-----------
.../metadata/declared/MetadataProvider.java | 10 ++---
.../asterix/metadata/utils/MetadataConstants.java | 2 +-
3 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
index 39e2ece..f8d6aeb 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
@@ -83,23 +83,23 @@ public class MetadataManagerTest {
Arrays.asList(
// #1. max single-part name
DataverseName.createSinglePartName(
- StringUtils.repeat('a', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8)),
+ StringUtils.repeat('a', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8)),
// #2. max single-part name (2-byte characters)
DataverseName.createSinglePartName(
- StringUtils.repeat(auml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2)),
+ StringUtils.repeat(auml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2)),
// #3. 4 max parts
DataverseName.create(Arrays.asList(
- StringUtils.repeat('a', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
- StringUtils.repeat('b', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
- StringUtils.repeat('c', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
- StringUtils.repeat('d', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8))),
+ StringUtils.repeat('a', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('b', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('c', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('d', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8))),
// #3. 4 max parts (2-byte characters)
DataverseName.create(Arrays.asList(
- StringUtils.repeat(auml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
- StringUtils.repeat(euml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
- StringUtils.repeat(auml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(auml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(euml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(auml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
StringUtils.repeat(euml,
- MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2))));
+ MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2))));
for (DataverseName dvNameOk : dvNameOkList) {
String sql = String.format("create dataverse %s;", dvNameOk);
@@ -114,31 +114,32 @@ public class MetadataManagerTest {
Arrays.asList(
// #1. single-part name exceeds part length limit
DataverseName.createSinglePartName(
- StringUtils.repeat('A', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 + 1)),
+ StringUtils.repeat('A', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 + 1)),
// #2 single-part name exceeds part length limit (2-byte characters)
DataverseName.createSinglePartName(StringUtils.repeat(iuml,
- MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2 + 1)),
+ MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2 + 1)),
// #3. 2-part name, 2nd part exceed part length limit
DataverseName.create(Arrays.asList("A",
- StringUtils.repeat('B', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 + 1))),
+ StringUtils.repeat('B', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 + 1))),
// #4. 2-part name, 2nd part exceed part length limit (2-byte characters)
DataverseName.create(Arrays.asList("A",
StringUtils.repeat(ouml,
- MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2 + 1))),
+ MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2 + 1))),
// #5. 5-part name, each part at the part length limit, total length limit is exceeded
DataverseName.create(Arrays.asList(
- StringUtils.repeat('A', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
- StringUtils.repeat('B', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
- StringUtils.repeat('C', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
- StringUtils.repeat('D', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8),
- StringUtils.repeat('E', MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8))),
+ StringUtils.repeat('A', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('B', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('C', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('D', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8),
+ StringUtils.repeat('E', MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8))),
// #6. 5-part name, each part at the part length limit, total length limit is exceeded (2-byte characters)
DataverseName.create(Arrays.asList(
- StringUtils.repeat(iuml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
- StringUtils.repeat(ouml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
- StringUtils.repeat(iuml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
- StringUtils.repeat(ouml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2),
- StringUtils.repeat(iuml, MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 / 2))),
+ StringUtils.repeat(iuml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(ouml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(iuml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(ouml, MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2),
+ StringUtils.repeat(iuml,
+ MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 / 2))),
// #7. Multi-part name, each part at the part length limit, total length limit is exceeded
DataverseName.create(
Collections.nCopies(MetadataConstants.DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8 + 1, "A")),
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index e2505b7..9b85601 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -1749,11 +1749,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
int totalLengthUTF8 = 0;
for (String dvNamePart : dataverseName.getParts()) {
validateDatabaseObjectNameImpl(dvNamePart, sourceLoc);
- int lengthUTF8 = dvNamePart.getBytes(StandardCharsets.UTF_8).length;
- if (lengthUTF8 > MetadataConstants.DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8) {
- throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, dvNamePart);
- }
- totalLengthUTF8 += lengthUTF8;
+ totalLengthUTF8 += dvNamePart.getBytes(StandardCharsets.UTF_8).length;
}
if (totalLengthUTF8 > MetadataConstants.DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8) {
throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, dataverseName.toString());
@@ -1775,5 +1771,9 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
if (Character.isWhitespace(name.codePointAt(0))) {
throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, name);
}
+ int lengthUTF8 = name.getBytes(StandardCharsets.UTF_8).length;
+ if (lengthUTF8 > MetadataConstants.METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8) {
+ throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, name);
+ }
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
index 9b037d6..6bd52fa 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
@@ -26,7 +26,7 @@ import org.apache.asterix.common.metadata.DataverseName;
*/
public class MetadataConstants {
- public static final int DATAVERSE_NAME_PART_LENGTH_LIMIT_UTF8 = 255;
+ public static final int METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 = 255;
public static final int DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8 = 1023;
// Name of the dataverse the metadata lives in.
[asterixdb] 13/17: [ASTERIXDB-2841][*DB][STO] Encode
multiple-dataverse parts as subdirs on disk
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4b3583211aa27922555a78993b79f04afa4d8bdb
Author: Michael Blow <mb...@apache.org>
AuthorDate: Tue Mar 2 16:28:47 2021 -0500
[ASTERIXDB-2841][*DB][STO] Encode multiple-dataverse parts as subdirs on disk
- Multipart dataverse names are expressed on disk as a directory tree
- The first part is expressed normally, subsequent parts have a carat (^)
prepended
Change-Id: Idcfc45eb7f39153349a13d2baecb784244bdf177
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10324
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
---
.../common/storage/DatasetCopyIdentifier.java | 4 +-
.../asterix/common/storage/ResourceReference.java | 61 +++++++++++++++++-----
.../asterix/common/utils/StoragePathUtil.java | 18 +++++--
.../apache/asterix/external/util/FeedUtils.java | 4 +-
.../metadata/utils/SplitsAndConstraintsUtil.java | 9 ++--
5 files changed, 72 insertions(+), 24 deletions(-)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetCopyIdentifier.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetCopyIdentifier.java
index bf72c19..e520271 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetCopyIdentifier.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetCopyIdentifier.java
@@ -71,8 +71,8 @@ public class DatasetCopyIdentifier implements Serializable {
}
public boolean isMatch(ResourceReference resourceReference) {
- return resourceReference.getDataverse().equals(dataverse.getCanonicalForm())
- && resourceReference.getDataset().equals(dataset) && resourceReference.getRebalance().equals(rebalance);
+ return resourceReference.getDataverse().equals(dataverse) && resourceReference.getDataset().equals(dataset)
+ && resourceReference.getRebalance().equals(rebalance);
}
@Override
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
index 7791926..0e78152 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
@@ -21,24 +21,34 @@ package org.apache.asterix.common.storage;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.utils.StorageConstants;
+import org.apache.asterix.common.utils.StoragePathUtil;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.storage.am.lsm.common.impls.IndexComponentFileReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class ResourceReference {
+ private static final Logger LOGGER = LogManager.getLogger();
protected final String root;
protected final String partition;
- protected final String dataverse; // == DataverseName.getCanonicalForm()
+ protected final DataverseName dataverse;
protected final String dataset;
protected final String rebalance;
protected final String index;
protected final String name;
- private volatile Path relativePath;
+ private final Path relativePath;
protected ResourceReference(String path) {
// format: root/partition/dataverse/dataset/rebalanceCount/index/fileName
+ // format: root/partition/dataverse_p1[/^dataverse_p2[/^dataverse_p3...]]/dataset/rebalanceCount/index/fileName
final String[] tokens = StringUtils.split(path, File.separatorChar);
if (tokens.length < 6) {
throw new IllegalStateException("Unrecognized path structure: " + path);
@@ -48,9 +58,40 @@ public class ResourceReference {
index = tokens[--offset];
rebalance = tokens[--offset];
dataset = tokens[--offset];
- dataverse = tokens[--offset]; //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
- partition = tokens[--offset];
- root = tokens[--offset];
+ List<String> dvParts = new ArrayList<>();
+ String dvPart = tokens[--offset];
+ while (dvPart.charAt(0) == StoragePathUtil.DATAVERSE_CONTINUATION_MARKER) {
+ dvParts.add(dvPart.substring(1));
+ dvPart = tokens[--offset];
+ }
+ String probablyPartition = tokens[--offset];
+ if (dvParts.isEmpty()) {
+ // root/partition/dataverse/dataset/rebalanceCount/index/fileName
+ dataverse = DataverseName.createSinglePartName(dvPart);
+ partition = probablyPartition;
+ root = tokens[--offset];
+ } else if (probablyPartition.startsWith(StorageConstants.PARTITION_DIR_PREFIX)) {
+ // root/partition/dataverse_p1/^dataverse_p2/.../^dataverse_pn/dataset/rebalanceCount/index/fileName
+ dvParts.add(dvPart);
+ Collections.reverse(dvParts);
+ dataverse = DataverseName.create(dvParts);
+ partition = probablyPartition;
+ root = tokens[--offset];
+ } else if (dvPart.startsWith(StorageConstants.PARTITION_DIR_PREFIX)) {
+ // root/partition/dataverse/dataset/rebalanceCount/index/fileName (where dataverse starts with ^)
+ if (dvParts.size() != 1) {
+ throw new IllegalArgumentException("unable to parse path: '" + path + "'!");
+ }
+ dataverse =
+ DataverseName.createSinglePartName(StoragePathUtil.DATAVERSE_CONTINUATION_MARKER + dvParts.get(0));
+ LOGGER.info("legacy dataverse starting with ^ found: '{}'; this is not supported for new dataverses",
+ dataverse);
+ partition = dvPart;
+ root = probablyPartition;
+ } else {
+ throw new IllegalArgumentException("unable to parse path: '" + path + "'!");
+ }
+ relativePath = Paths.get(root, ArrayUtils.subarray(tokens, offset + 1, tokens.length - 1));
}
public static ResourceReference ofIndex(String indexPath) {
@@ -65,7 +106,7 @@ public class ResourceReference {
return partition;
}
- public String getDataverse() { //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
+ public DataverseName getDataverse() {
return dataverse;
}
@@ -86,19 +127,15 @@ public class ResourceReference {
}
public Path getRelativePath() {
- if (relativePath == null) {
- relativePath = Paths.get(root, partition, dataverse, dataset, rebalance, index);
- }
return relativePath;
}
public ResourceReference getDatasetReference() {
- return ResourceReference
- .ofIndex(Paths.get(root, partition, dataverse, dataset, rebalance, dataset).toFile().getPath());
+ return ResourceReference.ofIndex(relativePath.getParent().resolve(dataset).toFile().getPath());
}
public Path getFileRelativePath() {
- return Paths.get(root, partition, dataverse, dataset, rebalance, index, name);
+ return relativePath.resolve(name);
}
public int getPartitionNum() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
index 587b8b3..32a226e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -21,6 +21,7 @@ package org.apache.asterix.common.utils;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Iterator;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.metadata.DataverseName;
@@ -40,6 +41,7 @@ import org.apache.logging.log4j.Logger;
public class StoragePathUtil {
private static final Logger LOGGER = LogManager.getLogger();
+ public static final char DATAVERSE_CONTINUATION_MARKER = '^';
private StoragePathUtil() {
}
@@ -66,11 +68,21 @@ public class StoragePathUtil {
public static String prepareDataverseIndexName(DataverseName dataverseName, String datasetName, String idxName,
long rebalanceCount) {
- return prepareDataverseIndexName(dataverseName, prepareFullIndexName(datasetName, idxName, rebalanceCount));
+ return prepareDataverseComponentName(dataverseName, prepareFullIndexName(datasetName, idxName, rebalanceCount));
}
- public static String prepareDataverseIndexName(DataverseName dataverseName, String fullIndexName) {
- return dataverseName.getCanonicalForm() + File.separator + fullIndexName; //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
+ public static String prepareDataverseName(DataverseName dataverseName) {
+ Iterator<String> dvParts = dataverseName.getParts().iterator();
+ StringBuilder builder = new StringBuilder();
+ builder.append(dvParts.next());
+ while (dvParts.hasNext()) {
+ builder.append(File.separatorChar).append(DATAVERSE_CONTINUATION_MARKER).append(dvParts.next());
+ }
+ return builder.toString();
+ }
+
+ public static String prepareDataverseComponentName(DataverseName dataverseName, String component) {
+ return prepareDataverseName(dataverseName) + File.separatorChar + component;
}
private static String prepareFullIndexName(String datasetName, String idxName, long rebalanceCount) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index 2110dee..7f3d911 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -86,10 +86,10 @@ public class FeedUtils {
public static FileSplit splitsForAdapter(DataverseName dataverseName, String feedName, String nodeName,
ClusterPartition partition) {
- String relPathFile = dataverseName.getCanonicalForm() + File.separator + feedName; //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
+ String relPathFile = StoragePathUtil.prepareDataverseComponentName(dataverseName, feedName);
String storagePartitionPath = StoragePathUtil.prepareStoragePartitionPath(partition.getPartitionId());
// Note: feed adapter instances in a single node share the feed logger
- // format: 'storage dir name'/partition_#/dataverse/feed/node
+ // format: 'storage dir name'/partition_#/dataverse_part1[/ dataverse_part2[...]]/feed/node
File f = new File(storagePartitionPath + File.separator + relPathFile + File.separator + nodeName);
return StoragePathUtil.getFileSplitForClusterPartition(partition, f.getPath());
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
index b93674c..c85f661 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
@@ -48,11 +48,10 @@ public class SplitsAndConstraintsUtil {
DataverseName dataverseName) {
List<FileSplit> splits = new ArrayList<>();
// get all partitions
- ClusterPartition[] clusterPartition = clusterStateManager.getClusterPartitons();
- for (int j = 0; j < clusterPartition.length; j++) {
- File f = new File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition[j].getPartitionId()),
- dataverseName.getCanonicalForm()); //TODO(MULTI_PART_DATAVERSE_NAME):REVISIT
- splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition[j], f.getPath()));
+ for (ClusterPartition clusterPartition : clusterStateManager.getClusterPartitons()) {
+ File f = new File(StoragePathUtil.prepareStoragePartitionPath(clusterPartition.getPartitionId()),
+ StoragePathUtil.prepareDataverseName(dataverseName));
+ splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition, f.getPath()));
}
return splits.toArray(new FileSplit[] {});
}
[asterixdb] 06/17: [NO ISSUE][COMP] CREATE FUNCTION IF NOT EXISTS
improvements
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 265e3ce136970640709242658e234a77e7d1b06b
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue Mar 2 12:22:38 2021 -0800
[NO ISSUE][COMP] CREATE FUNCTION IF NOT EXISTS improvements
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Move IF NOT EXISTS modifier after function parameter list
in CREATE FUNCTION
- Parser now prohibits IF NOT EXISTS with OR REPLACE in
CREATE FUNCTION. Remove this check from QueryTranslator
- Add testcases and update documentation
Change-Id: Iddaab4f829574de47cb9baf42cc7def92429f2c8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10323
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../apache/asterix/app/translator/QueryTranslator.java | 8 ++------
.../bad-ext-function-ddl-1.4.ddl.sqlpp} | 12 ++++++------
.../external-library/mysum/mysum.2.ddl.sqlpp | 2 +-
.../bad-function-ddl-11/bad-function-ddl-11.4.ddl.sqlpp} | 15 ++++++++++-----
.../src/test/resources/runtimets/testsuite_it_sqlpp.xml | 1 +
.../src/test/resources/runtimets/testsuite_sqlpp.xml | 1 +
asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf | 2 +-
asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 16 +++++++++-------
8 files changed, 31 insertions(+), 26 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 0a34559..233a678 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2040,14 +2040,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
List<TypeSignature> existingInlineTypes;
Function existingFunction = MetadataManager.INSTANCE.getFunction(mdTxnCtx, functionSignature);
if (existingFunction != null) {
- if (cfs.getReplaceIfExists()) {
- if (cfs.getIfNotExists()) {
- throw new CompilationException(ErrorCode.PARSE_ERROR, cfs.getSourceLocation(), "IF NOT EXISTS");
- }
- } else if (cfs.getIfNotExists()) {
+ if (cfs.getIfNotExists()) {
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
return;
- } else {
+ } else if (!cfs.getReplaceIfExists()) {
throw new CompilationException(ErrorCode.FUNCTION_EXISTS, cfs.getSourceLocation(),
functionSignature.toString(false));
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/bad-ext-function-ddl-1/bad-ext-function-ddl-1.4.ddl.sqlpp
similarity index 69%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/bad-ext-function-ddl-1/bad-ext-function-ddl-1.4.ddl.sqlpp
index 9f325a3..3aa6169 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/bad-ext-function-ddl-1/bad-ext-function-ddl-1.4.ddl.sqlpp
@@ -17,10 +17,10 @@
* under the License.
*/
-create function externallibtest.mysum(a: int32, b: int32) returns int32
- as "org.apache.asterix.external.library.MySumFactory" at externallibtest.testlib;
-
-/* test if not exists */
-create function externallibtest.mysum if not exists (a: int32, b: int32) returns int32
- as "org.apache.asterix.external.library.MySumFactory" at externallibtest.testlib;
+/*
+ * Description : IF NOT EXISTS is not allowed if OR REPLACE is present
+ * Expected Res : Error
+ */
+create or replace function externallibtest.f4(a) if not exists
+ as "org.apache.asterix.external.library.OpenCapitalFinderFactory" at testlib;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp
index 9f325a3..3acb768 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp
@@ -21,6 +21,6 @@ create function externallibtest.mysum(a: int32, b: int32) returns int32
as "org.apache.asterix.external.library.MySumFactory" at externallibtest.testlib;
/* test if not exists */
-create function externallibtest.mysum if not exists (a: int32, b: int32) returns int32
+create function externallibtest.mysum(a: int32, b: int32) if not exists returns int32
as "org.apache.asterix.external.library.MySumFactory" at externallibtest.testlib;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/bad-function-ddl-11/bad-function-ddl-11.4.ddl.sqlpp
similarity index 69%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/bad-function-ddl-11/bad-function-ddl-11.4.ddl.sqlpp
index 9f325a3..73cad66 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/mysum/mysum.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/bad-function-ddl-11/bad-function-ddl-11.4.ddl.sqlpp
@@ -17,10 +17,15 @@
* under the License.
*/
-create function externallibtest.mysum(a: int32, b: int32) returns int32
- as "org.apache.asterix.external.library.MySumFactory" at externallibtest.testlib;
+/*
+ * Description : IF NOT EXISTS is not allowed if OR REPLACE is present
+ * Expected Res : Error
+ */
-/* test if not exists */
-create function externallibtest.mysum if not exists (a: int32, b: int32) returns int32
- as "org.apache.asterix.external.library.MySumFactory" at externallibtest.testlib;
+drop dataverse experiments4 if exists;
+create dataverse experiments4;
+use experiments4;
+create or replace function myfn004() if not exists {
+ 1
+};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 1ebc78c..a9f52b4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -27,6 +27,7 @@
<compilation-unit name="bad-ext-function-ddl-1">
<output-dir compare="Text">none</output-dir>
<expected-error>ASX1079: Compilation error: Variable number of parameters is not supported for external functions</expected-error>
+ <expected-error>ASX1001: Syntax error: Unexpected IF NOT EXISTS (in line 25, at column 1)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index d2135df..8963f49 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -12012,6 +12012,7 @@
<expected-error>ASX1001: Syntax error: Unexpected type declaration for parameter a in function myfn001</expected-error>
<expected-error>ASX1001: Syntax error: Unexpected return type declaration for function myfn002</expected-error>
<expected-error>ASX1001: Syntax error: Unexpected return type declaration for function myfn003</expected-error>
+ <expected-error>ASX1001: Syntax error: Unexpected IF NOT EXISTS (in line 29, at column 1)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
diff --git a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
index aae81ec..cf7d4a5 100644
--- a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
+++ b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
@@ -226,7 +226,7 @@ CreateSynonym ::= "CREATE" "SYNONYM" QualifiedName "FOR" QualifiedName ("IF" "NO
FunctionDeclaration ::= "DECLARE" "FUNCTION" Identifier "(" ( (Identifier ("," Identifier)*) | "..." )? ")" "{" Expr "}"
-CreateFunction ::= "CREATE" ("OR" "REPLACE")? "FUNCTION" QualifiedName ("IF" "NOT" "EXISTS")? "(" FunctionParameters? ")"
+CreateFunction ::= "CREATE" ("OR" "REPLACE")? "FUNCTION" QualifiedName "(" FunctionParameters? ")" ("IF" "NOT" "EXISTS")?
( ("{" Expr "}") | ExternalFunctionDef )
FunctionParameters ::= ( Identifier ((":")? TypeExpr)? ("," Identifier ((":")? TypeExpr)? )* ) | "..."
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 687920a..77765b1 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -1259,12 +1259,13 @@ CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orRe
{
defaultDataverse = fctName.dataverse;
}
- ifNotExists = IfNotExists()
paramsWithArity = FunctionParameters()
{
arity = paramsWithArity.first;
params = paramsWithArity.second;
+ signature = new FunctionSignature(fctName.dataverse, fctName.function, arity);
}
+ ifNotExists = IfNotExists()
returnType = FunctionReturnType()
(
(
@@ -1279,13 +1280,10 @@ CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orRe
endPos = token;
String functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine,
endPos.beginColumn);
- signature = new FunctionSignature(fctName.dataverse, fctName.function, arity);
getCurrentScope().addFunctionDescriptor(signature, false);
removeCurrentScope();
- defaultDataverse = currentDataverse;
ensureNoTypeDeclsInFunction(fctName.function, params, returnType, startStmtToken);
stmt = new CreateFunctionStatement(signature, params, functionBody, functionBodyExpr, orReplace, ifNotExists);
- return addSourceLocation(stmt, startStmtToken);
}
)
|
@@ -1294,18 +1292,22 @@ CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orRe
<AT> libraryName = QualifiedName()
(<WITH> withOptions = RecordConstructor())?
{
- signature = new FunctionSignature(fctName.dataverse, fctName.function, arity);
- defaultDataverse = currentDataverse;
try {
stmt = new CreateFunctionStatement(signature, params, returnType, libraryName.first,
libraryName.second.getValue(), externalIdentifier, withOptions, orReplace, ifNotExists);
} catch (AlgebricksException e) {
throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
}
- return addSourceLocation(stmt, startStmtToken);
}
)
)
+ {
+ if (orReplace && ifNotExists) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken), "Unexpected IF NOT EXISTS");
+ }
+ defaultDataverse = currentDataverse;
+ return addSourceLocation(stmt, startStmtToken);
+ }
}
Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>> FunctionParameters() :
[asterixdb] 12/17: [ASTERIXDB-2841][*DB][STO] Metadata object name
restrictions
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1cb3a92e5c28dd8479634ac44d2a613e504f4d7f
Author: Michael Blow <mb...@apache.org>
AuthorDate: Tue Mar 2 17:01:41 2021 -0500
[ASTERIXDB-2841][*DB][STO] Metadata object name restrictions
- names limited to 251 bytes (UTF-8 length)
- forbid restricted characters by platform
Change-Id: Ib8f7dd09643c61e05a3d1eb798b884f0a5d93aa1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10325
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
---
.../asterix/test/metadata/MetadataManagerTest.java | 26 ++++++
.../metadata/MetadataManagerWindowsOsTest.java | 92 ++++++++++++++++++++++
.../metadata/declared/MetadataProvider.java | 4 +-
.../asterix/metadata/utils/MetadataConstants.java | 9 ++-
4 files changed, 128 insertions(+), 3 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
index f8d6aeb..946d116 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.test.metadata;
+import java.io.File;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -161,4 +162,29 @@ public class MetadataManagerTest {
}
}
}
+
+ @Test
+ public void testInvalidCharacters() throws Exception {
+ TestCaseContext.OutputFormat cleanJson = TestCaseContext.OutputFormat.CLEAN_JSON;
+
+ List<DataverseName> dvNameBadCharsList = Arrays.asList(
+ // #1. nul characters
+ DataverseName.createSinglePartName("abc\u0000def"),
+ // #2. leading whitespace
+ DataverseName.createSinglePartName(" abcdef"),
+ // #2. file separator
+ DataverseName.createSinglePartName("abc" + File.separatorChar + "def"));
+
+ ErrorCode invalidNameErrCode = ErrorCode.INVALID_DATABASE_OBJECT_NAME;
+ for (DataverseName dvNameOk : dvNameBadCharsList) {
+ String sql = String.format("create dataverse %s;", dvNameOk);
+ try {
+ testExecutor.executeSqlppUpdateOrDdl(sql, cleanJson);
+ Assert.fail("Expected failure: " + invalidNameErrCode);
+ } catch (Exception e) {
+ Assert.assertTrue("Unexpected error message: " + e.getMessage(),
+ e.getMessage().contains(invalidNameErrCode.errorCode()));
+ }
+ }
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerWindowsOsTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerWindowsOsTest.java
new file mode 100644
index 0000000..b9f25e1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataManagerWindowsOsTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.commons.lang3.SystemUtils;
+import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MetadataManagerWindowsOsTest {
+
+ static {
+ System.setProperty("os.name", "Windows");
+ }
+
+ protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
+ private static final TestExecutor testExecutor = new TestExecutor();
+ private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
+ integrationUtil.init(true, TEST_CONFIG_FILE_NAME);
+ Assert.assertTrue("wrong os reported", SystemUtils.IS_OS_WINDOWS);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ integrationUtil.deinit(true);
+ }
+
+ @Test
+ public void testInvalidCharacters() throws Exception {
+ TestCaseContext.OutputFormat cleanJson = TestCaseContext.OutputFormat.CLEAN_JSON;
+
+ List<DataverseName> dvNameBadCharsList = new ArrayList<>();
+
+ for (char c = 0; c <= 0x1F; c++) {
+ dvNameBadCharsList.add(badCharName(c));
+ }
+ dvNameBadCharsList.add(badCharName('\u007f'));
+ dvNameBadCharsList.add(badCharName('\\'));
+ dvNameBadCharsList.add(badCharName('/'));
+ dvNameBadCharsList.add(badCharName('>'));
+ dvNameBadCharsList.add(badCharName('\n'));
+ dvNameBadCharsList.add(badCharName('|'));
+
+ ErrorCode invalidNameErrCode = ErrorCode.INVALID_DATABASE_OBJECT_NAME;
+ for (DataverseName dvNameOk : dvNameBadCharsList) {
+ String sql = String.format("create dataverse %s;", dvNameOk);
+ try {
+ testExecutor.executeSqlppUpdateOrDdl(sql, cleanJson);
+ Assert.fail("Expected failure: " + invalidNameErrCode);
+ } catch (Exception e) {
+ Assert.assertTrue("Unexpected error message: " + e.getMessage(),
+ e.getMessage().contains(invalidNameErrCode.errorCode()));
+ }
+ }
+ }
+
+ @NotNull
+ protected DataverseName badCharName(char c) {
+ return DataverseName.createSinglePartName("abc" + c + "def");
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 9b85601..a534954 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.declared;
+import static org.apache.asterix.metadata.utils.MetadataConstants.METADATA_OBJECT_NAME_INVALID_CHARS;
+
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -1768,7 +1770,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
if (name == null || name.isEmpty()) {
throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, "<empty>");
}
- if (Character.isWhitespace(name.codePointAt(0))) {
+ if (Character.isWhitespace(name.codePointAt(0)) || METADATA_OBJECT_NAME_INVALID_CHARS.matcher(name).find()) {
throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, name);
}
int lengthUTF8 = name.getBytes(StandardCharsets.UTF_8).length;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
index 6bd52fa..f1412db 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
@@ -19,15 +19,20 @@
package org.apache.asterix.metadata.utils;
+import java.util.regex.Pattern;
+
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.commons.lang3.SystemUtils;
/**
* Contains metadata constants
*/
public class MetadataConstants {
- public static final int METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 = 255;
- public static final int DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8 = 1023;
+ public static final int METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 = 251;
+ public static final int DATAVERSE_NAME_TOTAL_LENGTH_LIMIT_UTF8 = METADATA_OBJECT_NAME_LENGTH_LIMIT_UTF8 * 4;
+ public static final Pattern METADATA_OBJECT_NAME_INVALID_CHARS =
+ Pattern.compile(SystemUtils.IS_OS_WINDOWS ? "[\u0000-\u001F\u007F\"*/:<>\\\\|+,.;=\\[\\]\n]" : "[\u0000/]");
// Name of the dataverse the metadata lives in.
public static final DataverseName METADATA_DATAVERSE_NAME = DataverseName.createBuiltinDataverseName("Metadata");
[asterixdb] 01/17: [NO ISSUE] Update netty
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e05261136ed2a6554c015bd306e8000c6cbb95cb
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Feb 27 09:04:57 2021 -0500
[NO ISSUE] Update netty
- io.netty:netty-all:4.1.48.Final -> 4.1.59.Final
Change-Id: I1891bbb203b6f381e126e822e390386277e147c5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10243
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
asterixdb/asterix-server/pom.xml | 4 ++--
.../src/main/appended-resources/supplemental-models.xml | 6 +++---
...ontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt} | 16 ++++++++--------
hyracks-fullstack/pom.xml | 2 +-
4 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index f60b798..877293c 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -169,8 +169,8 @@
<url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url>
</override>
<override>
- <gav>io.netty:netty-all:4.1.48.Final</gav>
- <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.48.Final/NOTICE.txt</noticeUrl>
+ <gav>io.netty:netty-all:4.1.59.Final</gav>
+ <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.59.Final/NOTICE.txt</noticeUrl>
</override>
</overrides>
<licenses>
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml
index 3283976..3f176a7 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -160,9 +160,9 @@
<artifactId>netty-all</artifactId>
<properties>
<!-- netty is ALv2, and does not contain any embedded LICENSE or NOTICE file -->
- <license.ignoreMissingEmbeddedLicense>4.1.25.Final,4.1.32.Final,4.1.42.Final,4.1.48.Final</license.ignoreMissingEmbeddedLicense>
- <license.ignoreMissingEmbeddedNotice>4.1.25.Final,4.1.32.Final,4.1.42.Final,4.1.48.Final</license.ignoreMissingEmbeddedNotice>
- <license.ignoreNoticeOverride>4.1.25.Final,4.1.32.Final,4.1.42.Final,4.1.48.Final</license.ignoreNoticeOverride>
+ <license.ignoreMissingEmbeddedLicense>4.1.59.Final</license.ignoreMissingEmbeddedLicense>
+ <license.ignoreMissingEmbeddedNotice>4.1.59.Final</license.ignoreMissingEmbeddedNotice>
+ <license.ignoreNoticeOverride>4.1.59.Final</license.ignoreNoticeOverride>
</properties>
</project>
</supplement>
diff --git a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.48.Final_NOTICE.txt b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt
similarity index 95%
rename from asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.48.Final_NOTICE.txt
rename to asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt
index d2fda58..a771573 100644
--- a/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.48.Final_NOTICE.txt
+++ b/asterixdb/src/main/licenses/content/raw.githubusercontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt
@@ -12,7 +12,7 @@ The Netty Project licenses this file to you under the Apache License,
version 2.0 (the "License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at:
- http://www.apache.org/licenses/LICENSE-2.0
+ https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
@@ -56,7 +56,7 @@ facade for Java, which can be obtained at:
* LICENSE:
* license/LICENSE.slf4j.txt (MIT License)
* HOMEPAGE:
- * http://www.slf4j.org/
+ * https://www.slf4j.org/
This product contains a modified portion of 'Apache Harmony', an open source
Java SE, which can be obtained at:
@@ -66,7 +66,7 @@ Java SE, which can be obtained at:
* LICENSE:
* license/LICENSE.harmony.txt (Apache License 2.0)
* HOMEPAGE:
- * http://archive.apache.org/dist/harmony/
+ * https://archive.apache.org/dist/harmony/
This product contains a modified portion of 'jbzip2', a Java bzip2 compression
and decompression library written by Matthew J. Francis. It can be obtained at:
@@ -148,7 +148,7 @@ equivalent functionality. It can be obtained at:
* LICENSE:
* license/LICENSE.bouncycastle.txt (MIT License)
* HOMEPAGE:
- * http://www.bouncycastle.org/
+ * https://www.bouncycastle.org/
This product optionally depends on 'Snappy', a compression library produced
by Google Inc, which can be obtained at:
@@ -180,7 +180,7 @@ framework, which can be obtained at:
* LICENSE:
* license/LICENSE.commons-logging.txt (Apache License 2.0)
* HOMEPAGE:
- * http://commons.apache.org/logging/
+ * https://commons.apache.org/logging/
This product optionally depends on 'Apache Log4J', a logging framework, which
can be obtained at:
@@ -188,7 +188,7 @@ can be obtained at:
* LICENSE:
* license/LICENSE.log4j.txt (Apache License 2.0)
* HOMEPAGE:
- * http://logging.apache.org/log4j/
+ * https://logging.apache.org/log4j/
This product optionally depends on 'Aalto XML', an ultra-high performance
non-blocking XML processor, which can be obtained at:
@@ -243,6 +243,6 @@ This private header is also used by Apple's open source
mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/).
* LICENSE:
- * license/LICENSE.dnsinfo.txt (Apache License 2.0)
+ * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0)
* HOMEPAGE:
- * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
\ No newline at end of file
+ * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
\ No newline at end of file
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index c8478e2..2f34980 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -81,7 +81,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
- <version>4.1.48.Final</version>
+ <version>4.1.59.Final</version>
</dependency>
<dependency>
<groupId>junit</groupId>
[asterixdb] 03/17: Merge branch 'gerrit/stabilization-5949a1cb71'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 604a57eafdca3a2cafe58049b545c4af476c1217
Merge: c52df3b f9f467a
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Feb 27 21:31:59 2021 -0500
Merge branch 'gerrit/stabilization-5949a1cb71'
Change-Id: I17441a402cd038e1cc9548912960bdd1f9844903
asterixdb/asterix-server/pom.xml | 4 ++--
.../src/main/appended-resources/supplemental-models.xml | 6 +++---
...ontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt} | 16 ++++++++--------
hyracks-fullstack/pom.xml | 2 +-
4 files changed, 14 insertions(+), 14 deletions(-)
diff --cc asterixdb/asterix-server/pom.xml
index 1fa9a0f,09fe21d..ba0abdd
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@@ -169,38 -169,9 +169,38 @@@
<url>https://raw.githubusercontent.com/mojohaus/appassembler/appassembler-2.0.0/LICENSE.txt</url>
</override>
<override>
- <gav>io.netty:netty-all:4.1.48.Final</gav>
- <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.48.Final/NOTICE.txt</noticeUrl>
+ <gav>io.netty:netty-all:4.1.59.Final</gav>
+ <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.59.Final/NOTICE.txt</noticeUrl>
</override>
+ <override>
+ <gav>org.reactivestreams:reactive-streams:1.0.2</gav>
+ <noticeUrl>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/COPYING.txt</noticeUrl>
+ <url>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/LICENSE.txt</url>
+ </override>
+ <override>
+ <gavs>
+ <gav>software.amazon.awssdk:sdk-core:2.10.83</gav>
+ <gav>software.amazon.awssdk:aws-core:2.10.83</gav>
+ <gav>software.amazon.awssdk:auth:2.10.83</gav>
+ <gav>software.amazon.awssdk:arns:2.10.83</gav>
+ <gav>software.amazon.awssdk:annotations:2.10.83</gav>
+ <gav>software.amazon.awssdk:utils:2.10.83</gav>
+ <gav>software.amazon.awssdk:aws-query-protocol:2.10.83</gav>
+ <gav>software.amazon.awssdk:profiles:2.10.83</gav>
+ <gav>software.amazon.awssdk:protocol-core:2.10.83</gav>
+ <gav>software.amazon.awssdk:s3:2.10.83</gav>
+ <gav>software.amazon.awssdk:netty-nio-client:2.10.83</gav>
+ <gav>software.amazon.awssdk:apache-client:2.10.83</gav>
+ <gav>software.amazon.awssdk:aws-xml-protocol:2.10.83</gav>
+ <gav>software.amazon.awssdk:regions:2.10.83</gav>
+ <gav>software.amazon.awssdk:http-client-spi:2.10.83</gav>
+ </gavs>
+ <noticeUrl>https://raw.githubusercontent.com/aws/aws-sdk-java-v2/2.10.83/NOTICE.txt</noticeUrl>
+ </override>
+ <override>
+ <gav>software.amazon.eventstream:eventstream:1.0.1</gav>
+ <noticeUrl>https://raw.githubusercontent.com/awslabs/aws-eventstream-java/7be2dd80e12f8835674c8ffb0f4a2efb64c7b585/NOTICE</noticeUrl>
+ </override>
</overrides>
<licenses>
<license>
[asterixdb] 05/17: [NO ISSUE][MTD] Add error codes in MetadataNode
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1b282efc9efc70d05957d00f6aa25edcfcef239e
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Fri Feb 26 17:53:00 2021 -0800
[NO ISSUE][MTD] Add error codes in MetadataNode
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add error codes for exceptions raised by MetadataNode
- Introduce MetadataUtil.getFullyQualifiedDisplayName()
Change-Id: Idf827fd3c0e824468634a1755c96182c62577433
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10231
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../resources/runtimets/testsuite_it_sqlpp.xml | 2 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 18 +-
.../asterix/common/exceptions/ErrorCode.java | 14 +
.../src/main/resources/asx_errormsg/en.properties | 16 +-
.../org/apache/asterix/metadata/MetadataNode.java | 283 +++++++++++----------
.../apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../asterix/metadata/utils/MetadataUtil.java | 6 +
.../apache/asterix/metadata/utils/TypeUtil.java | 2 +-
8 files changed, 201 insertions(+), 142 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
index 954252c..1ebc78c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_sqlpp.xml
@@ -63,7 +63,7 @@
<test-case FilePath="external-library">
<compilation-unit name="mysum_dropinuse">
<output-dir compare="Text">mysum_dropinuse</output-dir>
- <expected-error>Cannot drop library externallibtest.testlib being used by funciton externallibtest.mysum(2)</expected-error>
+ <expected-error>ASX1148: Cannot drop library externallibtest.testlib being used by function externallibtest.mysum(2)</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-library">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index ebc8ec0..d2135df 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -11912,7 +11912,7 @@
<test-case FilePath="cross-dataverse">
<compilation-unit name="drop-dataverse">
<output-dir compare="Text">drop-dataverse</output-dir>
- <expected-error>Cannot drop dataverse. Type a.a used by dataset b.b1</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: type a.a being used by dataset b.b1</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
@@ -12027,18 +12027,18 @@
<test-case FilePath="user-defined-functions">
<compilation-unit name="drop-dependency-1">
<output-dir compare="Text">drop-dependency-1</output-dir>
- <expected-error>Cannot drop dataverse. Function B.f0(2) depends on function C.f1(2)</expected-error>
- <expected-error>Cannot drop dataverse. Function B.f3(2) depends on function C.f2(...)</expected-error>
- <expected-error>Cannot drop dataverse. Function B.f5(...) depends on function C.f4(2)</expected-error>
- <expected-error>Cannot drop dataverse. Function B.f7(...) depends on function C.f6(...)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f1(2) being used by function B.f0(2)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f2(...) being used by function B.f3(2)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f4(2) being used by function B.f5(...)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function C.f6(...) being used by function B.f7(...)</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
<compilation-unit name="drop-dependency-2">
<output-dir compare="Text">drop-dependency-2</output-dir>
- <expected-error>Cannot drop dataverse. Function B.f2(2) depends on dataset C.TweetMessages</expected-error>
- <expected-error>Cannot drop dataverse. Function B.f3(...) depends on dataset C.TweetMessages</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset C.TweetMessages being used by function B.f2(2)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: dataset C.TweetMessages being used by function B.f3(...)</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
@@ -12780,7 +12780,7 @@
<test-case FilePath="feeds">
<compilation-unit name="drop-function-used-by-feed">
<output-dir compare="Text">drop-function-used-by-feed</output-dir>
- <expected-error>Cannot drop function experiments.test_func0(1) being used by feed connection TwitterUsers.UserFeed</expected-error>
+ <expected-error>ASX1148: Cannot drop function experiments.test_func0(1) being used by feed connection experiments.UserFeed</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
@@ -12792,7 +12792,7 @@
<test-case FilePath="feeds">
<compilation-unit name="drop-dataverse-with-function-used-by-feed">
<output-dir compare="Text">drop-dataverse-with-function-used-by-feed</output-dir>
- <expected-error>Cannot drop dataverse. Feed connection feeddv.UserFeed depends on function fundv.test_func0(1)</expected-error>
+ <expected-error>ASX1147: Cannot drop dataverse: function fundv.test_func0(1) being used by feed connection feeddv.UserFeed</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 19be2c8..47a388d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -219,6 +219,20 @@ public enum ErrorCode implements IError {
INVALID_HINT(1132),
ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED(1133),
NO_AUTH_METHOD_PROVIDED(1134),
+ NODE_EXISTS(1135),
+ NODEGROUP_EXISTS(1136),
+ COMPACTION_POLICY_EXISTS(1137),
+ EXTERNAL_FILE_EXISTS(1138),
+ FEED_EXISTS(1139),
+ FEED_POLICY_EXISTS(1140),
+ FEED_CONNECTION_EXISTS(1141),
+ LIBRARY_EXISTS(1142),
+ UNKNOWN_EXTERNAL_FILE(1143),
+ UNKNOWN_FEED(1144),
+ UNKNOWN_FEED_CONNECTION(1145),
+ UNKNOWN_FEED_POLICY(1146),
+ CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS(1147),
+ CANNOT_DROP_OBJECT_DEPENDENT_EXISTS(1148),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 3eed1d8..50f6458 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -203,7 +203,7 @@
1114 = The provided external dataset configuration returned no files from the external source
1115 = Invalid name for a database object: \"%1$s\"
1116 = Cannot find synonym with name %1$s
-1117 = Unknown library %1$s
+1117 = Cannot find library with name %1$s
1118 = Too many grouping sets in group by clause: %1$s. Maximum allowed: %2$s.
1119 = Invalid argument to grouping() function
1120 = Unexpected alias: %1$s
@@ -221,6 +221,20 @@
1132 = Invalid specification for hint %1$s. %2$s
1133 = Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature
1134 = No authentication parameters provided
+1135 = A node with this name %1$s already exists
+1136 = A node group with this name %1$s already exists
+1137 = A compaction policy with this name %1$s already exists
+1138 = A external file with this number %1$s already exists in dataset %2$s
+1139 = A feed with this name %1$s already exists
+1140 = A feed policy with this name %1$s already exists
+1141 = A feed connection between feed %1$s and dataset %2$s already exists
+1142 = A library with this name %1$s already exists
+1143 = Cannot find external file with number %1$s in dataset %2$s
+1144 = Cannot find feed with name %1$s
+1145 = Cannot find feed connection between feed %1$s and dataset %2$s
+1146 = Cannot find feed policy with name %1$s
+1147 = Cannot drop dataverse: %1$s %2$s being used by %3$s %4$s
+1148 = Cannot drop %1$s %2$s being used by %3$s %4$s
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 13d6270..088bbc6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -25,12 +25,14 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.dataflow.LSMIndexUtil;
+import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
@@ -86,6 +88,7 @@ import org.apache.asterix.metadata.entitytupletranslators.NodeGroupTupleTranslat
import org.apache.asterix.metadata.entitytupletranslators.NodeTupleTranslator;
import org.apache.asterix.metadata.entitytupletranslators.SynonymTupleTranslator;
import org.apache.asterix.metadata.utils.DatasetUtil;
+import org.apache.asterix.metadata.utils.MetadataUtil;
import org.apache.asterix.metadata.utils.TypeUtil;
import org.apache.asterix.metadata.valueextractors.MetadataEntityValueExtractor;
import org.apache.asterix.metadata.valueextractors.TupleCopyValueExtractor;
@@ -283,30 +286,21 @@ public class MetadataNode implements IMetadataNode {
@Override
public <T extends IExtensionMetadataEntity> void addEntity(TxnId txnId, T entity) throws AlgebricksException {
- ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(entity.getDatasetId());
- if (index == null) {
- throw new AlgebricksException("Metadata Extension Index: " + entity.getDatasetId() + " was not found");
- }
+ ExtensionMetadataDataset<T> index = getExtensionMetadataDataset(entity.getDatasetId());
IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator(true);
addEntity(txnId, entity, tupleTranslator, index);
}
@Override
public <T extends IExtensionMetadataEntity> void upsertEntity(TxnId txnId, T entity) throws AlgebricksException {
- ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(entity.getDatasetId());
- if (index == null) {
- throw new AlgebricksException("Metadata Extension Index: " + entity.getDatasetId() + " was not found");
- }
+ ExtensionMetadataDataset<T> index = getExtensionMetadataDataset(entity.getDatasetId());
IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator(true);
upsertEntity(txnId, entity, tupleTranslator, index);
}
@Override
public <T extends IExtensionMetadataEntity> void deleteEntity(TxnId txnId, T entity) throws AlgebricksException {
- ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(entity.getDatasetId());
- if (index == null) {
- throw new AlgebricksException("Metadata Extension Index: " + entity.getDatasetId() + " was not found");
- }
+ ExtensionMetadataDataset<T> index = getExtensionMetadataDataset(entity.getDatasetId());
IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator(true);
deleteEntity(txnId, entity, tupleTranslator, index);
}
@@ -314,15 +308,21 @@ public class MetadataNode implements IMetadataNode {
@Override
public <T extends IExtensionMetadataEntity> List<T> getEntities(TxnId txnId, IExtensionMetadataSearchKey searchKey)
throws AlgebricksException {
- ExtensionMetadataDataset<T> index =
- (ExtensionMetadataDataset<T>) extensionDatasets.get(searchKey.getDatasetId());
- if (index == null) {
- throw new AlgebricksException("Metadata Extension Index: " + searchKey.getDatasetId() + " was not found");
- }
+ ExtensionMetadataDataset<T> index = getExtensionMetadataDataset(searchKey.getDatasetId());
IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator(false);
return getEntities(txnId, searchKey.getSearchKey(), tupleTranslator, index);
}
+ private <T extends IExtensionMetadataEntity> ExtensionMetadataDataset<T> getExtensionMetadataDataset(
+ ExtensionMetadataDatasetId datasetId) throws AlgebricksException {
+ ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(datasetId);
+ if (index == null) {
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.METADATA_ERROR,
+ "Metadata Extension Index: " + datasetId + " was not found");
+ }
+ return index;
+ }
+
@Override
public void addDataverse(TxnId txnId, Dataverse dataverse) throws AlgebricksException {
try {
@@ -331,8 +331,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.DATAVERSE_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException(
- "A dataverse with this name " + dataverse.getDataverseName() + " already exists.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.DATAVERSE_EXISTS, e,
+ dataverse.getDataverseName());
} else {
throw new AlgebricksException(e);
}
@@ -357,8 +357,8 @@ public class MetadataNode implements IMetadataNode {
}
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A dataset with this name " + dataset.getDatasetName()
- + " already exists in dataverse '" + dataset.getDataverseName() + "'.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.DATASET_EXISTS, e,
+ dataset.getDatasetName(), dataset.getDataverseName());
} else {
throw new AlgebricksException(e);
}
@@ -373,7 +373,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.INDEX_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("An index with name '" + index.getIndexName() + "' already exists.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.INDEX_EXISTS, e,
+ index.getIndexName());
} else {
throw new AlgebricksException(e);
}
@@ -388,7 +389,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.NODE_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A node with name '" + node.getNodeName() + "' already exists.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.NODE_EXISTS, e,
+ node.getNodeName());
} else {
throw new AlgebricksException(e);
}
@@ -403,8 +405,8 @@ public class MetadataNode implements IMetadataNode {
modifyMetadataIndex(modificationOp, txnId, MetadataPrimaryIndexes.NODEGROUP_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException(
- "A nodegroup with name '" + nodeGroup.getNodeGroupName() + "' already exists.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.NODEGROUP_EXISTS, e,
+ nodeGroup.getNodeGroupName());
} else {
throw new AlgebricksException(e);
}
@@ -420,8 +422,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.DATATYPE_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException(
- "A datatype with name '" + datatype.getDatatypeName() + "' already exists.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.TYPE_EXISTS, e,
+ datatype.getDatatypeName());
} else {
throw new AlgebricksException(e);
}
@@ -438,8 +440,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.FUNCTION_DATASET, functionTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A function with this name " + function.getSignature()
- + " already exists in dataverse '" + function.getDataverseName() + "'.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.FUNCTION_EXISTS, e,
+ function.getName());
} else {
throw new AlgebricksException(e);
}
@@ -583,8 +585,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.DATAVERSE_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop dataverse '" + dataverseName + "' because it doesn't exist.",
- e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_DATAVERSE, e,
+ dataverseName);
} else {
throw new AlgebricksException(e);
}
@@ -612,7 +614,8 @@ public class MetadataNode implements IMetadataNode {
Dataset dataset = getDataset(txnId, dataverseName, datasetName);
if (dataset == null) {
- throw new AlgebricksException("Cannot drop dataset '" + datasetName + "' because it doesn't exist.");
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE,
+ datasetName, dataverseName);
}
try {
// Delete entry from the 'datasets' dataset.
@@ -668,8 +671,7 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.INDEX_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException(
- "Cannot drop index '" + datasetName + "." + indexName + "' because it doesn't exist.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_INDEX, e, indexName);
} else {
throw new AlgebricksException(e);
}
@@ -678,18 +680,15 @@ public class MetadataNode implements IMetadataNode {
@Override
public boolean dropNodegroup(TxnId txnId, String nodeGroupName, boolean failSilently) throws AlgebricksException {
- List<String> datasetNames = getDatasetNamesPartitionedOnThisNodeGroup(txnId, nodeGroupName);
- if (!datasetNames.isEmpty()) {
+ List<Dataset> datasets = getDatasetsPartitionedOnThisNodeGroup(txnId, nodeGroupName);
+ if (!datasets.isEmpty()) {
if (failSilently) {
return false;
}
- StringBuilder sb = new StringBuilder();
- sb.append("Nodegroup '" + nodeGroupName
- + "' cannot be dropped; it was used for partitioning these datasets:");
- for (int i = 0; i < datasetNames.size(); i++) {
- sb.append("\n" + (i + 1) + "- " + datasetNames.get(i) + ".");
- }
- throw new AlgebricksException(sb.toString());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "node group",
+ nodeGroupName, "datasets",
+ datasets.stream().map(DatasetUtil::getFullyQualifiedDisplayName).collect(Collectors.joining(", ")));
}
try {
ITupleReference searchKey = createTuple(nodeGroupName);
@@ -700,8 +699,8 @@ public class MetadataNode implements IMetadataNode {
return true;
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop nodegroup '" + nodeGroupName + "' because it doesn't exist",
- e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_NODEGROUP, e,
+ nodeGroupName);
} else {
throw new AlgebricksException(e);
}
@@ -735,7 +734,8 @@ public class MetadataNode implements IMetadataNode {
}
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop type '" + datatypeName + "' because it doesn't exist", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_TYPE, e,
+ datatypeName);
} else {
throw new AlgebricksException(e);
}
@@ -752,7 +752,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.DATATYPE_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop type '" + datatypeName + "' because it doesn't exist", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_TYPE, e,
+ datatypeName);
} else {
throw new AlgebricksException(e);
}
@@ -932,16 +933,18 @@ public class MetadataNode implements IMetadataNode {
continue;
}
if (set.getItemTypeDataverseName().equals(dataverseName)) {
- throw new AlgebricksException("Cannot drop dataverse. Type "
- + TypeUtil.getFullyQualifiedDisplayName(set.getItemTypeDataverseName(), set.getItemTypeName())
- + " used by dataset " + DatasetUtil.getFullyQualifiedDisplayName(set));
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, "type",
+ TypeUtil.getFullyQualifiedDisplayName(set.getItemTypeDataverseName(), set.getItemTypeName()),
+ "dataset", DatasetUtil.getFullyQualifiedDisplayName(set));
}
if (set.getMetaItemTypeDataverseName() != null
&& set.getMetaItemTypeDataverseName().equals(dataverseName)) {
- throw new AlgebricksException("Cannot drop dataverse. Type "
- + TypeUtil.getFullyQualifiedDisplayName(set.getMetaItemTypeDataverseName(),
- set.getMetaItemTypeName())
- + " used by dataset " + DatasetUtil.getFullyQualifiedDisplayName(set));
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, "type",
+ TypeUtil.getFullyQualifiedDisplayName(set.getMetaItemTypeDataverseName(),
+ set.getMetaItemTypeName()),
+ "dataset", DatasetUtil.getFullyQualifiedDisplayName(set));
}
}
@@ -955,22 +958,28 @@ public class MetadataNode implements IMetadataNode {
}
for (Triple<DataverseName, String, String> datasetDependency : function.getDependencies().get(0)) {
if (datasetDependency.first.equals(dataverseName)) {
- throw new AlgebricksException("Cannot drop dataverse. Function " + function.getSignature()
- + " depends on dataset " + DatasetUtil.getFullyQualifiedDisplayName(datasetDependency.first,
- datasetDependency.second));
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS,
+ "dataset",
+ DatasetUtil.getFullyQualifiedDisplayName(datasetDependency.first, datasetDependency.second),
+ "function", function.getSignature());
}
}
for (Triple<DataverseName, String, String> functionDependency : function.getDependencies().get(1)) {
if (functionDependency.first.equals(dataverseName)) {
- throw new AlgebricksException("Cannot drop dataverse. Function " + function.getSignature()
- + " depends on function " + new FunctionSignature(functionDependency.first,
- functionDependency.second, Integer.parseInt(functionDependency.third)));
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS,
+ "function", new FunctionSignature(functionDependency.first, functionDependency.second,
+ Integer.parseInt(functionDependency.third)),
+ "function", function.getSignature());
}
}
for (Triple<DataverseName, String, String> type : function.getDependencies().get(2)) {
if (type.first.equals(dataverseName)) {
- throw new AlgebricksException("Cannot drop dataverse. Function " + function.getSignature()
- + " depends on type " + TypeUtil.getFullyQualifiedDisplayName(type.first, type.second));
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS,
+ "type", TypeUtil.getFullyQualifiedDisplayName(type.first, type.second), "function",
+ function.getSignature());
}
}
}
@@ -984,9 +993,10 @@ public class MetadataNode implements IMetadataNode {
}
for (FunctionSignature functionSignature : feedConnection.getAppliedFunctions()) {
if (dataverseName.equals(functionSignature.getDataverseName())) {
- throw new AlgebricksException(
- "Cannot drop dataverse. Feed connection " + feedConnection.getDataverseName() + "."
- + feedConnection.getFeedName() + " depends on function " + functionSignature);
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS,
+ "function", functionSignature, "feed connection", MetadataUtil.getFullyQualifiedDisplayName(
+ feedConnection.getDataverseName(), feedConnection.getFeedName()));
}
}
}
@@ -1000,8 +1010,9 @@ public class MetadataNode implements IMetadataNode {
if (functionalDependency.first.equals(signature.getDataverseName())
&& functionalDependency.second.equals(signature.getName())
&& functionalDependency.third.equals(Integer.toString(signature.getArity()))) {
- throw new AlgebricksException(
- "Cannot drop function " + signature + " being used by function " + function.getSignature());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS,
+ "function", signature, "function", function.getSignature());
}
}
}
@@ -1010,8 +1021,10 @@ public class MetadataNode implements IMetadataNode {
List<FeedConnection> feedConnections = getAllFeedConnections(txnId);
for (FeedConnection feedConnection : feedConnections) {
if (feedConnection.containsFunction(signature)) {
- throw new AlgebricksException("Cannot drop function " + signature + " being used by feed connection "
- + feedConnection.getDatasetName() + "." + feedConnection.getFeedName());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "function",
+ signature, "feed connection", MetadataUtil.getFullyQualifiedDisplayName(
+ feedConnection.getDataverseName(), feedConnection.getFeedName()));
}
}
}
@@ -1023,9 +1036,10 @@ public class MetadataNode implements IMetadataNode {
for (Function function : functions) {
for (Triple<DataverseName, String, String> datasetDependency : function.getDependencies().get(0)) {
if (datasetDependency.first.equals(dataverseName) && datasetDependency.second.equals(datasetName)) {
- throw new AlgebricksException("Cannot drop dataset "
- + DatasetUtil.getFullyQualifiedDisplayName(dataverseName, datasetName)
- + " being used by function " + function.getSignature());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS,
+ "dataset", DatasetUtil.getFullyQualifiedDisplayName(dataverseName, datasetName), "function",
+ function.getSignature());
}
}
}
@@ -1043,8 +1057,10 @@ public class MetadataNode implements IMetadataNode {
for (Function function : functions) {
if (libraryName.equals(function.getLibraryName())
&& dataverseName.equals(function.getLibraryDataverseName())) {
- throw new AlgebricksException("Cannot drop library " + dataverseName + '.' + libraryName
- + " being used by funciton " + function.getSignature());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "library",
+ MetadataUtil.getFullyQualifiedDisplayName(dataverseName, libraryName), "function",
+ function.getSignature());
}
}
}
@@ -1055,9 +1071,11 @@ public class MetadataNode implements IMetadataNode {
for (DatasourceAdapter adapter : adapters) {
if (libraryName.equals(adapter.getLibraryName())
&& dataverseName.equals(adapter.getLibraryDataverseName())) {
- throw new AlgebricksException("Cannot drop library " + dataverseName + '.' + libraryName
- + " being used by adapter " + adapter.getAdapterIdentifier().getDataverseName() + '.'
- + adapter.getAdapterIdentifier().getName());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "library",
+ MetadataUtil.getFullyQualifiedDisplayName(dataverseName, libraryName), "adapter",
+ MetadataUtil.getFullyQualifiedDisplayName(adapter.getAdapterIdentifier().getDataverseName(),
+ adapter.getAdapterIdentifier().getName()));
}
}
}
@@ -1075,9 +1093,10 @@ public class MetadataNode implements IMetadataNode {
List<Dataset> datasets = getAllDatasets(txnId);
for (Dataset set : datasets) {
if (set.getItemTypeName().equals(datatypeName) && set.getItemTypeDataverseName().equals(dataverseName)) {
- throw new AlgebricksException(
- "Cannot drop type " + TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName)
- + " being used by dataset " + DatasetUtil.getFullyQualifiedDisplayName(set));
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "type",
+ TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName), "dataset",
+ DatasetUtil.getFullyQualifiedDisplayName(set));
}
}
}
@@ -1099,9 +1118,10 @@ public class MetadataNode implements IMetadataNode {
}
AbstractComplexType recType = (AbstractComplexType) dataType.getDatatype();
if (recType.containsType(typeToBeDropped)) {
- throw new AlgebricksException("Cannot drop type "
- + TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName) + " being used by type "
- + TypeUtil.getFullyQualifiedDisplayName(dataverseName, recType.getTypeName()));
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "type",
+ TypeUtil.getFullyQualifiedDisplayName(dataverseName, datatypeName), "type",
+ TypeUtil.getFullyQualifiedDisplayName(dataverseName, recType.getTypeName()));
}
}
}
@@ -1113,9 +1133,10 @@ public class MetadataNode implements IMetadataNode {
for (Function function : functions) {
for (Triple<DataverseName, String, String> datasetDependency : function.getDependencies().get(2)) {
if (datasetDependency.first.equals(dataverseName) && datasetDependency.second.equals(dataTypeName)) {
- throw new AlgebricksException(
- "Cannot drop type " + TypeUtil.getFullyQualifiedDisplayName(dataverseName, dataTypeName)
- + " is being used by function " + function.getSignature());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "type",
+ TypeUtil.getFullyQualifiedDisplayName(dataverseName, dataTypeName), "function",
+ function.getSignature());
}
}
}
@@ -1146,19 +1167,17 @@ public class MetadataNode implements IMetadataNode {
return nestedTypes;
}
- private List<String> getDatasetNamesPartitionedOnThisNodeGroup(TxnId txnId, String nodegroup)
+ private List<Dataset> getDatasetsPartitionedOnThisNodeGroup(TxnId txnId, String nodegroup)
throws AlgebricksException {
- // this needs to scan the datasets and return the datasets that use this
- // nodegroup
- List<String> nodeGroupDatasets = new ArrayList<>();
+ // this needs to scan the datasets and return the datasets that use this nodegroup
+ List<Dataset> nodeGroupDatasets = new ArrayList<>();
List<Dataset> datasets = getAllDatasets(txnId);
for (Dataset set : datasets) {
if (set.getNodeGroupName().equals(nodegroup)) {
- nodeGroupDatasets.add(set.getDatasetName());
+ nodeGroupDatasets.add(set);
}
}
return nodeGroupDatasets;
-
}
@Override
@@ -1277,8 +1296,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.FUNCTION_DATASET, functionTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException(
- "Cannot drop function '" + functionSignature + "' because it doesn't exist", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_FUNCTION, e,
+ functionSignature.toString());
} else {
throw new AlgebricksException(e);
}
@@ -1374,7 +1393,8 @@ public class MetadataNode implements IMetadataNode {
IValueExtractor<T> valueExtractor, List<T> results) throws AlgebricksException, HyracksDataException {
IBinaryComparatorFactory[] comparatorFactories = index.getKeyBinaryComparatorFactory();
if (index.getFile() == null) {
- throw new AlgebricksException("No file for Index " + index.getDataverseName() + "." + index.getIndexName());
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.METADATA_ERROR,
+ "No file for Index " + index.getDataverseName() + "." + index.getIndexName());
}
String resourceName = index.getFile().getRelativePath();
IIndex indexInstance = datasetLifecycleManager.get(resourceName);
@@ -1514,9 +1534,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET, adapterTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A adapter with this name " + adapter.getAdapterIdentifier().getName()
- + " already exists in dataverse '" + adapter.getAdapterIdentifier().getDataverseName() + "'.",
- e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.ADAPTER_EXISTS, e,
+ adapter.getAdapterIdentifier().getName());
} else {
throw new AlgebricksException(e);
}
@@ -1525,10 +1544,6 @@ public class MetadataNode implements IMetadataNode {
@Override
public void dropAdapter(TxnId txnId, DataverseName dataverseName, String adapterName) throws AlgebricksException {
- DatasourceAdapter adapter = getAdapter(txnId, dataverseName, adapterName);
- if (adapter == null) {
- throw new AlgebricksException("Cannot drop adapter '" + adapter + "' because it doesn't exist.");
- }
try {
// Delete entry from the 'Adapter' dataset.
ITupleReference searchKey = createTuple(dataverseName, adapterName);
@@ -1539,7 +1554,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET, datasetTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop adapter '" + adapterName + " since it doesn't exist", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_ADAPTER, e,
+ adapterName);
} else {
throw new AlgebricksException(e);
}
@@ -1575,8 +1591,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET, compactionPolicyTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A compaction policy with this name " + compactionPolicy.getPolicyName()
- + " already exists in dataverse '" + compactionPolicy.getPolicyName() + "'.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.COMPACTION_POLICY_EXISTS, e,
+ compactionPolicy.getPolicyName());
} else {
throw new AlgebricksException(e);
}
@@ -1627,8 +1643,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.LIBRARY_DATASET, libraryTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A library with this name " + library.getDataverseName()
- + " already exists in dataverse '" + library.getDataverseName() + "'.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.LIBRARY_EXISTS, e,
+ library.getName());
} else {
throw new AlgebricksException(e);
}
@@ -1654,7 +1670,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.LIBRARY_DATASET, datasetTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop library '" + libraryName + "' because it doesn't exist", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_LIBRARY, e,
+ libraryName);
} else {
throw new AlgebricksException(e);
}
@@ -1692,8 +1709,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, feedPolicyTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A feed policy with this name " + feedPolicy.getPolicyName()
- + " already exists in dataverse '" + feedPolicy.getPolicyName() + "'.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.FEED_POLICY_EXISTS, e,
+ feedPolicy.getPolicyName());
} else {
throw new AlgebricksException(e);
}
@@ -1726,7 +1743,12 @@ public class MetadataNode implements IMetadataNode {
ITupleReference feedConnTuple = tupleReaderWriter.getTupleFromMetadataEntity(feedConnection);
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.FEED_CONNECTION_DATASET, feedConnTuple);
} catch (HyracksDataException e) {
- throw new AlgebricksException(e);
+ if (e.matches(ErrorCode.DUPLICATE_KEY)) {
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.FEED_CONNECTION_EXISTS, e,
+ feedConnection.getFeedName(), feedConnection.getDatasetName());
+ } else {
+ throw new AlgebricksException(e);
+ }
}
}
@@ -1774,7 +1796,12 @@ public class MetadataNode implements IMetadataNode {
getTupleToBeDeleted(txnId, MetadataPrimaryIndexes.FEED_CONNECTION_DATASET, searchKey);
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.FEED_CONNECTION_DATASET, tuple);
} catch (HyracksDataException e) {
- throw new AlgebricksException(e);
+ if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_FEED_CONNECTION, e,
+ feedName, datasetName);
+ } else {
+ throw new AlgebricksException(e);
+ }
}
}
@@ -1787,8 +1814,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.FEED_DATASET, feedTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A feed with this name " + feed.getFeedName()
- + " already exists in dataverse '" + feed.getDataverseName() + "'.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.FEED_EXISTS, e,
+ feed.getFeedName());
} else {
throw new AlgebricksException(e);
}
@@ -1836,7 +1863,7 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.FEED_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop feed '" + feedName + "' because it doesn't exist", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_FEED, e, feedName);
} else {
throw new AlgebricksException(e);
}
@@ -1851,7 +1878,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, tuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Unknown feed policy " + policyName, e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_FEED_POLICY, e,
+ policyName);
} else {
throw new AlgebricksException(e);
}
@@ -1883,9 +1911,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET, externalFileTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("An externalFile with this number " + externalFile.getFileNumber()
- + " already exists in dataset '" + externalFile.getDatasetName() + "' in dataverse '"
- + externalFile.getDataverseName() + "'.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_FILE_EXISTS, e,
+ externalFile.getFileNumber(), externalFile.getDatasetName());
} else {
throw new AlgebricksException(e);
}
@@ -1920,7 +1947,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET, datasetTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Couldn't drop externalFile.", e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_EXTERNAL_FILE, e,
+ fileNumber, datasetName);
} else {
throw new AlgebricksException(e);
}
@@ -1996,8 +2024,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.SYNONYM_DATASET, synonymTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.DUPLICATE_KEY)) {
- throw new AlgebricksException("A synonym with name '" + synonym.getSynonymName() + "' already exists.",
- e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.SYNONYM_EXISTS, e,
+ synonym.getSynonymName());
} else {
throw new AlgebricksException(e);
}
@@ -2006,10 +2034,6 @@ public class MetadataNode implements IMetadataNode {
@Override
public void dropSynonym(TxnId txnId, DataverseName dataverseName, String synonymName) throws AlgebricksException {
- Synonym synonym = getSynonym(txnId, dataverseName, synonymName);
- if (synonym == null) {
- throw new AlgebricksException("Cannot drop synonym '" + synonym + "' because it doesn't exist.");
- }
try {
// Delete entry from the 'Synonym' dataset.
ITupleReference searchKey = createTuple(dataverseName, synonymName);
@@ -2020,7 +2044,8 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(txnId, MetadataPrimaryIndexes.SYNONYM_DATASET, synonymTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException("Cannot drop synonym '" + synonymName, e);
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_SYNONYM, e,
+ synonymName);
} else {
throw new AlgebricksException(e);
}
@@ -2074,8 +2099,8 @@ public class MetadataNode implements IMetadataNode {
insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple);
} catch (HyracksDataException e) {
if (e.matches(ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY)) {
- throw new AlgebricksException(
- "Cannot drop dataset '" + dataset.getDatasetName() + "' because it doesn't exist");
+ throw new AsterixException(org.apache.asterix.common.exceptions.ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE,
+ e, dataset.getDatasetName(), dataset.getDataverseName());
} else {
throw new AlgebricksException(e);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 52a133d..4bcc5f0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -563,7 +563,7 @@ public class DatasetUtil {
}
public static String getFullyQualifiedDisplayName(DataverseName dataverseName, String datasetName) {
- return dataverseName + "." + datasetName;
+ return MetadataUtil.getFullyQualifiedDisplayName(dataverseName, datasetName);
}
/***
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java
index 3133aba..7bc6e98 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.metadata.utils;
+import org.apache.asterix.common.metadata.DataverseName;
+
public class MetadataUtil {
public static final int PENDING_NO_OP = 0;
public static final int PENDING_ADD_OP = 1;
@@ -38,4 +40,8 @@ public class MetadataUtil {
return "Unknown Pending Operation";
}
}
+
+ public static String getFullyQualifiedDisplayName(DataverseName dataverseName, String objectName) {
+ return dataverseName + "." + objectName;
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
index 7660909..65a800e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
@@ -289,7 +289,7 @@ public class TypeUtil {
}
public static String getFullyQualifiedDisplayName(DataverseName dataverseName, String typeName) {
- return dataverseName + "." + typeName;
+ return MetadataUtil.getFullyQualifiedDisplayName(dataverseName, typeName);
}
/**
[asterixdb] 02/17: Merge branch 'gerrit/stabilization-f69489'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit f9f467a91f447651f4c8b16707644904db7a03a3
Merge: 5a377cf e052611
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Feb 27 16:14:51 2021 -0500
Merge branch 'gerrit/stabilization-f69489'
Change-Id: Ia2209dfb4f89da8102774cece7525348bd021523
asterixdb/asterix-server/pom.xml | 4 ++--
.../src/main/appended-resources/supplemental-models.xml | 6 +++---
...ontent.com_netty_netty_netty-4.1.59.Final_NOTICE.txt} | 16 ++++++++--------
hyracks-fullstack/pom.xml | 2 +-
4 files changed, 14 insertions(+), 14 deletions(-)
[asterixdb] 07/17: [NO ISSUE][OTH] Introduce
DatasetFullyQualifiedName
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 08200153038688b7ce392df7e53317590fc9ab02
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Mon Mar 1 23:26:59 2021 +0300
[NO ISSUE][OTH] Introduce DatasetFullyQualifiedName
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Introduce DatasetFullyQualifiedName that is used to represent
a dataset's DataverseName as well as its name.
- This new class can be used as an identifier for a dataset rather
than using a Pair object of a DataverseName and a String.
Change-Id: Ie94b5dc400c04a143f5be3b1cb2a652d765dfdba
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10303
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../common/metadata/DatasetFullyQualifiedName.java | 64 ++++++++++++++++++++++
.../apache/asterix/metadata/entities/Dataset.java | 8 +++
2 files changed, 72 insertions(+)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DatasetFullyQualifiedName.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DatasetFullyQualifiedName.java
new file mode 100644
index 0000000..4a1a118
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/DatasetFullyQualifiedName.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.metadata;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DatasetFullyQualifiedName implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final DataverseName dataverseName;
+ private final String datasetName;
+
+ public DatasetFullyQualifiedName(DataverseName dataverseName, String datasetName) {
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ }
+
+ public DataverseName getDataverseName() {
+ return dataverseName;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ @Override
+ public String toString() {
+ return dataverseName + "." + datasetName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o instanceof DatasetFullyQualifiedName) {
+ DatasetFullyQualifiedName that = (DatasetFullyQualifiedName) o;
+ return dataverseName.equals(that.dataverseName) && datasetName.equals(that.datasetName);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(dataverseName, datasetName);
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index dd3f67d..39a8eff 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -38,6 +38,7 @@ import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.ioopcallbacks.LSMIndexIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMIndexPageWriteCallbackFactory;
+import org.apache.asterix.common.metadata.DatasetFullyQualifiedName;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.IDataset;
import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
@@ -154,6 +155,7 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset {
private final long rebalanceCount;
private int pendingOp;
private final String compressionScheme;
+ private final DatasetFullyQualifiedName datasetFullyQualifiedName;
public Dataset(DataverseName dataverseName, String datasetName, DataverseName recordTypeDataverseName,
String recordTypeName, String nodeGroupName, String compactionPolicy,
@@ -203,6 +205,7 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset {
this.hints = hints;
this.rebalanceCount = rebalanceCount;
this.compressionScheme = compressionScheme;
+ datasetFullyQualifiedName = new DatasetFullyQualifiedName(dataverseName, datasetName);
}
@Override
@@ -877,4 +880,9 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset {
public String getCompressionScheme() {
return compressionScheme;
}
+
+ public DatasetFullyQualifiedName getDatasetFullyQualifiedName() {
+ return datasetFullyQualifiedName;
+ }
+
}
[asterixdb] 11/17: Merge branch 'gerrit/mad-hatter'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1d95047d63acb01a12b9067125cd7bff4aca74ab
Merge: 2ee9720 038a627
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu Mar 4 13:50:42 2021 -0500
Merge branch 'gerrit/mad-hatter'
Change-Id: I9b1eea10bffbcd2e6647a7a2100b82a276e9e1cd
hyracks-fullstack/hyracks/hyracks-api/pom.xml | 4 ++++
.../apache/hyracks/api/util/ExceptionUtils.java | 25 ++++++++++++++++++++++
.../java/org/apache/hyracks/util/MXHelper.java | 3 +--
3 files changed, 30 insertions(+), 2 deletions(-)
[asterixdb] 17/17: Merge branch 'gerrit/cheshire-cat'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b99ca94f7d62e51b0806e0615a2876a888ac0e1e
Merge: 67fd1f3 e3e4220
Author: Michael Blow <mb...@apache.org>
AuthorDate: Fri Mar 5 08:55:24 2021 -0500
Merge branch 'gerrit/cheshire-cat'
Change-Id: I6ffd99a066577e26494765f404f8694c3359d8b1
.../operators/physical/AssignBatchPOperator.java | 103 +++++
.../jobgen/QueryLogicalExpressionJobGen.java | 8 +-
.../asterix/optimizer/base/RuleCollections.java | 6 +-
.../ExtractBatchableExternalFunctionCallsRule.java | 220 +++++++++
.../rules/SetAsterixPhysicalOperatorsRule.java | 61 +++
.../apache/asterix/api/common/APIFramework.java | 4 +-
.../asterix/app/translator/QueryTranslator.java | 8 +-
.../asterix/test/metadata/MetadataManagerTest.java | 88 +++-
.../metadata/MetadataManagerWindowsOsTest.java | 92 ++++
.../bad-ext-function-ddl-1.4.ddl.sqlpp} | 12 +-
.../external-library/mysum/mysum.2.ddl.sqlpp | 2 +-
.../bad-function-ddl-11.4.ddl.sqlpp} | 15 +-
.../resources/runtimets/testsuite_it_sqlpp.xml | 3 +-
.../test/resources/runtimets/testsuite_sqlpp.xml | 23 +-
.../asterix/common/exceptions/ErrorCode.java | 26 +-
.../common/metadata/DatasetFullyQualifiedName.java | 64 +++
.../common/storage/DatasetCopyIdentifier.java | 4 +-
.../asterix/common/storage/ResourceReference.java | 61 ++-
.../asterix/common/utils/StoragePathUtil.java | 18 +-
.../src/main/resources/asx_errormsg/en.properties | 32 +-
asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf | 2 +-
.../src/main/markdown/builtins/14_window.md | 22 +-
.../src/main/markdown/builtins/15_bitwise.md | 1 +
.../src/main/markdown/builtins/9_aggregate_sql.md | 4 +-
.../main/markdown/datamodel/datamodel_composite.md | 1 +
.../datamodel/datamodel_primitive_common.md | 2 +-
.../asterix-doc/src/main/markdown/sqlpp/1_intro.md | 3 +-
.../asterix-doc/src/main/markdown/sqlpp/2_expr.md | 289 +++++-------
.../asterix-doc/src/main/markdown/sqlpp/3_query.md | 515 +++++++++------------
.../src/main/markdown/sqlpp/3_query_title.md | 20 -
.../src/main/markdown/sqlpp/4_windowfunctions.md | 136 ++----
.../src/main/markdown/sqlpp/7_ddl_dml.md | 386 +++++++--------
.../src/main/markdown/sqlpp/7_ddl_head.md | 12 +-
.../main/markdown/sqlpp/appendix_2_parameters.md | 7 +-
.../main/markdown/sqlpp/appendix_3_resolution.md | 10 +-
.../main/markdown/sqlpp/appendix_4_manual_data.md | 1 +
.../ExternalFunctionDescriptorProvider.java | 19 +-
.../library/ExternalScalarFunctionDescriptor.java | 14 +-
.../ExternalAssignBatchRuntimeFactory.java | 52 +++
.../apache/asterix/external/util/FeedUtils.java | 4 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 16 +-
.../apache/asterix/metadata/MetadataManager.java | 3 +-
.../org/apache/asterix/metadata/MetadataNode.java | 283 +++++------
.../metadata/declared/MetadataProvider.java | 13 +-
.../apache/asterix/metadata/entities/Dataset.java | 8 +
.../functions/ExternalFunctionCompilerUtil.java | 20 +
.../apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../asterix/metadata/utils/MetadataConstants.java | 9 +-
.../asterix/metadata/utils/MetadataUtil.java | 6 +
.../metadata/utils/SplitsAndConstraintsUtil.java | 9 +-
.../apache/asterix/metadata/utils/TypeUtil.java | 2 +-
.../om/functions/IExternalFunctionDescriptor.java} | 13 +-
asterixdb/asterix-server/pom.xml | 4 +-
.../appended-resources/supplemental-models.xml | 6 +-
....com_netty_netty_netty-4.1.59.Final_NOTICE.txt} | 16 +-
.../core/algebra/base/PhysicalOperatorTag.java | 2 +-
...POperator.java => AbstractAssignPOperator.java} | 36 +-
.../operators/physical/AssignPOperator.java | 84 +---
.../core/algebra/plan/PlanStabilityVerifier.java | 16 +-
.../algebra/util/OperatorManipulationUtil.java | 16 +-
.../rewriter/rules/ConsolidateAssignsRule.java | 9 +
.../rules/SetAlgebricksPhysicalOperatorsRule.java | 2 +-
hyracks-fullstack/hyracks/hyracks-api/pom.xml | 4 +
.../apache/hyracks/api/util/ExceptionUtils.java | 25 +
.../java/org/apache/hyracks/util/MXHelper.java | 3 +-
hyracks-fullstack/pom.xml | 2 +-
66 files changed, 1736 insertions(+), 1223 deletions(-)
diff --cc asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
index 8707be5,2a662d0..b1d0b47
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
@@@ -138,17 -137,9 +137,14 @@@ public class QueryLogicalExpressionJobG
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
- IFunctionDescriptor fd = null;
- IFunctionDescriptor fd = expr.getFunctionInfo() instanceof IExternalFunctionInfo
- ? ExternalFunctionDescriptorProvider.resolveExternalFunction(expr, env, context)
- : resolveFunction(expr, env, context);
++ IFunctionDescriptor fd;
+ if (expr.getFunctionInfo() instanceof IExternalFunctionInfo) {
+ // Expr is an external function
- fd = ExternalFunctionDescriptorProvider
- .getExternalFunctionDescriptor((IExternalFunctionInfo) expr.getFunctionInfo());
- CompilerProperties props = ((IApplicationContext) context.getAppContext()).getCompilerProperties();
- FunctionTypeInferers.SET_ARGUMENTS_TYPE.infer(expr, fd, env, props);
++ fd = ExternalFunctionDescriptorProvider.resolveExternalFunction(expr, env, context);
+ } else {
+ // Expr is an internal (built-in) function
+ fd = resolveFunction(expr, env, context);
+ }
return fd.createEvaluatorFactory(args);
}
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 2b4b983,8963f49..e0471ff
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@@ -7808,416 -7803,6 +7808,416 @@@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="array-index">
+ <test-group name="array-index/error-handling">
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-two-array-fields">
+ <output-dir compare="Text">index-two-array-fields</output-dir>
+ <expected-error>ASX1079: Compilation error: Cannot create composite index with multiple array fields using different arrays</expected-error>
+ <expected-error>ASX1079: Compilation error: Cannot create composite index with multiple array fields using different arrays</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="invalid-array-path">
+ <output-dir compare="Text">invalid-array-path</output-dir>
+ <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type CheckinType_checkin_time:</expected-error>
+ <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type string</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-on-closed-array">
+ <output-dir compare="Text">index-on-closed-array</output-dir>
+ <expected-error>ASX1014: Field "date" is not found</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-with-enforced-type">
+ <output-dir compare="Text">index-with-enforced-type</output-dir>
- <expected-error>ASX1140: Incompatible index type ARRAY</expected-error>
++ <expected-error>ASX1154: Incompatible index type ARRAY</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/error-handling">
+ <compilation-unit name="index-mixed-composite">
+ <output-dir compare="Text">index-mixed-composite</output-dir>
- <expected-error>ASX1140: Incompatible index type ARRAY</expected-error>
++ <expected-error>ASX1154: Incompatible index type ARRAY</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/metadata">
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <!-- <test-case FilePath="array-index/metadata/closed">-->
+ <!-- <compilation-unit name="with-composite-sk">-->
+ <!-- <output-dir compare="Text">with-composite-sk</output-dir>-->
+ <!-- </compilation-unit>-->
+ <!-- </test-case>-->
+ <!-- <test-case FilePath="array-index/metadata/open">-->
+ <!-- <compilation-unit name="with-composite-sk">-->
+ <!-- <output-dir compare="Text">with-composite-sk</output-dir>-->
+ <!-- </compilation-unit>-->
+ <!-- </test-case>-->
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="with-composite-array-different-indicators">
+ <output-dir compare="Text">with-composite-array-different-indicators</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="with-composite-array-different-indicators">
+ <output-dir compare="Text">with-composite-array-different-indicators</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/closed">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/bulk-loading/on-index-creation">
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/on-index-creation/closed">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/bulk-loading/after-index-creation">
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/bulk-loading/after-index-creation">
+ <compilation-unit name="with-open-index">
+ <output-dir compare="Text">with-open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/insert-upsert-delete">
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="with-additional-atomic-index">
+ <output-dir compare="Text">with-additional-atomic-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/open">
+ <compilation-unit name="with-additional-atomic-index">
+ <output-dir compare="Text">with-additional-atomic-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/insert-upsert-delete/closed">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/join-unnest-queries">
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/join-unnest-queries">
+ <compilation-unit name="with-open-index">
+ <output-dir compare="Text">with-open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/select-unnest-queries">
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="with-3-level-record-path">
+ <output-dir compare="Text">with-3-level-record-path</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/open">
+ <compilation-unit name="with-composite-sk">
+ <output-dir compare="Text">with-composite-sk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-unnest-queries/closed">
+ <compilation-unit name="with-filter-fields">
+ <output-dir compare="Text">with-filter-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/select-quantified-queries">
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-1">
+ <output-dir compare="Text">use-case-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-2">
+ <output-dir compare="Text">use-case-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-3">
+ <output-dir compare="Text">use-case-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="use-case-4">
+ <output-dir compare="Text">use-case-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="with-composite-pk">
+ <output-dir compare="Text">with-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="array-index/select-quantified-queries">
+ <compilation-unit name="with-open-index">
+ <output-dir compare="Text">with-open-index</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ </test-group>
<test-group name="nestrecords">
<test-case FilePath="nestrecords">
<compilation-unit name="nestrecord">
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index bcaf0ae,47a388d..0406132
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@@ -219,12 -219,20 +219,26 @@@ public enum ErrorCode implements IErro
INVALID_HINT(1132),
ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED(1133),
NO_AUTH_METHOD_PROVIDED(1134),
- FULL_TEXT_CONFIG_ALREADY_EXISTS(1135),
- FULL_TEXT_FILTER_ALREADY_EXISTS(1136),
- FULL_TEXT_CONFIG_NOT_FOUND(1137),
- FULL_TEXT_FILTER_NOT_FOUND(1138),
- FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED(1139),
- COMPILATION_INCOMPATIBLE_INDEX_TYPE(1140),
+ NODE_EXISTS(1135),
+ NODEGROUP_EXISTS(1136),
+ COMPACTION_POLICY_EXISTS(1137),
+ EXTERNAL_FILE_EXISTS(1138),
+ FEED_EXISTS(1139),
+ FEED_POLICY_EXISTS(1140),
+ FEED_CONNECTION_EXISTS(1141),
+ LIBRARY_EXISTS(1142),
+ UNKNOWN_EXTERNAL_FILE(1143),
+ UNKNOWN_FEED(1144),
+ UNKNOWN_FEED_CONNECTION(1145),
+ UNKNOWN_FEED_POLICY(1146),
+ CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS(1147),
+ CANNOT_DROP_OBJECT_DEPENDENT_EXISTS(1148),
++ FULL_TEXT_CONFIG_ALREADY_EXISTS(1149),
++ FULL_TEXT_FILTER_ALREADY_EXISTS(1150),
++ FULL_TEXT_CONFIG_NOT_FOUND(1151),
++ FULL_TEXT_FILTER_NOT_FOUND(1152),
++ FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED(1153),
++ COMPILATION_INCOMPATIBLE_INDEX_TYPE(1154),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --cc asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index f8f00db,50f6458..cc56ede
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@@ -219,14 -219,22 +219,28 @@@
1130 = Illegal use of RIGHT OUTER JOIN
1131 = A synonym with this name %1$s already exists
1132 = Invalid specification for hint %1$s. %2$s
- 1133 = Full-text filter %1$s not found
- 1134 = Default full-text config with a name of null cannot be deleted or created
- 1135 = Full-text config %1$s already exists
- 1136 = Full-text filter %1$s already exists
- 1137 = Full-text config %1$s not found
- 1138 = Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature
- 1139 = No authentication parameters provided
- 1140 = Incompatible index type %1$s
+ 1133 = Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature
+ 1134 = No authentication parameters provided
+ 1135 = A node with this name %1$s already exists
+ 1136 = A node group with this name %1$s already exists
+ 1137 = A compaction policy with this name %1$s already exists
+ 1138 = A external file with this number %1$s already exists in dataset %2$s
+ 1139 = A feed with this name %1$s already exists
+ 1140 = A feed policy with this name %1$s already exists
+ 1141 = A feed connection between feed %1$s and dataset %2$s already exists
+ 1142 = A library with this name %1$s already exists
+ 1143 = Cannot find external file with number %1$s in dataset %2$s
+ 1144 = Cannot find feed with name %1$s
+ 1145 = Cannot find feed connection between feed %1$s and dataset %2$s
+ 1146 = Cannot find feed policy with name %1$s
+ 1147 = Cannot drop dataverse: %1$s %2$s being used by %3$s %4$s
+ 1148 = Cannot drop %1$s %2$s being used by %3$s %4$s
++1149 = Full-text config %1$s already exists
++1150 = Full-text filter %1$s already exists
++1151 = Full-text config %1$s not found
++1152 = Full-text filter %1$s not found
++1153 = Default full-text config with a name of null cannot be deleted or created
++1154 = Incompatible index type %1$s
# Feed Errors
3001 = Illegal state.
diff --cc asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
index b591b3e,e064f06..b2b893b
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
@@@ -312,32 -295,23 +295,29 @@@ the URL and path needed to locate the d
#### <a id="Indices">Create Index</a>
- ---
- ### CreateIndex
- ****
+ ##### CreateIndex
+ 
- ### CreateSecondaryIndex
- ****
+ ##### CreateSecondaryIndex
+ 
- ### CreatePrimaryKeyIndex
- ****
+ ##### CreatePrimaryKeyIndex
+ 
- ### IndexedElement
++##### IndexedElement
+****
+
- ### ArrayIndexElement
++##### ArrayIndexElement
+****
+
- ### IndexField
+ ##### IndexField
-
+****
- ### NestedField
- ****
+ ##### NestedField
+ 
- ### IndexType
- ****
-
- ---
+ ##### IndexType
+ 
The `CREATE INDEX` statement creates a secondary index on one or more fields of a specified dataset.
Supported index types include `BTREE` for totally ordered datatypes, `RTREE` for spatial data,
@@@ -383,22 -355,15 +363,22 @@@ The following example creates a btree i
a nested field residing within a object-valued user field in the `orders` dataset.
This index can be useful for accelerating exact-match queries, range search queries,
and joins involving the nested `orderUserName` field.
-Such nested fields must be singular, i.e., one cannot index through (or on) an array-valued field.
- #### Example
+ ##### Example
CREATE INDEX oOrderUserNameIdx ON orders(order.orderUserName) TYPE BTREE;
+The following example creates an array index called `oItemsPriceIdx` on the `price` field inside the `items` array of the `orders` dataset.
+This index can be useful for accelerating membership queries, existential or universal quantification queries, or joins involving the `price` field inside this array.
+(To enable array index query optimization, be sure to set the [`arrayindex` compiler option](manual.html#ArrayIndexFlag).)
+
+#### Example
+
+ CREATE INDEX oItemsPriceIdx ON orders(UNNEST items SELECT price);
+
The following example creates an open rtree index called `oOrderLocIdx` on the order-location field of the `orders` dataset. This index can be useful for accelerating queries that use the [`spatial-intersect` function](builtins.html#spatial_intersect) in a predicate involving the sender-location field.
- #### Example
+ ##### Example
CREATE INDEX oOrderLocIDx ON orders(`order-location` : point?) TYPE RTREE ENFORCED;
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index 20c852a,a481f99..699ad74
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@@ -627,141 -622,6 +626,141 @@@ public abstract class MetadataManager i
}
@Override
+ public void addFullTextFilter(MetadataTransactionContext mdTxnCtx, FullTextFilterMetadataEntity filter)
+ throws AlgebricksException {
+ try {
+ metadataNode.addFullTextFilter(mdTxnCtx.getTxnId(), filter);
+ } catch (RemoteException e) {
+ throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
+ }
+ mdTxnCtx.addFullTextFilter(filter);
+ }
+
+ @Override
+ public void dropFullTextFilter(MetadataTransactionContext mdTxnCtx, DataverseName dataverseName, String filterName)
+ throws AlgebricksException {
+ try {
+ metadataNode.dropFullTextFilter(mdTxnCtx.getTxnId(), dataverseName, filterName);
+ } catch (RemoteException e) {
+ throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
+ }
+ mdTxnCtx.dropFullTextFilter(dataverseName, filterName);
+ }
+
+ @Override
+ public FullTextFilterMetadataEntity getFullTextFilter(MetadataTransactionContext ctx, DataverseName dataverseName,
+ String filterName) throws AlgebricksException {
+ // First look in the context to see if this transaction created the
+ // requested full-text filter itself (but the full-text filter is still uncommitted).
+ FullTextFilterMetadataEntity filter = ctx.getFullTextFilter(dataverseName, filterName);
+ if (filter != null) {
+ // Don't add this filter to the cache, since it is still
+ // uncommitted.
+ return filter;
+ }
+
+ if (ctx.fullTextFilterIsDropped(dataverseName, filterName)) {
+ // Filter has been dropped by this transaction but could still be
+ // in the cache.
+ return null;
+ }
+
+ if (ctx.getDataverse(dataverseName) != null) {
+ // This transaction has dropped and subsequently created the same
+ // dataverse.
+ return null;
+ }
+
+ filter = cache.getFullTextFilter(dataverseName, filterName);
+ if (filter != null) {
+ // filter is already in the cache, don't add it again.
+ return filter;
+ }
+
+ try {
+ filter = metadataNode.getFullTextFilter(ctx.getTxnId(), dataverseName, filterName);
+ } catch (RemoteException e) {
+ throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
+ }
+ // We fetched the filter from the MetadataNode. Add it to the cache
+ // when this transaction commits.
+ if (filter != null) {
+ ctx.addFullTextFilter(filter);
+ }
+ return filter;
+ }
+
+ @Override
+ public void addFullTextConfig(MetadataTransactionContext mdTxnCtx,
+ FullTextConfigMetadataEntity configMetadataEntity) throws AlgebricksException {
+ if (Strings.isNullOrEmpty(configMetadataEntity.getFullTextConfig().getName())) {
- throw new AsterixException(ErrorCode.FULL_TEXT_CONFIG_ALREADY_EXISTS);
++ throw new MetadataException(ErrorCode.FULL_TEXT_CONFIG_ALREADY_EXISTS);
+ }
+
+ try {
+ metadataNode.addFullTextConfig(mdTxnCtx.getTxnId(), configMetadataEntity);
+ } catch (RemoteException e) {
+ throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
+ }
+ mdTxnCtx.addFullTextConfig(configMetadataEntity);
+ }
+
+ @Override
+ public FullTextConfigMetadataEntity getFullTextConfig(MetadataTransactionContext ctx, DataverseName dataverseName,
+ String configName) throws AlgebricksException {
+ // First look in the context to see if this transaction created the
+ // requested full-text config itself (but the full-text config is still uncommitted).
+ FullTextConfigMetadataEntity configMetadataEntity = ctx.getFullTextConfig(dataverseName, configName);
+ if (configMetadataEntity != null) {
+ // Don't add this config to the cache, since it is still
+ // uncommitted.
+ return configMetadataEntity;
+ }
+
+ if (ctx.fullTextConfigIsDropped(dataverseName, configName)) {
+ // config has been dropped by this transaction but could still be
+ // in the cache.
+ return null;
+ }
+
+ if (ctx.getDataverse(dataverseName) != null) {
+ // This transaction has dropped and subsequently created the same
+ // dataverse.
+ return null;
+ }
+
+ configMetadataEntity = cache.getFullTextConfig(dataverseName, configName);
+ if (configMetadataEntity != null) {
+ // config is already in the cache, don't add it again.
+ return configMetadataEntity;
+ }
+
+ try {
+ configMetadataEntity = metadataNode.getFullTextConfig(ctx.getTxnId(), dataverseName, configName);
+ } catch (RemoteException e) {
+ throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
+ }
+
+ // We fetched the config from the MetadataNode. Add it to the cache
+ // when this transaction commits.
+ if (configMetadataEntity != null) {
+ ctx.addFullTextConfig(configMetadataEntity);
+ }
+ return configMetadataEntity;
+ }
+
+ @Override
+ public void dropFullTextConfig(MetadataTransactionContext mdTxnCtx, DataverseName dataverseName, String configName)
+ throws AlgebricksException {
+ try {
+ metadataNode.dropFullTextConfig(mdTxnCtx.getTxnId(), dataverseName, configName);
+ } catch (RemoteException e) {
+ throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
+ }
+ mdTxnCtx.dropFullTextConfig(dataverseName, configName);
+ }
+
+ @Override
public void addFeedPolicy(MetadataTransactionContext mdTxnCtx, FeedPolicyEntity feedPolicy)
throws AlgebricksException {
try {
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 72b3e9a,088bbc6..37aae78
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@@ -33,7 -32,7 +34,8 @@@ import org.apache.asterix.common.api.IN
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.dataflow.LSMIndexUtil;
+ import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.MetadataException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
@@@ -1332,25 -1133,10 +1352,26 @@@ public class MetadataNode implements IM
for (Function function : functions) {
for (Triple<DataverseName, String, String> datasetDependency : function.getDependencies().get(2)) {
if (datasetDependency.first.equals(dataverseName) && datasetDependency.second.equals(dataTypeName)) {
- throw new AlgebricksException(
- "Cannot drop type " + TypeUtil.getFullyQualifiedDisplayName(dataverseName, dataTypeName)
- + " is being used by function " + function.getSignature());
+ throw new AsterixException(
+ org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "type",
+ TypeUtil.getFullyQualifiedDisplayName(dataverseName, dataTypeName), "function",
+ function.getSignature());
+ }
+ }
+ }
+ }
+
+ private void confirmFullTextFilterCanBeDeleted(TxnId txnId, DataverseName dataverseName, String fullTextFilterName)
+ throws AlgebricksException {
+ List<FullTextConfigMetadataEntity> configMetadataEntities = getDataverseFullTextConfigs(txnId, dataverseName);
+ for (FullTextConfigMetadataEntity configMetadataEntity : configMetadataEntities) {
+ FullTextConfigDescriptor config = configMetadataEntity.getFullTextConfig();
+ for (String filterName : config.getFilterNames()) {
+ if (filterName.equals(fullTextFilterName)) {
+ throw new AlgebricksException("Cannot drop full-text filter "
+ + TypeUtil.getFullyQualifiedDisplayName(dataverseName, fullTextFilterName)
+ + " being used by full-text config "
+ + TypeUtil.getFullyQualifiedDisplayName(dataverseName, config.getName()));
}
}
}
[asterixdb] 15/17: [NO ISSUE][COMP] Support batch assign for
external functions
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a4bf1451de142c30caa653db238d104894786d3a
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Fri Feb 12 12:09:36 2021 -0800
[NO ISSUE][COMP] Support batch assign for external functions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add compiler support for batch invocation of external functions
Change-Id: I1ed1f5c51628d996327de843f4977d083e9b4bd4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10006
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ian Maxon <im...@uci.edu>
---
.../operators/physical/AssignBatchPOperator.java | 103 ++++++++++
.../jobgen/QueryLogicalExpressionJobGen.java | 13 +-
.../asterix/optimizer/base/RuleCollections.java | 6 +-
.../ExtractBatchableExternalFunctionCallsRule.java | 220 +++++++++++++++++++++
.../rules/SetAsterixPhysicalOperatorsRule.java | 61 ++++++
.../apache/asterix/api/common/APIFramework.java | 4 +-
.../ExternalFunctionDescriptorProvider.java | 19 +-
.../library/ExternalScalarFunctionDescriptor.java | 14 +-
.../ExternalAssignBatchRuntimeFactory.java | 52 +++++
.../functions/ExternalFunctionCompilerUtil.java | 20 ++
.../om/functions/IExternalFunctionDescriptor.java | 29 +++
.../core/algebra/base/PhysicalOperatorTag.java | 2 +-
...POperator.java => AbstractAssignPOperator.java} | 36 ++--
.../operators/physical/AssignPOperator.java | 84 +-------
.../core/algebra/plan/PlanStabilityVerifier.java | 16 +-
.../algebra/util/OperatorManipulationUtil.java | 16 +-
.../rewriter/rules/ConsolidateAssignsRule.java | 9 +
.../rules/SetAlgebricksPhysicalOperatorsRule.java | 2 +-
18 files changed, 573 insertions(+), 133 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/AssignBatchPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/AssignBatchPOperator.java
new file mode 100644
index 0000000..0ec5ee7
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/AssignBatchPOperator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.external.library.ExternalFunctionDescriptorProvider;
+import org.apache.asterix.external.operators.ExternalAssignBatchRuntimeFactory;
+import org.apache.asterix.metadata.functions.ExternalFunctionCompilerUtil;
+import org.apache.asterix.om.functions.IExternalFunctionDescriptor;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import org.apache.hyracks.algebricks.core.algebra.operators.AbstractAssignPOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+public final class AssignBatchPOperator extends AbstractAssignPOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.ASSIGN_BATCH;
+ }
+
+ @Override
+ protected IPushRuntimeFactory createRuntimeFactory(JobGenContext context, AssignOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, int[] outColumns, int[] projectionList)
+ throws AlgebricksException {
+ IVariableTypeEnvironment inputTypeEnv = context.getTypeEnvironment(op.getInputs().get(0).getValue());
+ List<Mutable<ILogicalExpression>> exprList = op.getExpressions();
+ int exprCount = exprList.size();
+ IExternalFunctionDescriptor[] fnDescs = new IExternalFunctionDescriptor[exprCount];
+ int[][] fnArgColumns = new int[exprCount][];
+ for (int i = 0; i < exprCount; i++) {
+ Mutable<ILogicalExpression> exprRef = exprList.get(i);
+ ILogicalExpression expr = exprRef.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, op.getSourceLocation(),
+ String.valueOf(expr.getExpressionTag()));
+ }
+ AbstractFunctionCallExpression callExpr = (AbstractFunctionCallExpression) expr;
+ IFunctionInfo fi = callExpr.getFunctionInfo();
+ if (!ExternalFunctionCompilerUtil.supportsBatchInvocation(callExpr.getKind(), fi)) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, op.getSourceLocation(),
+ fi.toString());
+ }
+ fnDescs[i] = ExternalFunctionDescriptorProvider.resolveExternalFunction(callExpr, inputTypeEnv, context);
+ fnArgColumns[i] = getColumns(callExpr.getArguments(), opSchema, op.getSourceLocation());
+ }
+
+ return new ExternalAssignBatchRuntimeFactory(outColumns, fnDescs, fnArgColumns, projectionList);
+ }
+
+ private int[] getColumns(List<Mutable<ILogicalExpression>> exprList, IOperatorSchema opSchema,
+ SourceLocation sourceLoc) throws CompilationException {
+ int n = exprList.size();
+ int[] columns = new int[n];
+ for (int i = 0; i < n; i++) {
+ ILogicalExpression expr = exprList.get(i).getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(expr.getExpressionTag()));
+ }
+ VariableReferenceExpression argVarRef = (VariableReferenceExpression) expr;
+ LogicalVariable argVar = argVarRef.getVariableReference();
+ int argColumn = opSchema.findVariable(argVar);
+ if (argColumn < 0) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, String.valueOf(argVar));
+ }
+ columns[i] = argColumn;
+ }
+ return columns;
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
index 74656e4..2a662d0 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
@@ -30,7 +30,6 @@ import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionManager;
import org.apache.asterix.om.functions.IFunctionTypeInferer;
-import org.apache.asterix.runtime.functions.FunctionTypeInferers;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -138,15 +137,9 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
throws AlgebricksException {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
- IFunctionDescriptor fd = null;
- if (expr.getFunctionInfo() instanceof IExternalFunctionInfo) {
- fd = ExternalFunctionDescriptorProvider
- .getExternalFunctionDescriptor((IExternalFunctionInfo) expr.getFunctionInfo());
- CompilerProperties props = ((IApplicationContext) context.getAppContext()).getCompilerProperties();
- FunctionTypeInferers.SET_ARGUMENTS_TYPE.infer(expr, fd, env, props);
- } else {
- fd = resolveFunction(expr, env, context);
- }
+ IFunctionDescriptor fd = expr.getFunctionInfo() instanceof IExternalFunctionInfo
+ ? ExternalFunctionDescriptorProvider.resolveExternalFunction(expr, env, context)
+ : resolveFunction(expr, env, context);
return fd.createEvaluatorFactory(args);
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 4a2b629..d2fe2f5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -40,6 +40,7 @@ import org.apache.asterix.optimizer.rules.CheckInsertUpsertReturningRule;
import org.apache.asterix.optimizer.rules.ConstantFoldingRule;
import org.apache.asterix.optimizer.rules.CountVarToCountOneRule;
import org.apache.asterix.optimizer.rules.DisjunctivePredicateToJoinRule;
+import org.apache.asterix.optimizer.rules.ExtractBatchableExternalFunctionCallsRule;
import org.apache.asterix.optimizer.rules.ExtractDistinctByExpressionsRule;
import org.apache.asterix.optimizer.rules.ExtractOrderExpressionsRule;
import org.apache.asterix.optimizer.rules.ExtractWindowExpressionsRule;
@@ -284,7 +285,7 @@ public final class RuleCollections {
public static final List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() {
List<IAlgebraicRewriteRule> consolidation = new LinkedList<>();
consolidation.add(new ConsolidateSelectsRule());
- consolidation.add(new ConsolidateAssignsRule());
+ consolidation.add(new ConsolidateAssignsRule(false));
consolidation.add(new InlineAssignIntoAggregateRule());
consolidation.add(new RewriteDistinctAggregateRule());
// The following rule should run after RewriteDistinctAggregateRule
@@ -353,6 +354,7 @@ public final class RuleCollections {
public static final List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() {
List<IAlgebraicRewriteRule> physicalRewritesAllLevels = new LinkedList<>();
physicalRewritesAllLevels.add(new PullSelectOutOfEqJoin());
+ physicalRewritesAllLevels.add(new ExtractBatchableExternalFunctionCallsRule());
//Turned off the following rule for now not to change OptimizerTest results.
physicalRewritesAllLevels.add(new SetupCommitExtensionOpRule());
physicalRewritesAllLevels.add(new SetAsterixPhysicalOperatorsRule());
@@ -369,7 +371,7 @@ public final class RuleCollections {
physicalRewritesAllLevels.add(new IntroduceMaterializationForInsertWithSelfScanRule());
physicalRewritesAllLevels.add(new InlineSingleReferenceVariablesRule());
physicalRewritesAllLevels.add(new RemoveUnusedAssignAndAggregateRule());
- physicalRewritesAllLevels.add(new ConsolidateAssignsRule());
+ physicalRewritesAllLevels.add(new ConsolidateAssignsRule(true));
// After adding projects, we may need need to set physical operators again.
physicalRewritesAllLevels.add(new SetAsterixPhysicalOperatorsRule());
return physicalRewritesAllLevels;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
new file mode 100644
index 0000000..1e5d805
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractBatchableExternalFunctionCallsRule.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.metadata.functions.ExternalFunctionCompilerUtil;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+public class ExtractBatchableExternalFunctionCallsRule implements IAlgebraicRewriteRule {
+
+ private final ExtractFunctionCallsVisitor extractVisitor = new ExtractFunctionCallsVisitor();
+
+ private Boolean isRuleEnabled;
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ if (isRuleEnabled == null) {
+ isRuleEnabled = SetAsterixPhysicalOperatorsRule.isBatchAssignEnabled(context);
+ }
+ if (!isRuleEnabled) {
+ return false;
+ }
+
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ List<Mutable<ILogicalExpression>> assignTopExprRefs = Collections.emptyList();
+ switch (op.getOperatorTag()) {
+ case ASSIGN:
+ assignTopExprRefs = ((AssignOperator) op).getExpressions();
+ break;
+ case SELECT:
+ break;
+ default:
+ return false;
+ }
+
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ context.addToDontApplySet(this, op);
+
+ extractVisitor.reset(context, assignTopExprRefs);
+ if (!op.acceptExpressionTransform(extractVisitor)) {
+ return false;
+ }
+ SourceLocation sourceLoc = op.getSourceLocation();
+
+ ILogicalOperator inputOp = op.getInputs().get(0).getValue();
+ for (int i = 0, ln = extractVisitor.assignVars.size(); i < ln; i++) {
+ List<LogicalVariable> assignVarList = extractVisitor.assignVars.get(i);
+ List<Mutable<ILogicalExpression>> assignExprList = extractVisitor.assignExprs.get(i);
+ AssignOperator assignOp = new AssignOperator(assignVarList, assignExprList);
+ assignOp.setSourceLocation(sourceLoc);
+ assignOp.getInputs().add(new MutableObject<>(inputOp));
+ context.computeAndSetTypeEnvironmentForOperator(assignOp);
+ assignOp.recomputeSchema();
+ OperatorPropertiesUtil.markMovable(assignOp, false);
+
+ context.addToDontApplySet(this, assignOp);
+ for (LogicalVariable assignVar : assignVarList) {
+ context.addNotToBeInlinedVar(assignVar);
+ }
+
+ inputOp = assignOp;
+ }
+
+ op.getInputs().clear();
+ op.getInputs().add(new MutableObject<>(inputOp));
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ op.recomputeSchema();
+ return true;
+ }
+
+ private static final class ExtractFunctionCallsVisitor implements ILogicalExpressionReferenceTransform {
+
+ private final List<List<LogicalVariable>> assignVars = new ArrayList<>();
+
+ private final List<List<Mutable<ILogicalExpression>>> assignExprs = new ArrayList<>();
+
+ private final List<LogicalVariable> usedVarList = new ArrayList<>();
+
+ private IOptimizationContext context;
+
+ private List<Mutable<ILogicalExpression>> dontExtractFromExprRefs;
+
+ public void reset(IOptimizationContext context, List<Mutable<ILogicalExpression>> dontExtractFromExprRefs) {
+ this.context = context;
+ this.dontExtractFromExprRefs = dontExtractFromExprRefs;
+ assignVars.clear();
+ assignExprs.clear();
+ }
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+ ILogicalExpression expr = exprRef.getValue();
+ switch (expr.getExpressionTag()) {
+ case FUNCTION_CALL:
+ AbstractFunctionCallExpression callExpr = (AbstractFunctionCallExpression) expr;
+ boolean applied = false;
+ for (Mutable<ILogicalExpression> argRef : callExpr.getArguments()) {
+ applied |= transform(argRef);
+ }
+ AbstractFunctionCallExpression.FunctionKind fnKind = callExpr.getKind();
+ IFunctionInfo fnInfo = callExpr.getFunctionInfo();
+ if (ExternalFunctionCompilerUtil.supportsBatchInvocation(fnKind, fnInfo)
+ && callExpr.isFunctional()) {
+ // need to extract non-variable arguments into separate ASSIGNS
+ // because batched assign can only operate on columns
+ for (Mutable<ILogicalExpression> argRef : callExpr.getArguments()) {
+ ILogicalExpression argExpr = argRef.getValue();
+ if (argExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ LogicalVariable newArgVar = context.newVar();
+ VariableReferenceExpression newArgVarRef = new VariableReferenceExpression(newArgVar);
+ newArgVarRef.setSourceLocation(expr.getSourceLocation());
+ saveAssignVar(newArgVar, argExpr);
+ argRef.setValue(newArgVarRef);
+ applied = true;
+ }
+ }
+ // need extract function call itself into a separate ASSIGN
+ // (unless it's already a top level expression of the ASSIGN operator we're visiting)
+ boolean dontExtractExprRef = indexOf(dontExtractFromExprRefs, exprRef) >= 0;
+ if (!dontExtractExprRef) {
+ LogicalVariable newVar = context.newVar();
+ VariableReferenceExpression newVarRef = new VariableReferenceExpression(newVar);
+ newVarRef.setSourceLocation(expr.getSourceLocation());
+ saveAssignVar(newVar, expr);
+ exprRef.setValue(newVarRef);
+ applied = true;
+ }
+ }
+ return applied;
+ case VARIABLE:
+ case CONSTANT:
+ return false;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, expr.getSourceLocation(),
+ expr.getExpressionTag().toString());
+ }
+ }
+
+ private void saveAssignVar(LogicalVariable var, ILogicalExpression expr) {
+ List<LogicalVariable> assignVarList = null;
+ List<Mutable<ILogicalExpression>> assignExprList = null;
+
+ if (!assignVars.isEmpty()) {
+ usedVarList.clear();
+ expr.getUsedVariables(usedVarList);
+ for (int i = 0, ln = assignVars.size(); i < ln; i++) {
+ List<LogicalVariable> candidateVarList = assignVars.get(i);
+ if (OperatorPropertiesUtil.disjoint(candidateVarList, usedVarList)) {
+ assignVarList = candidateVarList;
+ assignExprList = assignExprs.get(i);
+ break;
+ }
+ }
+ }
+
+ if (assignVarList == null) {
+ // first time, or couldn't find a disjoint var list
+ assignVarList = new ArrayList<>();
+ assignExprList = new ArrayList<>();
+ assignVars.add(assignVarList);
+ assignExprs.add(assignExprList);
+ }
+
+ assignVarList.add(var);
+ assignExprList.add(new MutableObject<>(expr));
+ }
+
+ public static int indexOf(List<Mutable<ILogicalExpression>> exprList, Mutable<ILogicalExpression> exprRef) {
+ return OperatorManipulationUtil.indexOf(exprList,
+ (listItemExprRef, paramExprRef) -> listItemExprRef == paramExprRef, exprRef);
+ }
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
index e662737..ea51fc3 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
@@ -21,6 +21,7 @@ package org.apache.asterix.optimizer.rules;
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.algebra.operators.physical.AssignBatchPOperator;
import org.apache.asterix.algebra.operators.physical.BTreeSearchPOperator;
import org.apache.asterix.algebra.operators.physical.InvertedIndexPOperator;
import org.apache.asterix.algebra.operators.physical.RTreeSearchPOperator;
@@ -30,6 +31,7 @@ import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.functions.ExternalFunctionCompilerUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.optimizer.base.AnalysisUtil;
import org.apache.asterix.optimizer.rules.am.AccessMethodJobGenParams;
@@ -49,10 +51,12 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
@@ -69,16 +73,47 @@ import org.apache.hyracks.algebricks.rewriter.rules.SetAlgebricksPhysicalOperato
public final class SetAsterixPhysicalOperatorsRule extends SetAlgebricksPhysicalOperatorsRule {
+ // Disable ASSIGN_BATCH physical operator if this option is set to 'false'
+ public static final String REWRITE_ATTEMPT_BATCH_ASSIGN = "rewrite_attempt_batch_assign";
+ static final boolean REWRITE_ATTEMPT_BATCH_ASSIGN_DEFAULT = false;
+
@Override
protected ILogicalOperatorVisitor<IPhysicalOperator, Boolean> createPhysicalOperatorFactoryVisitor(
IOptimizationContext context) {
return new AsterixPhysicalOperatorFactoryVisitor(context);
}
+ static boolean isBatchAssignEnabled(IOptimizationContext context) {
+ MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+ return metadataProvider.getBooleanProperty(REWRITE_ATTEMPT_BATCH_ASSIGN, REWRITE_ATTEMPT_BATCH_ASSIGN_DEFAULT);
+ }
+
private static class AsterixPhysicalOperatorFactoryVisitor extends AlgebricksPhysicalOperatorFactoryVisitor {
+ private final boolean isBatchAssignEnabled;
+
private AsterixPhysicalOperatorFactoryVisitor(IOptimizationContext context) {
super(context);
+ isBatchAssignEnabled = isBatchAssignEnabled(context);
+ }
+
+ @Override
+ public IPhysicalOperator visitAssignOperator(AssignOperator op, Boolean topLevelOp) throws AlgebricksException {
+ List<Mutable<ILogicalExpression>> exprList = op.getExpressions();
+ boolean batchMode = isBatchAssignEnabled && exprList.size() > 0 && allBatchableFunctionCalls(exprList);
+ if (batchMode) {
+ // disable inlining of variable arguments
+ for (Mutable<ILogicalExpression> exprRef : exprList) {
+ AbstractFunctionCallExpression callExpr = (AbstractFunctionCallExpression) exprRef.getValue();
+ for (Mutable<ILogicalExpression> argRef : callExpr.getArguments()) {
+ LogicalVariable var = ((VariableReferenceExpression) argRef.getValue()).getVariableReference();
+ context.addNotToBeInlinedVar(var);
+ }
+ }
+ return new AssignBatchPOperator();
+ } else {
+ return super.visitAssignOperator(op, topLevelOp);
+ }
}
@Override
@@ -280,5 +315,31 @@ public final class SetAsterixPhysicalOperatorsRule extends SetAlgebricksPhysical
return new WindowStreamPOperator(winOp.getPartitionVarList(), winOp.getOrderColumnList());
}
}
+
+ private boolean allBatchableFunctionCalls(List<Mutable<ILogicalExpression>> exprList)
+ throws CompilationException {
+ for (Mutable<ILogicalExpression> exprRef : exprList) {
+ if (!isBatchableFunctionCall(exprRef.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean isBatchableFunctionCall(ILogicalExpression expr) throws CompilationException {
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression callExpr = (AbstractFunctionCallExpression) expr;
+ if (!ExternalFunctionCompilerUtil.supportsBatchInvocation(callExpr.getKind(), callExpr.getFunctionInfo())) {
+ return false;
+ }
+ for (Mutable<ILogicalExpression> argRef : callExpr.getArguments()) {
+ if (argRef.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+ }
+ return true;
+ }
}
}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 9088db6..d60047e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -72,6 +72,7 @@ import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.optimizer.base.AsterixOptimizationContext;
import org.apache.asterix.optimizer.base.FuzzyUtils;
import org.apache.asterix.optimizer.rules.DisjunctivePredicateToJoinRule;
+import org.apache.asterix.optimizer.rules.SetAsterixPhysicalOperatorsRule;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.ExecutionPlans;
@@ -141,7 +142,8 @@ public class APIFramework {
StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME,
FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION,
SqlppExpressionToPlanTranslator.REWRITE_IN_AS_OR_OPTION, "hash_merge", "output-record-type",
- DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION);
+ DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
+ SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN);
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
index 8ac507f..dc51c5f 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
@@ -18,15 +18,30 @@
*/
package org.apache.asterix.external.library;
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.om.functions.IExternalFunctionDescriptor;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
public class ExternalFunctionDescriptorProvider {
- public static IFunctionDescriptor getExternalFunctionDescriptor(IExternalFunctionInfo finfo)
+ public static IExternalFunctionDescriptor resolveExternalFunction(AbstractFunctionCallExpression expr,
+ IVariableTypeEnvironment inputTypeEnv, JobGenContext context) throws AlgebricksException {
+ IExternalFunctionDescriptor fd = getExternalFunctionDescriptor((IExternalFunctionInfo) expr.getFunctionInfo());
+ CompilerProperties props = ((IApplicationContext) context.getAppContext()).getCompilerProperties();
+ FunctionTypeInferers.SET_ARGUMENTS_TYPE.infer(expr, fd, inputTypeEnv, props);
+ fd.setSourceLocation(expr.getSourceLocation());
+ return fd;
+ }
+
+ private static IExternalFunctionDescriptor getExternalFunctionDescriptor(IExternalFunctionInfo finfo)
throws AlgebricksException {
switch (finfo.getKind()) {
case SCALAR:
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionDescriptor.java
index 61ab3ea..63f0a13 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionDescriptor.java
@@ -19,15 +19,15 @@
package org.apache.asterix.external.library;
+import org.apache.asterix.om.functions.IExternalFunctionDescriptor;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
public class ExternalScalarFunctionDescriptor extends AbstractScalarFunctionDynamicDescriptor
- implements IFunctionDescriptor {
+ implements IExternalFunctionDescriptor {
private static final long serialVersionUID = 2L;
private final IExternalFunctionInfo finfo;
@@ -54,4 +54,14 @@ public class ExternalScalarFunctionDescriptor extends AbstractScalarFunctionDyna
public FunctionIdentifier getIdentifier() {
return finfo.getFunctionIdentifier();
}
+
+ @Override
+ public IExternalFunctionInfo getFunctionInfo() {
+ return finfo;
+ }
+
+ @Override
+ public IAType[] getArgumentTypes() {
+ return argTypes;
+ }
}
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
new file mode 100644
index 0000000..44a17a9
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalAssignBatchRuntimeFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.operators;
+
+import org.apache.asterix.om.functions.IExternalFunctionDescriptor;
+import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
+import org.apache.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public final class ExternalAssignBatchRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private int[] outColumns;
+ private final IExternalFunctionDescriptor[] fnDescs;
+ private final int[][] fnArgColumns;
+
+ public ExternalAssignBatchRuntimeFactory(int[] outColumns, IExternalFunctionDescriptor[] fnDescs,
+ int[][] fnArgColumns, int[] projectionList) {
+ super(projectionList);
+ this.outColumns = outColumns;
+ this.fnDescs = fnDescs;
+ this.fnArgColumns = fnArgColumns;
+ }
+
+ @Override
+ public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(IHyracksTaskContext ctx)
+ throws HyracksDataException {
+ throw new HyracksDataException(ErrorCode.OPERATOR_NOT_IMPLEMENTED, sourceLoc,
+ PhysicalOperatorTag.ASSIGN_BATCH.toString());
+ }
+}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 029b13a..f900c92 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -29,6 +29,7 @@ import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.BuiltinTypeMap;
import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
@@ -167,4 +168,23 @@ public class ExternalFunctionCompilerUtil {
String.valueOf(actualSize), language.name());
}
}
+
+ public static boolean supportsBatchInvocation(FunctionKind fnKind, IFunctionInfo fnInfo)
+ throws CompilationException {
+ if (fnKind != FunctionKind.SCALAR) {
+ return false;
+ }
+ if (!(fnInfo instanceof IExternalFunctionInfo)) {
+ return false;
+ }
+ ExternalFunctionLanguage language = ((IExternalFunctionInfo) fnInfo).getLanguage();
+ switch (language) {
+ case JAVA:
+ return false;
+ case PYTHON:
+ return false;
+ default:
+ throw new CompilationException(ErrorCode.METADATA_ERROR, language.name());
+ }
+ }
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionDescriptor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionDescriptor.java
new file mode 100644
index 0000000..8d362ce
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionDescriptor.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.functions;
+
+import org.apache.asterix.om.types.IAType;
+
+public interface IExternalFunctionDescriptor extends IFunctionDescriptor {
+
+ IExternalFunctionInfo getFunctionInfo();
+
+ IAType[] getArgumentTypes();
+}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
index b7f6d62..d590f71 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.algebricks.core.algebra.base;
public enum PhysicalOperatorTag {
AGGREGATE,
ASSIGN,
+ ASSIGN_BATCH,
BROADCAST_EXCHANGE,
BTREE_SEARCH,
BULKLOAD,
@@ -33,7 +34,6 @@ public enum PhysicalOperatorTag {
FORWARD,
HASH_PARTITION_EXCHANGE,
HASH_PARTITION_MERGE_EXCHANGE,
- HDFS_READER,
HYBRID_HASH_JOIN,
IN_MEMORY_HASH_JOIN,
MICRO_STABLE_SORT,
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/AbstractAssignPOperator.java
similarity index 75%
copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
copy to hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/AbstractAssignPOperator.java
index 23fa1ee..4466a25 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/AbstractAssignPOperator.java
@@ -16,39 +16,32 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.algebricks.core.algebra.operators.physical;
+
+package org.apache.hyracks.algebricks.core.algebra.operators;
import java.util.List;
-import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import org.apache.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-public class AssignPOperator extends AbstractPhysicalOperator {
+public abstract class AbstractAssignPOperator extends AbstractPhysicalOperator {
- private boolean flushFramesRapidly;
- private String[] locations;
+ protected boolean flushFramesRapidly;
- @Override
- public PhysicalOperatorTag getOperatorTag() {
- return PhysicalOperatorTag.ASSIGN;
- }
+ protected String[] locations;
@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
@@ -72,23 +65,16 @@ public class AssignPOperator extends AbstractPhysicalOperator {
throws AlgebricksException {
AssignOperator assign = (AssignOperator) op;
List<LogicalVariable> variables = assign.getVariables();
- List<Mutable<ILogicalExpression>> expressions = assign.getExpressions();
int[] outColumns = new int[variables.size()];
for (int i = 0; i < outColumns.length; i++) {
outColumns[i] = opSchema.findVariable(variables.get(i));
}
- IScalarEvaluatorFactory[] evalFactories = new IScalarEvaluatorFactory[expressions.size()];
- IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
- for (int i = 0; i < evalFactories.length; i++) {
- evalFactories[i] = expressionRuntimeProvider.createEvaluatorFactory(expressions.get(i).getValue(),
- context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
- }
// TODO push projections into the operator
int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
- AssignRuntimeFactory runtime =
- new AssignRuntimeFactory(outColumns, evalFactories, projectionList, flushFramesRapidly);
+ IPushRuntimeFactory runtime =
+ createRuntimeFactory(context, assign, opSchema, inputSchemas, outColumns, projectionList);
runtime.setSourceLocation(assign.getSourceLocation());
// contribute one Asterix framewriter
@@ -105,6 +91,10 @@ public class AssignPOperator extends AbstractPhysicalOperator {
builder.contributeGraphEdge(src, 0, assign, 0);
}
+ protected abstract IPushRuntimeFactory createRuntimeFactory(JobGenContext context, AssignOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, int[] outColumns, int[] projectionList)
+ throws AlgebricksException;
+
@Override
public boolean isMicroOperator() {
return true;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
index 23fa1ee..5f927fd 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
@@ -21,105 +21,35 @@ package org.apache.hyracks.algebricks.core.algebra.operators.physical;
import java.util.List;
import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import org.apache.hyracks.algebricks.core.algebra.operators.AbstractAssignPOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import org.apache.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-public class AssignPOperator extends AbstractPhysicalOperator {
-
- private boolean flushFramesRapidly;
- private String[] locations;
+public class AssignPOperator extends AbstractAssignPOperator {
@Override
public PhysicalOperatorTag getOperatorTag() {
return PhysicalOperatorTag.ASSIGN;
}
- @Override
- public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
- AssignOperator assignOp = (AssignOperator) op;
- ILogicalOperator op2 = op.getInputs().get(0).getValue();
- deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
- if (assignOp.getExplicitOrderingProperty() != null) {
- deliveredProperties.getLocalProperties().add(assignOp.getExplicitOrderingProperty());
- }
- }
-
- @Override
- public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
- IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
- return emptyUnaryRequirements();
- }
-
- @Override
- public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
- IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ protected IPushRuntimeFactory createRuntimeFactory(JobGenContext context, AssignOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, int[] outColumns, int[] projectionList)
throws AlgebricksException {
- AssignOperator assign = (AssignOperator) op;
- List<LogicalVariable> variables = assign.getVariables();
- List<Mutable<ILogicalExpression>> expressions = assign.getExpressions();
- int[] outColumns = new int[variables.size()];
- for (int i = 0; i < outColumns.length; i++) {
- outColumns[i] = opSchema.findVariable(variables.get(i));
- }
+ List<Mutable<ILogicalExpression>> expressions = op.getExpressions();
IScalarEvaluatorFactory[] evalFactories = new IScalarEvaluatorFactory[expressions.size()];
IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
for (int i = 0; i < evalFactories.length; i++) {
evalFactories[i] = expressionRuntimeProvider.createEvaluatorFactory(expressions.get(i).getValue(),
context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
}
-
- // TODO push projections into the operator
- int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
-
- AssignRuntimeFactory runtime =
- new AssignRuntimeFactory(outColumns, evalFactories, projectionList, flushFramesRapidly);
- runtime.setSourceLocation(assign.getSourceLocation());
-
- // contribute one Asterix framewriter
- RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
- if (locations != null && locations.length > 0) {
- AlgebricksAbsolutePartitionConstraint locationConstraint =
- new AlgebricksAbsolutePartitionConstraint(locations);
- builder.contributeMicroOperator(assign, runtime, recDesc, locationConstraint);
- } else {
- builder.contributeMicroOperator(assign, runtime, recDesc);
- }
- // and contribute one edge from its child
- ILogicalOperator src = assign.getInputs().get(0).getValue();
- builder.contributeGraphEdge(src, 0, assign, 0);
- }
-
- @Override
- public boolean isMicroOperator() {
- return true;
- }
-
- public void setRapidFrameFlush(boolean flushFramesRapidly) {
- this.flushFramesRapidly = flushFramesRapidly;
- }
-
- public void setLocationConstraint(String[] locations) {
- this.locations = locations;
- }
-
- @Override
- public boolean expensiveThanMaterialization() {
- return false;
+ return new AssignRuntimeFactory(outColumns, evalFactories, projectionList, flushFramesRapidly);
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/PlanStabilityVerifier.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/PlanStabilityVerifier.java
index 3f7a0c1..5e10819 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/PlanStabilityVerifier.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/PlanStabilityVerifier.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.function.BiFunction;
-import java.util.function.BiPredicate;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -34,6 +33,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import org.apache.hyracks.algebricks.core.algebra.prettyprint.IPlanPrettyPrinter;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
@@ -250,21 +250,11 @@ public final class PlanStabilityVerifier {
}
private static <T> int findItem(List<T> list, T item) {
- return indexOf(list, (listItem, paramItem) -> listItem == paramItem, item);
+ return OperatorManipulationUtil.indexOf(list, (listItem, paramItem) -> listItem == paramItem, item);
}
private static <T> int findNonNull(List<T> list) {
- return indexOf(list, (listItem, none) -> listItem != null, null);
- }
-
- private static <T, U> int indexOf(List<T> list, BiPredicate<T, U> predicate, U predicateParam) {
- for (int i = 0, n = list.size(); i < n; i++) {
- T listItem = list.get(i);
- if (predicate.test(listItem, predicateParam)) {
- return i;
- }
- }
- return -1;
+ return OperatorManipulationUtil.indexOf(list, (listItem, none) -> listItem != null, null);
}
static String printOperator(Mutable<ILogicalOperator> opRef, IPlanPrettyPrinter printer) {
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
index dd109ff..aa2ecdc 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -27,6 +27,7 @@ import java.util.Deque;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.BiPredicate;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
@@ -385,8 +386,21 @@ public class OperatorManipulationUtil {
* @return operator position in the given list or {@code -1} if not found
*/
public static int indexOf(List<Mutable<ILogicalOperator>> list, ILogicalOperator op) {
+ return indexOf(list, (listItemOpRef, paramOp) -> listItemOpRef.getValue() == paramOp, op);
+ }
+
+ /**
+ * Find an item a given list
+ *
+ * @param list list to search in
+ * @param predicate predicate to test
+ * @param predicateParam parameter to pass to the predicate
+ * @return item position in the given list or {@code -1} if not found
+ */
+ public static <T, U> int indexOf(List<T> list, BiPredicate<T, U> predicate, U predicateParam) {
for (int i = 0, ln = list.size(); i < ln; i++) {
- if (list.get(i).getValue() == op) {
+ T listItem = list.get(i);
+ if (predicate.test(listItem, predicateParam)) {
return i;
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
index bbb6cbd..bf9ab6a 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
@@ -35,6 +35,12 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class ConsolidateAssignsRule implements IAlgebraicRewriteRule {
+ private final boolean recomputeSchema;
+
+ public ConsolidateAssignsRule(boolean recomputeSchema) {
+ this.recomputeSchema = recomputeSchema;
+ }
+
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
return false;
@@ -76,6 +82,9 @@ public class ConsolidateAssignsRule implements IAlgebraicRewriteRule {
asgnInpList.clear();
asgnInpList.add(botOpRef);
context.computeAndSetTypeEnvironmentForOperator(assign1);
+ if (recomputeSchema) {
+ assign1.recomputeSchema();
+ }
return true;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
index 7243827..b95971f 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -190,7 +190,7 @@ public class SetAlgebricksPhysicalOperatorsRule implements IAlgebraicRewriteRule
}
@Override
- public IPhysicalOperator visitAssignOperator(AssignOperator op, Boolean topLevelOp) {
+ public IPhysicalOperator visitAssignOperator(AssignOperator op, Boolean topLevelOp) throws AlgebricksException {
return new AssignPOperator();
}
[asterixdb] 10/17: [NO ISSUE][HYR] Exception map utility method,
fix getBootClassPath() on java >= 9
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 038a627bf799c6d02273e581559031e66429d24c
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Thu Mar 4 09:17:20 2021 -0500
[NO ISSUE][HYR] Exception map utility method, fix getBootClassPath() on java >= 9
Change-Id: Ic4cb7a6c3c3636578ba02584cebdba55e6da9995
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10363
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
hyracks-fullstack/hyracks/hyracks-api/pom.xml | 4 ++++
.../apache/hyracks/api/util/ExceptionUtils.java | 25 ++++++++++++++++++++++
.../java/org/apache/hyracks/util/MXHelper.java | 3 +--
3 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/pom.xml b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
index 90fb5ed..2c50ee4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-api/pom.xml
@@ -94,5 +94,9 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java
index 9067dce..d748ed8 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java
@@ -21,9 +21,14 @@ package org.apache.hyracks.api.util;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.util.ThrowingFunction;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
/**
* @author yingyib
@@ -133,4 +138,24 @@ public class ExceptionUtils {
public static boolean causedByInterrupt(Throwable th) {
return getRootCause(th) instanceof InterruptedException;
}
+
+ /**
+ * Convenience utility method to provide a form of {@link Map#computeIfAbsent(Object, Function)} which allows
+ * throwable mapping functions. Any exceptions thrown by the mapping function is propagated as an instance of
+ * {@link HyracksDataException}
+ */
+ public static <K, V> V computeIfAbsent(Map<K, V> map, K key, ThrowingFunction<K, V> function)
+ throws HyracksDataException {
+ try {
+ return map.computeIfAbsent(key, k -> {
+ try {
+ return function.process(k);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e);
+ }
+ });
+ } catch (UncheckedExecutionException e) {
+ throw HyracksDataException.create(e.getCause());
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
index 247c001..c238b50 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
@@ -108,9 +108,8 @@ public class MXHelper {
public static String getBootClassPath() {
try {
- return runtimeMXBean.getBootClassPath();
+ return runtimeMXBean.isBootClassPathSupported() ? runtimeMXBean.getBootClassPath() : null;
} catch (UnsupportedOperationException e) {
- // boot classpath is not supported in Java 9 and later
LOGGER.debug("ignoring exception calling RuntimeMXBean.getBootClassPath; returning null", e);
return null;
}