You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/06/23 18:53:08 UTC
[asterixdb] branch master updated (60b27fc -> 6a8dafa)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 60b27fc Merge branch 'gerrit/mad-hatter'
new 0df6cfb [NO ISSUE][HYR] +=ThrowingIntConsumer, unchecked wrapper helpers
new b0acb8b [NO ISSUE][MISC] IRetryPolicy requires non-null throwable
new 0eeeed1 [ASTERIXDB-2750][COMP] Incorrect result with unnest and join
new 6c239e7 [NO ISSUE][COMP] Improve error reporting in TypeTranslator
new 6a8dafa Merge branch 'gerrit/mad-hatter'
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../SqlppExpressionToPlanTranslator.java | 24 ++++++++----------
.../apache/asterix/translator/TypeTranslator.java | 20 +++++++++------
.../apache/asterix/app/active/RecoveryTask.java | 6 ++---
.../asterix/app/translator/QueryTranslator.java | 14 ++++++-----
.../create-dataset-inline-type-2.5.ddl.sqlpp} | 10 +++++---
.../ASTERIXDB-2750_unnest_join.1.query.sqlpp} | 8 +++---
.../ASTERIXDB-2750_unnest_join.2.query.sqlpp} | 14 +++++------
.../ASTERIXDB-2750_unnest_join.1.adm | 1 +
.../ASTERIXDB-2750_unnest_join.2.adm} | 0
.../test/resources/runtimets/testsuite_sqlpp.xml | 6 +++++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 6 ++---
hyracks-fullstack/hyracks/hyracks-util/pom.xml | 4 +++
.../java/org/apache/hyracks/util/IRetryPolicy.java | 2 +-
.../org/apache/hyracks/util/ThrowingConsumer.java | 19 ++++++++++++++
.../apache/hyracks/util/ThrowingIntConsumer.java | 29 +++++++++++-----------
15 files changed, 99 insertions(+), 64 deletions(-)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/{create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp => create-dataset-inline-type-2/create-dataset-inline-type-2.5.ddl.sqlpp} (81%)
copy asterixdb/asterix-app/src/test/resources/{metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.3.query.aql => runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.query.sqlpp} (81%)
copy asterixdb/asterix-app/src/test/{java/org/apache/asterix/test/common/ComparisonException.java => resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.query.sqlpp} (76%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.adm
copy asterixdb/asterix-app/src/test/resources/runtimets/results/{fuzzyjoin/basic-1_3_1/basic-1_3_1.6.adm => unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.adm} (100%)
copy asterixdb/asterix-active/src/main/java/org/apache/asterix/active/InfiniteRetryPolicy.java => hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIntConsumer.java (62%)
[asterixdb] 03/05: [ASTERIXDB-2750][COMP] Incorrect result with
unnest and join
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0eeeed149baacba6751c422ea0e4324b268f5cf4
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Mon Jun 15 18:45:14 2020 -0700
[ASTERIXDB-2750][COMP] Incorrect result with unnest and join
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix incorrect result when FROM clause contains
simple unnest (",") followed by JOIN
Change-Id: Icfd174ad099aaf24f9553349b3d417d64f9c70b5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6824
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../SqlppExpressionToPlanTranslator.java | 24 ++++++++++-----------
.../ASTERIXDB-2750_unnest_join.1.query.sqlpp | 24 +++++++++++++++++++++
.../ASTERIXDB-2750_unnest_join.2.query.sqlpp | 25 ++++++++++++++++++++++
.../ASTERIXDB-2750_unnest_join.1.adm | 1 +
.../ASTERIXDB-2750_unnest_join.2.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +++++
6 files changed, 67 insertions(+), 13 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
index 82dc344..5363ae2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -149,7 +149,7 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
public static final String REWRITE_IN_AS_OR_OPTION = "rewrite_in_as_or";
private static final boolean REWRITE_IN_AS_OR_OPTION_DEFAULT = true;
- private Deque<Mutable<ILogicalOperator>> uncorrelatedLeftBranchStack = new ArrayDeque<>();
+ private Deque<Mutable<ILogicalOperator>> uncorrelatedRightBranchStack = new ArrayDeque<>();
private final Map<VarIdentifier, IAObject> externalVars;
private final boolean translateInAsOr;
@@ -297,10 +297,12 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
throws CompilationException {
Mutable<ILogicalOperator> inputSrc = arg;
Pair<ILogicalOperator, LogicalVariable> topUnnest = null;
+ uncorrelatedRightBranchStack.push(inputSrc);
for (FromTerm fromTerm : fromClause.getFromTerms()) {
topUnnest = fromTerm.accept(this, inputSrc);
inputSrc = new MutableObject<>(topUnnest.first);
}
+ uncorrelatedRightBranchStack.pop();
return topUnnest;
}
@@ -328,27 +330,20 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
Mutable<ILogicalOperator> topOpRef = new MutableObject<>(unnestOp);
if (fromTerm.hasCorrelateClauses()) {
for (AbstractBinaryCorrelateClause correlateClause : fromTerm.getCorrelateClauses()) {
- if (correlateClause.getClauseType() == ClauseType.UNNEST_CLAUSE) {
- // Correlation is allowed.
- topOpRef = new MutableObject<>(correlateClause.accept(this, topOpRef).first);
- } else {
- // Correlation is dis-allowed.
- uncorrelatedLeftBranchStack.push(topOpRef);
- topOpRef = new MutableObject<>(correlateClause.accept(this, tupSource).first);
- }
+ topOpRef = new MutableObject<>(correlateClause.accept(this, topOpRef).first);
}
}
return new Pair<>(topOpRef.getValue(), fromVar);
}
@Override
- public Pair<ILogicalOperator, LogicalVariable> visit(JoinClause joinClause, Mutable<ILogicalOperator> inputRef)
+ public Pair<ILogicalOperator, LogicalVariable> visit(JoinClause joinClause, Mutable<ILogicalOperator> leftInputRef)
throws CompilationException {
SourceLocation sourceLoc = joinClause.getSourceLocation();
- Mutable<ILogicalOperator> leftInputRef = uncorrelatedLeftBranchStack.pop();
if (joinClause.getJoinType() == JoinType.INNER) {
+ Mutable<ILogicalOperator> rightInputRef = uncorrelatedRightBranchStack.peek();
Pair<ILogicalOperator, LogicalVariable> rightBranch =
- generateUnnestForBinaryCorrelateRightBranch(joinClause, inputRef, true);
+ generateUnnestForBinaryCorrelateRightBranch(joinClause, rightInputRef, true);
// A join operator with condition TRUE.
AbstractBinaryJoinOperator joinOperator = new InnerJoinOperator(
new MutableObject<>(ConstantExpression.TRUE), leftInputRef, new MutableObject<>(rightBranch.first));
@@ -362,7 +357,7 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
filter.getInputs().add(conditionExprOpPair.second);
filter.setSourceLocation(conditionExprOpPair.first.getSourceLocation());
return new Pair<>(filter, rightBranch.second);
- } else {
+ } else if (joinClause.getJoinType() == JoinType.LEFTOUTER) {
// Creates a subplan operator.
SubplanOperator subplanOp = new SubplanOperator();
subplanOp.getInputs().add(leftInputRef);
@@ -498,6 +493,9 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
context.setVar(joinClause.getRightVariable(), outerUnnestVar);
}
return new Pair<>(currentTopOp, null);
+ } else {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, joinClause.getSourceLocation(),
+ String.valueOf(joinClause.getJoinType().toString()));
}
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.query.sqlpp
new file mode 100644
index 0000000..c608831
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+with
+ t1 as [ { "x": 1, "y": [10, 11, 12] }, { "x": 2, "y": [20, 21, 22] } ],
+ t2 as [ 100, 101, 102, 103 ]
+from t1 as t1, t1.y as y join t2 as t2 on true
+select value count(*)
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.query.sqlpp
new file mode 100644
index 0000000..0909785
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+with
+ t1 as [ { "x": 1, "y": [10, 11, 12] }, { "x": 2, "y": [20, 21, 22] } ],
+ t2 as [ 100, 101, 102, 103 ],
+ t3 as [ 1000, 1001, 1002, 1003, 1004 ]
+from t1 as t1, t1.y as y join t2 as t2 on true join t3 as t3 on true
+select value count(*)
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.adm
new file mode 100644
index 0000000..cabf43b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.1.adm
@@ -0,0 +1 @@
+24
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.adm
new file mode 100644
index 0000000..8bc6583
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2750_unnest_join/ASTERIXDB-2750_unnest_join.2.adm
@@ -0,0 +1 @@
+120
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 06ccf56..44e2bff 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -12366,6 +12366,11 @@
</test-group>
<test-group name="unnest">
<test-case FilePath="unnest">
+ <compilation-unit name="ASTERIXDB-2750_unnest_join">
+ <output-dir compare="Text">ASTERIXDB-2750_unnest_join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="unnest">
<compilation-unit name="left-outer-unnest">
<output-dir compare="Text">left-outer-unnest</output-dir>
</compilation-unit>
[asterixdb] 01/05: [NO ISSUE][HYR] +=ThrowingIntConsumer,
unchecked wrapper helpers
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0df6cfb5e1e243c7a380243c1523cf4420caeef2
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Jun 20 16:54:11 2020 -0400
[NO ISSUE][HYR] +=ThrowingIntConsumer, unchecked wrapper helpers
Change-Id: I43915b87f5a10dd024f5618c9e8da26c5ad5c496
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6903
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
hyracks-fullstack/hyracks/hyracks-util/pom.xml | 4 ++++
.../org/apache/hyracks/util/ThrowingConsumer.java | 19 ++++++++++++++++++
...owingConsumer.java => ThrowingIntConsumer.java} | 23 ++++++++++++++++++++--
3 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-util/pom.xml b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
index e2da20a..621e6f6 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-util/pom.xml
@@ -95,6 +95,10 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
index bfc5fdd..be9874a 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
@@ -18,7 +18,26 @@
*/
package org.apache.hyracks.util;
+import java.util.function.Consumer;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
@FunctionalInterface
public interface ThrowingConsumer<V> {
void process(V value) throws Exception;
+
+ @SuppressWarnings("Duplicates")
+ static <T> Consumer<T> asUnchecked(ThrowingConsumer<T> consumer) {
+ return input -> {
+ try {
+ consumer.process(input);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new UncheckedExecutionException(e);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e);
+ }
+ };
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIntConsumer.java
similarity index 56%
copy from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
copy to hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIntConsumer.java
index bfc5fdd..07575bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIntConsumer.java
@@ -18,7 +18,26 @@
*/
package org.apache.hyracks.util;
+import java.util.function.IntConsumer;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
@FunctionalInterface
-public interface ThrowingConsumer<V> {
- void process(V value) throws Exception;
+public interface ThrowingIntConsumer {
+ void process(int value) throws Exception;
+
+ @SuppressWarnings("Duplicates")
+ static IntConsumer asUnchecked(ThrowingIntConsumer consumer) {
+ return input -> {
+ try {
+ consumer.process(input);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new UncheckedExecutionException(e);
+ } catch (Exception e) {
+ throw new UncheckedExecutionException(e);
+ }
+ };
+ }
+
}
[asterixdb] 05/05: Merge branch 'gerrit/mad-hatter'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6a8dafa814c3b654df789dfe9fbb780989003482
Merge: 60b27fc 6c239e7
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon Jun 22 10:38:42 2020 -0400
Merge branch 'gerrit/mad-hatter'
Change-Id: I5cd06cc3cd69068faecced6a2b1cc558eecd0784
.../SqlppExpressionToPlanTranslator.java | 24 +++++++++----------
.../apache/asterix/translator/TypeTranslator.java | 20 +++++++++-------
.../apache/asterix/app/active/RecoveryTask.java | 6 ++---
.../asterix/app/translator/QueryTranslator.java | 14 ++++++-----
.../create-dataset-inline-type-2.5.ddl.sqlpp | 14 +++++++----
.../ASTERIXDB-2750_unnest_join.1.query.sqlpp | 10 ++++----
.../ASTERIXDB-2750_unnest_join.2.query.sqlpp | 11 +++++----
.../ASTERIXDB-2750_unnest_join.1.adm | 1 +
.../ASTERIXDB-2750_unnest_join.2.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 6 +++++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 6 ++---
hyracks-fullstack/hyracks/hyracks-util/pom.xml | 4 ++++
.../java/org/apache/hyracks/util/IRetryPolicy.java | 2 +-
.../org/apache/hyracks/util/ThrowingConsumer.java | 19 +++++++++++++++
...{IRetryPolicy.java => ThrowingIntConsumer.java} | 28 ++++++++++++++++------
15 files changed, 110 insertions(+), 56 deletions(-)
diff --cc asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TypeTranslator.java
index aa00fb3,cb6515b..6e02fc2
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TypeTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TypeTranslator.java
@@@ -28,7 -29,8 +28,9 @@@ import java.util.Map
import org.apache.asterix.common.annotations.IRecordFieldDataGen;
import org.apache.asterix.common.annotations.RecordDataGenAnnotation;
+ import org.apache.asterix.common.exceptions.CompilationException;
+ import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
import org.apache.asterix.lang.common.expression.RecordTypeDefinition.RecordKind;
@@@ -51,7 -52,7 +53,8 @@@ import org.apache.asterix.om.types.Abst
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.TypeSignature;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+ import org.apache.hyracks.api.exceptions.SourceLocation;
public class TypeTranslator {
@@@ -72,12 -70,12 +75,12 @@@
Map<String, Map<ARecordType, List<Integer>>> incompleteFieldTypes = new HashMap<>();
Map<TypeSignature, List<AbstractCollectionType>> incompleteItemTypes = new HashMap<>();
Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences = new HashMap<>();
- firstPass(typeExpr, typeName, typeMap, incompleteFieldTypes, incompleteItemTypes,
+ firstPass(typeDataverse, typeName, typeExpr, outTypeMap, incompleteFieldTypes, incompleteItemTypes,
- incompleteTopLevelTypeReferences, defaultDataverse);
+ incompleteTopLevelTypeReferences, typeDataverse);
- secondPass(mdTxnCtx, typeMap, incompleteFieldTypes, incompleteItemTypes, incompleteTopLevelTypeReferences,
+ secondPass(mdTxnCtx, outTypeMap, incompleteFieldTypes, incompleteItemTypes, incompleteTopLevelTypeReferences,
- typeDataverse);
+ typeDataverse, typeExpr.getSourceLocation());
- for (IAType type : typeMap.values()) {
+ for (IAType type : outTypeMap.values()) {
if (type.getTypeTag().isDerivedType()) {
((AbstractComplexType) type).generateNestedDerivedTypeNames();
}
@@@ -136,15 -137,15 +140,15 @@@
private static void secondPass(MetadataTransactionContext mdTxnCtx, Map<TypeSignature, IAType> typeMap,
Map<String, Map<ARecordType, List<Integer>>> incompleteFieldTypes,
Map<TypeSignature, List<AbstractCollectionType>> incompleteItemTypes,
- Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences, DataverseName typeDataverse)
- throws AlgebricksException {
- Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences, String typeDataverse,
++ Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences, DataverseName typeDataverse,
+ SourceLocation sourceLoc) throws AlgebricksException {
// solve remaining top level references
for (TypeSignature typeSignature : incompleteTopLevelTypeReferences.keySet()) {
IAType t;
- Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, typeSignature.getNamespace(),
+ Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, typeSignature.getDataverseName(),
typeSignature.getName());
if (dt == null) {
- throw new AlgebricksException("Could not resolve type " + typeSignature);
+ throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, typeSignature.getName());
} else {
t = dt.getDatatype();
}
@@@ -185,10 -186,10 +189,10 @@@
IAType t;
Datatype dt;
if (MetadataManager.INSTANCE != null) {
- dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, typeSignature.getNamespace(),
+ dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, typeSignature.getDataverseName(),
typeSignature.getName());
if (dt == null) {
- throw new AlgebricksException("Could not resolve type " + typeSignature);
+ throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, typeSignature.getName());
}
t = dt.getDatatype();
} else {
diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index c0a719b,f6536ac..a5fef11
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -694,11 -653,10 +695,11 @@@ public class QueryTranslator extends Ab
IAType itemType;
switch (itemTypeExpr.getTypeKind()) {
case TYPEREFERENCE:
- Datatype itemTypeEntity = metadataProvider.findTypeEntity(itemTypeDataverseName, itemTypeName);
+ itemTypeEntity = metadataProvider.findTypeEntity(itemTypeDataverseName, itemTypeName);
if (itemTypeEntity == null || itemTypeEntity.getIsAnonymous()) {
// anonymous types cannot be referred from CREATE DATASET
- throw new AsterixException(ErrorCode.UNKNOWN_TYPE, sourceLoc, itemTypeFullyQualifiedName);
+ throw new AsterixException(ErrorCode.UNKNOWN_TYPE, sourceLoc,
+ DatasetUtil.getFullyQualifiedDisplayName(itemTypeDataverseName, itemTypeName));
}
itemType = itemTypeEntity.getDatatype();
validateDatasetItemType(dsType, itemType, false, sourceLoc);
[asterixdb] 02/05: [NO ISSUE][MISC] IRetryPolicy requires non-null
throwable
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b0acb8bab6f5957f89bd560c143ea5dcf6bfe972
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Jun 21 18:35:52 2020 -0400
[NO ISSUE][MISC] IRetryPolicy requires non-null throwable
Change-Id: I4835266435fc31f5973f4312e6ddcbdb85d1e839
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6923
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../src/main/java/org/apache/asterix/app/active/RecoveryTask.java | 6 +++---
.../src/main/java/org/apache/hyracks/util/IRetryPolicy.java | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
index a1989fc..c795147 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
@@ -106,8 +106,8 @@ public class RecoveryTask {
protected Void doRecover(IRetryPolicy policy) throws AlgebricksException, InterruptedException {
LOGGER.log(level, "Actual Recovery task has started");
- Exception failure = null;
- while (policy.retry(failure)) {
+ Exception failure;
+ do {
synchronized (listener) {
while (!cancelRecovery && clusterStateManager.getState() != ClusterState.ACTIVE) {
listener.wait();
@@ -139,7 +139,7 @@ public class RecoveryTask {
} finally {
releaseRecoveryLocks(metadataProvider);
}
- }
+ } while (policy.retry(failure));
// Recovery task is essntially over now either through failure or through cancellation(stop)
synchronized (listener) {
listener.notifyAll();
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IRetryPolicy.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IRetryPolicy.java
index 29469d5..0d18a2b 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IRetryPolicy.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IRetryPolicy.java
@@ -22,7 +22,7 @@ package org.apache.hyracks.util;
public interface IRetryPolicy {
/**
* @param failure
- * the cause of the failure
+ * the cause of the failure (this cannot be null)
* @return true if one more attempt should be done
*/
boolean retry(Throwable failure);
[asterixdb] 04/05: [NO ISSUE][COMP] Improve error reporting in
TypeTranslator
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6c239e71a2fc99e4803e19424d51a43cab889938
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Fri Jun 19 19:46:27 2020 -0700
[NO ISSUE][COMP] Improve error reporting in TypeTranslator
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add error code and source location to exceptions
raised by TypeTranslator
- Pass dataset item type when calling
QueryTranslator.validateExternalDatasetProperties()
Change-Id: I2b62c162f37e72245cdbf70fe9f8307bbf9b2bf1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6885
Reviewed-by: Till Westmann <ti...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../apache/asterix/translator/TypeTranslator.java | 18 ++++++++------
.../asterix/app/translator/QueryTranslator.java | 14 ++++++-----
.../create-dataset-inline-type-2.5.ddl.sqlpp | 28 ++++++++++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 1 +
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 6 ++---
5 files changed, 51 insertions(+), 16 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TypeTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TypeTranslator.java
index cdb6346..cb6515b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TypeTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TypeTranslator.java
@@ -29,6 +29,8 @@ import java.util.Map;
import org.apache.asterix.common.annotations.IRecordFieldDataGen;
import org.apache.asterix.common.annotations.RecordDataGenAnnotation;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
import org.apache.asterix.lang.common.expression.RecordTypeDefinition.RecordKind;
@@ -50,6 +52,7 @@ import org.apache.asterix.om.types.AbstractComplexType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.TypeSignature;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.exceptions.SourceLocation;
public class TypeTranslator {
@@ -70,7 +73,7 @@ public class TypeTranslator {
firstPass(typeExpr, typeName, typeMap, incompleteFieldTypes, incompleteItemTypes,
incompleteTopLevelTypeReferences, typeDataverse);
secondPass(mdTxnCtx, typeMap, incompleteFieldTypes, incompleteItemTypes, incompleteTopLevelTypeReferences,
- typeDataverse);
+ typeDataverse, typeExpr.getSourceLocation());
for (IAType type : typeMap.values()) {
if (type.getTypeTag().isDerivedType()) {
@@ -87,7 +90,8 @@ public class TypeTranslator {
throws AlgebricksException {
if (BuiltinTypeMap.getBuiltinType(typeName) != null) {
- throw new AlgebricksException("Cannot redefine builtin type " + typeName + " .");
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, typeExpr.getSourceLocation(),
+ "Cannot redefine builtin type " + typeName);
}
TypeSignature typeSignature = new TypeSignature(typeDataverse, typeName);
switch (typeExpr.getTypeKind()) {
@@ -133,15 +137,15 @@ public class TypeTranslator {
private static void secondPass(MetadataTransactionContext mdTxnCtx, Map<TypeSignature, IAType> typeMap,
Map<String, Map<ARecordType, List<Integer>>> incompleteFieldTypes,
Map<TypeSignature, List<AbstractCollectionType>> incompleteItemTypes,
- Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences, String typeDataverse)
- throws AlgebricksException {
+ Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences, String typeDataverse,
+ SourceLocation sourceLoc) throws AlgebricksException {
// solve remaining top level references
for (TypeSignature typeSignature : incompleteTopLevelTypeReferences.keySet()) {
IAType t;
Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, typeSignature.getNamespace(),
typeSignature.getName());
if (dt == null) {
- throw new AlgebricksException("Could not resolve type " + typeSignature);
+ throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, typeSignature.getName());
} else {
t = dt.getDatatype();
}
@@ -158,7 +162,7 @@ public class TypeTranslator {
trefName);
}
if (dt == null) {
- throw new AlgebricksException("Could not resolve type " + trefName);
+ throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, trefName);
} else {
t = dt.getDatatype();
}
@@ -185,7 +189,7 @@ public class TypeTranslator {
dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, typeSignature.getNamespace(),
typeSignature.getName());
if (dt == null) {
- throw new AlgebricksException("Could not resolve type " + typeSignature);
+ throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, typeSignature.getName());
}
t = dt.getDatatype();
} else {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 1be3894..f6536ac 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -649,10 +649,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
throw new CompilationException(ErrorCode.DATASET_EXISTS, sourceLoc, datasetName, dataverseName);
}
}
+ Datatype itemTypeEntity;
IAType itemType;
switch (itemTypeExpr.getTypeKind()) {
case TYPEREFERENCE:
- Datatype itemTypeEntity = metadataProvider.findTypeEntity(itemTypeDataverseName, itemTypeName);
+ itemTypeEntity = metadataProvider.findTypeEntity(itemTypeDataverseName, itemTypeName);
if (itemTypeEntity == null || itemTypeEntity.getIsAnonymous()) {
// anonymous types cannot be referred from CREATE DATASET
throw new AsterixException(ErrorCode.UNKNOWN_TYPE, sourceLoc, itemTypeFullyQualifiedName);
@@ -663,8 +664,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
case RECORD:
itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx);
validateDatasetItemType(dsType, itemType, false, sourceLoc);
- MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
- new Datatype(itemTypeDataverseName, itemTypeName, itemType, true));
+ itemTypeEntity = new Datatype(itemTypeDataverseName, itemTypeName, itemType, true);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx, itemTypeEntity);
break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
@@ -740,7 +741,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
createExternalDatasetProperties(dataverseName, dd, metadataProvider, mdTxnCtx);
ExternalDataUtils.normalize(properties);
ExternalDataUtils.validate(properties);
- validateExternalDatasetProperties(externalDetails, properties, dd.getSourceLocation(), mdTxnCtx);
+ validateExternalDatasetProperties(externalDetails, properties, itemTypeEntity,
+ dd.getSourceLocation(), mdTxnCtx);
datasetDetails = new ExternalDatasetDetails(externalDetails.getAdapter(), properties, new Date(),
TransactionState.COMMIT);
break;
@@ -3238,8 +3240,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
protected void validateExternalDatasetProperties(ExternalDetailsDecl externalDetails,
- Map<String, String> properties, SourceLocation srcLoc, MetadataTransactionContext mdTxnCtx)
- throws AlgebricksException, HyracksDataException {
+ Map<String, String> properties, Datatype itemType, SourceLocation srcLoc,
+ MetadataTransactionContext mdTxnCtx) throws AlgebricksException, HyracksDataException {
// Validate adapter specific properties
String adapter = externalDetails.getAdapter();
Map<String, String> details = new HashMap<>(properties);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-2/create-dataset-inline-type-2.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-2/create-dataset-inline-type-2.5.ddl.sqlpp
new file mode 100644
index 0000000..62c0694
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-2/create-dataset-inline-type-2.5.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* Create dataset that attempts to use unknown type
+ in inline type definition */
+
+USE test;
+
+CREATE DATASET Cust3X(
+ c_custkey integer not unknown,
+ c_name my_unknown_type
+) PRIMARY KEY c_custkey;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 44e2bff..63773dd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3888,6 +3888,7 @@
<output-dir compare="Text">create-dataset-inline-type-2</output-dir>
<expected-error>ASX1082: Cannot find datatype with name test.$d$t$i$Cust1</expected-error>
<expected-error>ASX1082: Cannot find datatype with name test.$d$t$i$Cust2</expected-error>
+ <expected-error>ASX1082: Cannot find datatype with name my_unknown_type</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="ddl">
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 7538887..f4061e1 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -773,10 +773,10 @@ TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) t
{
RecordTypeDefinition recordTypeDef = null;
RecordTypeDefinition.RecordKind recordKind = null;
- Token recordKindToken = null;
+ Token startToken = null, recordKindToken = null;
}
{
- <LEFTPAREN> recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN>
+ <LEFTPAREN> { startToken = token; } recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN>
( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )?
{
if (recordKind == null) {
@@ -785,7 +785,7 @@ TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) t
throw createUnexpectedTokenError(recordKindToken);
}
recordTypeDef.setRecordKind(recordKind);
- return recordTypeDef;
+ return addSourceLocation(recordTypeDef, startToken);
}
}