You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2024/11/05 13:42:26 UTC
(asterixdb) branch master updated (7156522359 -> 2f061939c0)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
from 7156522359 [ASTERIXDB-3483]: Add method for extensions
new 3164da5f58 [ASTERIXDB-3503][EXT] Deltalake format support.
new 6a8121408b [ASTERIXDB-3502][STO] Incorrectly treating two expressions as commutative
new 5abe3c520b Merge branch 'gerrit/neo' into 'gerrit/trinity'
new 041571d17e [ASTERIXDB-3512][COMP] Error with query if secondary indexes created
new e49261f38c Merge branch 'gerrit/neo' into 'gerrit/trinity'
new 8e36160d5b Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
new 1723d0c835 Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish
new 7e7960c090 [ASTERIXDB-3503][EXT] Deltalake invalid type error code change
new 2f061939c0 Merge branch 'gerrit/goldfish' into 'master'
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../optimizerts/queries/query-ASTERIXDB-3512.sqlpp | 44 ++++++++++++++++++++++
.../optimizerts/results/query-ASTERIXDB-3512.plan | 28 ++++++++++++++
.../query-ASTERIXDB-3512.1.ddl.sqlpp | 34 +++++++++++++++++
.../query-ASTERIXDB-3512.2.update.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.query.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++
.../runtimets/testsuite_external_dataset_s3.xml | 4 +-
.../asterix/common/exceptions/ErrorCode.java | 4 +-
.../src/main/resources/asx_errormsg/en.properties | 4 +-
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 ++++++--
11 files changed, 206 insertions(+), 10 deletions(-)
create mode 100644 asterixdb/asterix-app/src/test/resources/optimizerts/queries/query-ASTERIXDB-3512.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/optimizerts/results/query-ASTERIXDB-3512.plan
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.1.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.2.update.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.query.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.adm
(asterixdb) 01/09: [ASTERIXDB-3502][STO] Incorrectly treating two expressions as commutative
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6a8121408b1b70ce4b12e5a9741c6367ef7a15fa
Author: Ritik Raj <ra...@gmail.com>
AuthorDate: Sat Sep 14 00:51:35 2024 +0530
[ASTERIXDB-3502][STO] Incorrectly treating two expressions as commutative
- user model changes: no
- storage format changes: no
- interface changes: no
- Incorrectly considering two different expression as commutative
Ext-ref: MB-63525
Change-Id: Ia83d1559f5f45406075a384ce25251e50b362b5b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18875
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../ASTERIXDB-3502/ASTERIXDB-3502.01.ddl.sqlpp | 61 +++++++++++++++
.../ASTERIXDB-3502/ASTERIXDB-3502.02.update.sqlpp | 26 +++++++
.../ASTERIXDB-3502/ASTERIXDB-3502.03.query.sqlpp | 39 ++++++++++
.../join/ASTERIXDB-3502/ASTERIXDB-3502.03.adm | 0
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 ++
.../asterix/lang/common/util/FunctionUtil.java | 17 +---
.../lang/expression/CommutativeEqualsTest.java | 90 ++++++++++++++++++++--
7 files changed, 217 insertions(+), 21 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.01.ddl.sqlpp
new file mode 100644
index 0000000000..e4c59d2831
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.01.ddl.sqlpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+DROP DATAVERSE tpch IF EXISTS;
+CREATE DATAVERSE tpch;
+
+USE tpch;
+
+CREATE TYPE tpch.CustomerType AS
+ CLOSED {
+ c_custkey : integer,
+ c_name : string,
+ c_address : string,
+ c_nationkey : integer,
+ c_phone : string,
+ c_acctbal : double,
+ c_mktsegment : string,
+ c_comment : string
+};
+
+CREATE TYPE tpch.SupplierType AS
+ CLOSED {
+ s_suppkey : integer,
+ s_name : string,
+ s_address : string,
+ s_nationkey : integer,
+ s_phone : string,
+ s_acctbal : double,
+ s_comment : string
+};
+
+CREATE TYPE tpch.NationType AS
+ CLOSED {
+ n_nationkey : integer,
+ n_name : string,
+ n_regionkey : integer,
+ n_comment : string
+};
+
+CREATE DATASET Supplier(SupplierType) PRIMARY KEY s_suppkey;
+
+CREATE DATASET Nation(NationType) PRIMARY KEY n_nationkey;
+
+CREATE DATASET Customer(CustomerType) PRIMARY KEY c_custkey;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.02.update.sqlpp
new file mode 100644
index 0000000000..b4ae2b6914
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.02.update.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE tpch;
+
+LOAD DATASET Supplier USING localfs ((`path`=`asterix_nc1://data/tpch0.001/supplier.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+LOAD DATASET Nation USING localfs ((`path`=`asterix_nc1://data/tpch0.001/nation.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+LOAD DATASET Customer USING localfs ((`path`=`asterix_nc1://data/tpch0.001/customer.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.03.query.sqlpp
new file mode 100644
index 0000000000..76750f9fe9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/ASTERIXDB-3502/ASTERIXDB-3502.03.query.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Similar to hash-join-with-redundant-variable.04.query.sqlpp
+ * But with Index NL. The plan of this test should has three
+ * hash-partition-exchange (as opposed to test 13 & 14). Because the parallelism
+ * is set to 3, then the last join requires both sides to be hash partitioned.
+ * Customer will need to duplicate its variable to join both with Nation and Supplier.
+ * This is the effect of using Index NL with parallelism != # of partitions
+ */
+
+USE tpch;
+
+-- this query should not give any results
+
+SELECT n.n_nationkey, s.s_nationkey, c.c_nationkey
+FROM Nation n, Supplier s, Customer c
+WHERE s.s_nationkey = n.n_nationkey
+AND c.c_nationkey = n.n_nationkey
+AND (s.s_nationkey = c.c_nationkey) = (s.s_nationkey = c.c_nationkey)
+-- before ASTERIXDB-3502, below expression was getting removed, hence evaluated to wrong result
+AND (s.s_nationkey != c.c_nationkey) = (s.s_nationkey = c.c_nationkey)
+ORDER BY n.n_nationkey, s.s_nationkey, c.c_nationkey;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/join/ASTERIXDB-3502/ASTERIXDB-3502.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/join/ASTERIXDB-3502/ASTERIXDB-3502.03.adm
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index c63de247ad..717700f96d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -6654,6 +6654,11 @@
<output-dir compare="Text">hash-join-with-redundant-variable</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="join">
+ <compilation-unit name="ASTERIXDB-3502">
+ <output-dir compare="Text">ASTERIXDB-3502</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="join">
<compilation-unit name="hash_join_array">
<output-dir compare="Text">hash_join_array</output-dir>
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index cce0bd8d31..24b38ab9d6 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -347,30 +347,17 @@ public class FunctionUtil {
return expr1.equals(expr2);
}
- return commutativeEquals(expr1, expr2, new BitSet());
- }
-
- private static boolean commutativeEquals(ILogicalExpression expr1, ILogicalExpression expr2, BitSet matched) {
- if (expr1.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL
- || expr2.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return expr1.equals(expr2);
- }
-
- AbstractFunctionCallExpression funcExpr1 = (AbstractFunctionCallExpression) expr1;
- AbstractFunctionCallExpression funcExpr2 = (AbstractFunctionCallExpression) expr2;
-
List<Mutable<ILogicalExpression>> args1 = funcExpr1.getArguments();
List<Mutable<ILogicalExpression>> args2 = funcExpr2.getArguments();
- BitSet childrenSet = new BitSet();
+ BitSet matched = new BitSet();
int numberOfMatches = 0;
for (Mutable<ILogicalExpression> arg1 : args1) {
int prevNumberOfMatches = numberOfMatches;
for (int i = 0; i < args2.size(); i++) {
Mutable<ILogicalExpression> arg2 = args2.get(i);
- childrenSet.clear();
- if (!matched.get(i) && commutativeEquals(arg1.getValue(), arg2.getValue(), childrenSet)) {
+ if (!matched.get(i) && commutativeEquals(arg1.getValue(), arg2.getValue())) {
matched.set(i);
numberOfMatches++;
break;
diff --git a/asterixdb/asterix-lang-sqlpp/src/test/java/org/apache/asterix/lang/expression/CommutativeEqualsTest.java b/asterixdb/asterix-lang-sqlpp/src/test/java/org/apache/asterix/lang/expression/CommutativeEqualsTest.java
index 06f5ba7187..aba83db4f8 100644
--- a/asterixdb/asterix-lang-sqlpp/src/test/java/org/apache/asterix/lang/expression/CommutativeEqualsTest.java
+++ b/asterixdb/asterix-lang-sqlpp/src/test/java/org/apache/asterix/lang/expression/CommutativeEqualsTest.java
@@ -24,11 +24,14 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
@@ -46,7 +49,7 @@ public class CommutativeEqualsTest {
private int varCounter;
@Test
- public void testTwoOperands() {
+ public void testExpressions() {
// EQ
reset();
ILogicalExpression expr1 = createExpression(BuiltinFunctions.EQ, 'x', 'y');
@@ -62,6 +65,69 @@ public class CommutativeEqualsTest {
expr1 = createExpression(BuiltinFunctions.EQ, 'x', 'x');
expr2 = createExpression(BuiltinFunctions.EQ, 'x', 'y');
Assert.assertFalse(FunctionUtil.commutativeEquals(expr1, expr2));
+
+ // ( ( NOT ( bool_field1 ) ) ) AND ( ( bool_field1 ) )
+ // ( ( bool_field1 ) ) AND ( ( bool_field1 = true ) )
+ reset();
+ expr1 = createExpression(BuiltinFunctions.AND,
+ createExpression(BuiltinFunctions.NOT, getVariableExpression('x')), getVariableExpression('x'));
+
+ expr2 = createExpression(BuiltinFunctions.AND, getVariableExpression('x'),
+ createExpression(BuiltinFunctions.EQ, getVariableExpression('x'), ConstantExpression.TRUE));
+ Assert.assertFalse(FunctionUtil.commutativeEquals(expr1, expr2));
+
+ // ( UPPER(varchar_field1) = varchar_field1 AND int_field1 = 2 )
+ // ( LOWER(varchar_field1) = varchar_field1 AND int_field1 = 2 )
+ reset();
+ expr1 = createExpression(BuiltinFunctions.AND,
+ createExpression(BuiltinFunctions.EQ,
+ createExpression(BuiltinFunctions.STRING_UPPERCASE, getVariableExpression('x')),
+ ConstantExpression.TRUE),
+ createExpression(BuiltinFunctions.EQ, getVariableExpression('y'),
+ new ConstantExpression(new AsterixConstantValue(new AInt32(2)))));
+
+ expr2 = createExpression(BuiltinFunctions.AND,
+ createExpression(BuiltinFunctions.EQ,
+ createExpression(BuiltinFunctions.STRING_LOWERCASE, getVariableExpression('x')),
+ ConstantExpression.TRUE),
+ createExpression(BuiltinFunctions.EQ, getVariableExpression('y'),
+ new ConstantExpression(new AsterixConstantValue(new AInt32(2)))));
+
+ Assert.assertFalse(FunctionUtil.commutativeEquals(expr1, expr2));
+
+ expr2 = createExpression(BuiltinFunctions.EQ,
+ createExpression(BuiltinFunctions.EQ,
+ createExpression(BuiltinFunctions.STRING_LOWERCASE, getVariableExpression('x')),
+ ConstantExpression.TRUE),
+ createExpression(BuiltinFunctions.EQ, getVariableExpression('y'),
+ new ConstantExpression(new AsterixConstantValue(new AInt32(2)))));
+
+ // ( LOWER(varchar_field1) = varchar_field1 AND int_field1 = 2 )
+ // ( int_field1 = 2 AND LOWER(varchar_field1) = varchar_field1)
+ // should evaluate to true
+ ILogicalExpression expr3 = createExpression(BuiltinFunctions.EQ,
+ createExpression(BuiltinFunctions.EQ, getVariableExpression('y'),
+ new ConstantExpression(new AsterixConstantValue(new AInt32(2)))),
+ createExpression(BuiltinFunctions.EQ,
+ createExpression(BuiltinFunctions.STRING_LOWERCASE, getVariableExpression('x')),
+ ConstantExpression.TRUE));
+
+ Assert.assertTrue(FunctionUtil.commutativeEquals(expr2, expr3));
+
+ // ( 10/int_field1=6911432 ) AND ( bool_field1=true )
+ // ( int_field1/10=6911432 ) AND ( bool_field1 = true )
+ reset();
+ expr1 = createExpression(BuiltinFunctions.AND,
+ createExpression(BuiltinFunctions.NUMERIC_DIV,
+ new ConstantExpression(new AsterixConstantValue(new AInt32(10))), getVariableExpression('i')),
+ createExpression(BuiltinFunctions.EQ, getVariableExpression('b'), ConstantExpression.TRUE));
+
+ expr2 = createExpression(BuiltinFunctions.AND,
+ createExpression(BuiltinFunctions.NUMERIC_DIV, getVariableExpression('i'),
+ new ConstantExpression(new AsterixConstantValue(new AInt32(10)))),
+ createExpression(BuiltinFunctions.EQ, getVariableExpression('b'), ConstantExpression.TRUE));
+
+ Assert.assertFalse(FunctionUtil.commutativeEquals(expr1, expr2));
}
private void reset() {
@@ -69,19 +135,31 @@ public class CommutativeEqualsTest {
varNameToVarMap.clear();
}
- private ILogicalExpression createExpression(FunctionIdentifier fid, char left, char right) {
+ private ILogicalExpression createExpression(FunctionIdentifier fid, ILogicalExpression... left) {
+ List<Mutable<ILogicalExpression>> args = new ArrayList<>();
+
+ for (ILogicalExpression expr : left) {
+ args.add(new MutableObject<>(expr));
+ }
+
+ IFunctionInfo funcInfo = BuiltinFunctions.getBuiltinFunctionInfo(fid);
+ return new ScalarFunctionCallExpression(funcInfo, args);
+ }
+
+ private ILogicalExpression createExpression(FunctionIdentifier fid, char... left) {
List<Mutable<ILogicalExpression>> args = new ArrayList<>();
- args.add(getVariableExpression(left));
- args.add(getVariableExpression(right));
+ for (int i = 0; i < left.length; i++) {
+ args.add(new MutableObject<>(getVariableExpression(left[i])));
+ }
IFunctionInfo funcInfo = BuiltinFunctions.getBuiltinFunctionInfo(fid);
return new ScalarFunctionCallExpression(funcInfo, args);
}
- private Mutable<ILogicalExpression> getVariableExpression(Character displayName) {
+ private ILogicalExpression getVariableExpression(Character displayName) {
LogicalVariable variable = varNameToVarMap.computeIfAbsent(displayName,
k -> new LogicalVariable(varCounter++, displayName.toString()));
- return new MutableObject<>(new VariableReferenceExpression(variable));
+ return new VariableReferenceExpression(variable);
}
}
(asterixdb) 05/09: [ASTERIXDB-3503][EXT] Deltalake format support.
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3164da5f58c2a760f3930876e796e9573f920c7b
Author: ayush.tripathi <ay...@couchbase.com>
AuthorDate: Wed Aug 21 15:49:53 2024 +0530
[ASTERIXDB-3503][EXT] Deltalake format support.
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Creating external dataset from a given Deltalake table path(S3 as source)..
Ext-ref: MB-63840
Change-Id: Iff608397aab711f324861fe83eeb428f73682912
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18708
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ayush Tripathi <Ay...@couchbase.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
(cherry picked from commit b96e2e181ef4960aeabbb2c1ff228b7dae8048e7)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18983
---
.../external_dataset/ExternalDatasetTestUtils.java | 34 ++++
.../aws/AwsS3ExternalDatasetTest.java | 2 +
.../deltalake/DeltaTableGenerator.java | 224 +++++++++++++++++++++
.../deltalake-empty/deltalake-empty.00.ddl.sqlpp | 35 ++++
.../deltalake-empty/deltalake-empty.01.query.sqlpp | 22 ++
.../deltalake-modified-data.00.ddl.sqlpp | 35 ++++
.../deltalake-modified-data.01.query.sqlpp | 22 ++
.../deltalake-multiple_file_read.00.ddl.sqlpp | 35 ++++
.../deltalake-multiple_file_read.01.query.sqlpp | 22 ++
.../common/deltalake-empty/deltalake-empty.01.adm | 0
.../deltalake-modified-data.01.adm | 1 +
.../deltalake-multiple-file-read/read-data.2.adm | 4 +
.../s3/deltalake-empty/deltalake-empty.01.adm | 0
.../deltalake-modified-data.01.adm | 1 +
.../external-dataset/s3/deltalake/read-data.2.adm | 4 +
.../runtimets/testsuite_external_dataset_s3.xml | 19 ++
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
asterixdb/asterix-external-data/pom.xml | 5 +
.../reader/aws/delta/AwsS3DeltaReaderFactory.java | 68 +++++++
.../provider/DatasourceFactoryProvider.java | 6 +
.../external/util/ExternalDataConstants.java | 3 +-
.../asterix/external/util/ExternalDataUtils.java | 134 +++++++-----
.../asterix/external/util/aws/s3/S3Utils.java | 8 +-
...pache.asterix.external.api.IRecordReaderFactory | 1 +
25 files changed, 635 insertions(+), 52 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 79d298a7b3..db1bf5039b 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -23,6 +23,7 @@ import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetT
import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.BROWSE_CONTAINER;
import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.DYNAMIC_PREFIX_AT_START_CONTAINER;
import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER;
+import static org.apache.asterix.test.external_dataset.deltalake.DeltaTableGenerator.DELTA_GEN_BASEDIR;
import static org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.BINARY_GEN_BASEDIR;
import java.io.BufferedWriter;
@@ -36,9 +37,11 @@ import java.nio.file.Paths;
import java.util.Collection;
import org.apache.asterix.test.external_dataset.avro.AvroFileConverterUtil;
+import org.apache.asterix.test.external_dataset.deltalake.DeltaTableGenerator;
import org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.commons.io.FilenameUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hyracks.api.util.IoUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -114,6 +117,13 @@ public class ExternalDatasetTestUtils {
AvroFileConverterUtil.convertToAvro(basePath, avroRawJsonDir, AVRO_GEN_BASEDIR);
}
+ public static void createDeltaTable() throws IOException {
+ File basePath = new File(".");
+ // cleaning directory
+ BinaryFileConverterUtil.cleanBinaryDirectory(basePath, DELTA_GEN_BASEDIR);
+ DeltaTableGenerator.prepareDeltaTableContainer(new Configuration());
+ }
+
/**
* Generate binary files (e.g., parquet files)
*/
@@ -192,6 +202,10 @@ public class ExternalDatasetTestUtils {
loadAvroFiles();
LOGGER.info("Avro files added successfully");
+ LOGGER.info("Adding Delta Table files to the bucket");
+ loadDeltaTableFiles();
+ LOGGER.info("Delta files added successfully");
+
LOGGER.info("Files added successfully");
}
@@ -412,6 +426,26 @@ public class ExternalDatasetTestUtils {
}
}
+ private static void loadDeltaTableFiles() {
+ String generatedDataBasePath = DELTA_GEN_BASEDIR;
+ loadDeltaDirectory(generatedDataBasePath, "/empty_delta_table", PARQUET_FILTER, "delta-data/");
+ loadDeltaDirectory(generatedDataBasePath, "/empty_delta_table/_delta_log", JSON_FILTER, "delta-data/");
+ loadDeltaDirectory(generatedDataBasePath, "/modified_delta_table", PARQUET_FILTER, "delta-data/");
+ loadDeltaDirectory(generatedDataBasePath, "/modified_delta_table/_delta_log", JSON_FILTER, "delta-data/");
+ loadDeltaDirectory(generatedDataBasePath, "/multiple_file_delta_table", PARQUET_FILTER, "delta-data/");
+ loadDeltaDirectory(generatedDataBasePath, "/multiple_file_delta_table/_delta_log", JSON_FILTER, "delta-data/");
+ }
+
+ private static void loadDeltaDirectory(String dataBasePath, String rootPath, FilenameFilter filter,
+ String definitionPart) {
+ Collection<File> files = IoUtil.getMatchingFiles(Paths.get(dataBasePath + rootPath), filter);
+ for (File file : files) {
+ String fileName = file.getName();
+ String externalFilterDefinition = file.getParent().substring(dataBasePath.length() + 1) + "/";
+ loadData(file.getParent(), "", fileName, definitionPart + externalFilterDefinition, "", false, false);
+ }
+ }
+
private static void loadDirectory(String dataBasePath, String rootPath, FilenameFilter filter) {
File dir = new File(dataBasePath, rootPath);
if (!dir.exists() || !dir.isDirectory()) {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 9a2bfbcc7c..665b8057ec 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -22,6 +22,7 @@ import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.
import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createAvroFilesRecursively;
import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createBinaryFiles;
import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createBinaryFilesRecursively;
+import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createDeltaTable;
import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.setDataPaths;
import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.setUploaders;
import static org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.DEFAULT_PARQUET_SRC_PATH;
@@ -199,6 +200,7 @@ public class AwsS3ExternalDatasetTest {
createBinaryFilesRecursively(EXTERNAL_FILTER_DATA_PATH);
createAvroFiles(DEFAULT_PARQUET_SRC_PATH);
createAvroFilesRecursively(EXTERNAL_FILTER_DATA_PATH);
+ createDeltaTable();
setNcEndpoints(testExecutor);
startAwsS3MockServer();
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaTableGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaTableGenerator.java
new file mode 100644
index 0000000000..f041fa2e9c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaTableGenerator.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.external_dataset.deltalake;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.generic.GenericData;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hyracks.api.util.IoUtil;
+import org.apache.parquet.avro.AvroParquetWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+
+import io.delta.standalone.DeltaLog;
+import io.delta.standalone.Operation;
+import io.delta.standalone.OptimisticTransaction;
+import io.delta.standalone.actions.Action;
+import io.delta.standalone.actions.AddFile;
+import io.delta.standalone.actions.Metadata;
+import io.delta.standalone.actions.RemoveFile;
+import io.delta.standalone.types.IntegerType;
+import io.delta.standalone.types.StringType;
+import io.delta.standalone.types.StructField;
+import io.delta.standalone.types.StructType;
+
+public class DeltaTableGenerator {
+ public static final String DELTA_GEN_BASEDIR = "target" + File.separatorChar + "generated_delta_files";
+ public static final String DELTA_EMPTY_TABLE =
+ "target" + File.separatorChar + "generated_delta_files" + File.separatorChar + "empty_delta_table";
+ public static final String DELTA_MODIFIED_TABLE =
+ "target" + File.separatorChar + "generated_delta_files" + File.separatorChar + "modified_delta_table";
+ public static final String DELTA_MULTI_FILE_TABLE =
+ "target" + File.separatorChar + "generated_delta_files" + File.separatorChar + "multiple_file_delta_table";
+
+ public static void prepareDeltaTableContainer(Configuration conf) {
+ File basePath = new File(".");
+ cleanBinaryDirectory(basePath, DELTA_GEN_BASEDIR);
+ prepareMultipleFilesTable(conf);
+ prepareModifiedTable(conf);
+ prepareEmptyTable(conf);
+ }
+
+ public static void cleanBinaryDirectory(File localDataRoot, String binaryFilesPath) {
+ try {
+ File destPath = new File(localDataRoot, binaryFilesPath);
+ //Delete old generated files
+ if (destPath.exists()) {
+ IoUtil.delete(destPath);
+ }
+ //Create new directory
+ Files.createDirectory(Paths.get(destPath.getAbsolutePath()));
+ } catch (IOException e) {
+
+ }
+
+ }
+
+ public static void prepareEmptyTable(Configuration conf) {
+ List<Action> actions = List.of();
+ DeltaLog log = DeltaLog.forTable(conf, DELTA_EMPTY_TABLE);
+ OptimisticTransaction txn = log.startTransaction();
+ Metadata metaData = txn.metadata().copyBuilder().partitionColumns(new ArrayList<>())
+ .schema(new StructType().add(new StructField("id", new IntegerType(), true))
+ .add(new StructField("data", new StringType(), true)))
+ .build();
+ txn.updateMetadata(metaData);
+ txn.commit(actions, new Operation(Operation.Name.CREATE_TABLE), "deltalake-table-create");
+ }
+
+ public static void prepareModifiedTable(Configuration conf) {
+ Schema schema = SchemaBuilder.record("MyRecord").fields().requiredInt("id").requiredString("data").endRecord();
+ try {
+ Path path = new Path(DELTA_MODIFIED_TABLE, "firstFile.parquet");
+ ParquetWriter<GenericData.Record> writer =
+ AvroParquetWriter.<GenericData.Record> builder(path).withConf(conf).withSchema(schema).build();
+
+ List<GenericData.Record> fileFirstSnapshotRecords = List.of(new GenericData.Record(schema),
+ new GenericData.Record(schema), new GenericData.Record(schema));
+ List<GenericData.Record> fileSecondSnapshotRecords = List.of(new GenericData.Record(schema));
+
+ fileFirstSnapshotRecords.get(0).put("id", 0);
+ fileFirstSnapshotRecords.get(0).put("data", "vibrant_mclean");
+
+ fileFirstSnapshotRecords.get(1).put("id", 1);
+ fileFirstSnapshotRecords.get(1).put("data", "frosty_wilson");
+
+ fileFirstSnapshotRecords.get(2).put("id", 2);
+ fileFirstSnapshotRecords.get(2).put("data", "serene_kirby");
+
+ fileSecondSnapshotRecords.get(0).put("id", 2);
+ fileSecondSnapshotRecords.get(0).put("data", "serene_kirby");
+
+ for (GenericData.Record record : fileFirstSnapshotRecords) {
+ writer.write(record);
+ }
+
+ long size = writer.getDataSize();
+ writer.close();
+
+ List<Action> actions = List.of(new AddFile("firstFile.parquet", new HashMap<>(), size,
+ System.currentTimeMillis(), true, null, null));
+ DeltaLog log = DeltaLog.forTable(conf, DELTA_MODIFIED_TABLE);
+ OptimisticTransaction txn = log.startTransaction();
+ Metadata metaData = txn.metadata().copyBuilder().partitionColumns(new ArrayList<>())
+ .schema(new StructType().add(new StructField("id", new IntegerType(), true))
+ .add(new StructField("data", new StringType(), true)))
+ .build();
+ txn.updateMetadata(metaData);
+ txn.commit(actions, new Operation(Operation.Name.CREATE_TABLE), "deltalake-table-create");
+
+ Path path2 = new Path(DELTA_MODIFIED_TABLE, "secondFile.parquet");
+ ParquetWriter<GenericData.Record> writer2 =
+ AvroParquetWriter.<GenericData.Record> builder(path2).withConf(conf).withSchema(schema).build();
+
+ for (GenericData.Record record : fileSecondSnapshotRecords) {
+ writer2.write(record);
+ }
+ long size2 = writer2.getDataSize();
+ writer2.close();
+ AddFile addFile = new AddFile("firstFile.parquet", new HashMap<>(), size, System.currentTimeMillis(), true,
+ null, null);
+ RemoveFile removeFile = addFile.remove();
+ List<Action> actions3 = List.of(removeFile, new AddFile("secondFile.parquet", new HashMap<>(), size2,
+ System.currentTimeMillis(), true, null, null));
+ OptimisticTransaction txn3 = log.startTransaction();
+ txn3.commit(actions3, new Operation(Operation.Name.DELETE), "deltalake-table-delete");
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void prepareMultipleFilesTable(Configuration conf) {
+ Schema schema = SchemaBuilder.record("MyRecord").fields().requiredInt("id").requiredString("name")
+ .requiredString("age").endRecord();
+ try {
+ Path path = new Path(DELTA_MULTI_FILE_TABLE, "firstFile.parquet");
+ ParquetWriter<GenericData.Record> writer =
+ AvroParquetWriter.<GenericData.Record> builder(path).withConf(conf).withSchema(schema).build();
+
+ List<GenericData.Record> fileFirstSnapshotRecords = List.of(new GenericData.Record(schema),
+ new GenericData.Record(schema), new GenericData.Record(schema));
+ List<GenericData.Record> fileSecondSnapshotRecords = List.of(new GenericData.Record(schema));
+
+ fileFirstSnapshotRecords.get(0).put("id", 0);
+ fileFirstSnapshotRecords.get(0).put("name", "Cooper");
+ fileFirstSnapshotRecords.get(0).put("age", "42");
+
+ fileFirstSnapshotRecords.get(1).put("id", 1);
+ fileFirstSnapshotRecords.get(1).put("name", "Murphy");
+ fileFirstSnapshotRecords.get(1).put("age", "16");
+
+ fileFirstSnapshotRecords.get(2).put("id", 2);
+ fileFirstSnapshotRecords.get(2).put("name", "Mann");
+ fileFirstSnapshotRecords.get(2).put("age", "45");
+
+ fileSecondSnapshotRecords.get(0).put("id", 3);
+ fileSecondSnapshotRecords.get(0).put("name", "Brand");
+ fileSecondSnapshotRecords.get(0).put("age", "35");
+
+ for (GenericData.Record record : fileFirstSnapshotRecords) {
+ writer.write(record);
+ }
+
+ long size = writer.getDataSize();
+ writer.close();
+
+ List<Action> actions = List.of(new AddFile("firstFile.parquet", new HashMap<>(), size,
+ System.currentTimeMillis(), true, null, null));
+ DeltaLog log = DeltaLog.forTable(conf, DELTA_MULTI_FILE_TABLE);
+ OptimisticTransaction txn = log.startTransaction();
+ Metadata metaData = txn.metadata().copyBuilder().partitionColumns(new ArrayList<>())
+ .schema(new StructType().add(new StructField("id", new IntegerType(), true))
+ .add(new StructField("name", new StringType(), true))
+ .add(new StructField("age", new IntegerType(), true)))
+ .build();
+ txn.updateMetadata(metaData);
+ txn.commit(actions, new Operation(Operation.Name.CREATE_TABLE), "deltalake-table-create");
+
+ Path path2 = new Path(DELTA_MULTI_FILE_TABLE, "secondFile.parquet");
+ ParquetWriter<GenericData.Record> writer2 =
+ AvroParquetWriter.<GenericData.Record> builder(path2).withConf(conf).withSchema(schema).build();
+
+ for (GenericData.Record record : fileSecondSnapshotRecords) {
+ writer2.write(record);
+ }
+
+ long size2 = writer2.getDataSize();
+ writer2.close();
+
+ List<Action> actions2 = List.of(new AddFile("secondFile.parquet", new HashMap<>(), size2,
+ System.currentTimeMillis(), true, null, null));
+ OptimisticTransaction txn2 = log.startTransaction();
+ txn2.commit(actions2, new Operation(Operation.Name.WRITE), "deltalake-table-create");
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
new file mode 100644
index 0000000000..f21f0131fc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ DROP DATAVERSE test IF EXISTS;
+ CREATE DATAVERSE test;
+
+ USE test;
+
+
+ CREATE TYPE DeltalakeTableType as {
+ };
+
+ CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ (
+ %template%,
+ ("container"="playground"),
+ ("definition"="my-table-empty"),
+ ("table-format" = "delta")
+ );
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.01.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.01.query.sqlpp
new file mode 100644
index 0000000000..84e79142a8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.01.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ USE test;
+
+ SELECT element ds FROM DeltalakeDataset as ds;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
new file mode 100644
index 0000000000..8adab7721c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+ CREATE DATAVERSE test;
+
+ USE test;
+
+
+ CREATE TYPE DeltalakeTableType as {
+ };
+
+ CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ (
+ %template%,
+ ("container"="playground"),
+ ("definition"="delta-data/modified_delta_table"),
+ ("table-format" = "delta")
+ );
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.01.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.01.query.sqlpp
new file mode 100644
index 0000000000..84e79142a8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.01.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ USE test;
+
+ SELECT element ds FROM DeltalakeDataset as ds;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
new file mode 100644
index 0000000000..61c5c3ac38
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+ CREATE DATAVERSE test;
+
+ USE test;
+
+
+ CREATE TYPE DeltalakeTableType as {
+ };
+
+ CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ (
+ %template%,
+ ("container"="playground"),
+ ("definition"="delta-data/multiple_file_delta_table"),
+ ("table-format" = "delta")
+ );
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.01.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.01.query.sqlpp
new file mode 100644
index 0000000000..bfd65819f2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.01.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ USE test;
+
+ SELECT element ds FROM DeltalakeDataset as ds order by ds.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-empty/deltalake-empty.01.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-empty/deltalake-empty.01.adm
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-modified-data/deltalake-modified-data.01.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-modified-data/deltalake-modified-data.01.adm
new file mode 100644
index 0000000000..dfdc3b0a1e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-modified-data/deltalake-modified-data.01.adm
@@ -0,0 +1 @@
+{ "id": 2, "data": "serene_kirby" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-multiple-file-read/read-data.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-multiple-file-read/read-data.2.adm
new file mode 100644
index 0000000000..afae366266
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-multiple-file-read/read-data.2.adm
@@ -0,0 +1,4 @@
+{ "id": 0, "name": "Cooper", "age": "42" }
+{ "id": 1, "name": "Murphy", "age": "16" }
+{ "id": 2, "name": "Mann", "age": "45" }
+{ "id": 3, "name": "Brand", "age": "35" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake-empty/deltalake-empty.01.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake-empty/deltalake-empty.01.adm
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake-modified-data/deltalake-modified-data.01.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake-modified-data/deltalake-modified-data.01.adm
new file mode 100644
index 0000000000..dfdc3b0a1e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake-modified-data/deltalake-modified-data.01.adm
@@ -0,0 +1 @@
+{ "id": 2, "data": "serene_kirby" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake/read-data.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake/read-data.2.adm
new file mode 100644
index 0000000000..afae366266
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/deltalake/read-data.2.adm
@@ -0,0 +1,4 @@
+{ "id": 0, "name": "Cooper", "age": "42" }
+{ "id": 1, "name": "Murphy", "age": "16" }
+{ "id": 2, "name": "Mann", "age": "45" }
+{ "id": 3, "name": "Brand", "age": "35" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index db612826e0..4e94791e2d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -301,6 +301,25 @@
</compilation-unit>
</test-case>
<!-- Parquet Tests End -->
+ <!-- Deltalake Tests Start -->
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/deltalake-multiple-file-read">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/deltalake-multiple-file-read</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/deltalake-empty">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/deltalake-empty</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/deltalake-modified-data">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/deltalake-modified-data</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="common/avro/avro-types/avro-map">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 3694bb2fb5..985a4f699d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -97,6 +97,7 @@ public enum ErrorCode implements IError {
UNSUPPORTED_COLUMN_TYPE(67),
INVALID_KEY_TYPE(68),
FAILED_TO_READ_KEY(69),
+ INVALID_DELTA_PARAMETER(70),
UNSUPPORTED_JRE(100),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 4b7da0c1ec..b5c06e3aa2 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -41,6 +41,7 @@
1009 = A returning expression cannot contain dataset access
37,1091 = Type mismatch: expected value of type %1$s, but got the value of type %2$s
51 = Incomparable input types: %1$s and %2$s
+70 = Table Type 'delta' supports parquet file formats only.
# Data errors
6 = Invalid format for %1$s in %2$s
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index 7f3c7ecea1..fdacc685c2 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -573,6 +573,11 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-standalone_2.12</artifactId>
+ <version>3.0.0</version>
+ </dependency>
</dependencies>
<!-- apply patch for HADOOP-17225 to workaround CVE-2019-10172 -->
<repositories>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
new file mode 100644
index 0000000000..f5a2cd5820
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.aws.delta;
+
+import static org.apache.asterix.external.util.ExternalDataUtils.prepareDeltaTableFormat;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
+import org.apache.asterix.external.input.record.reader.aws.parquet.AwsS3ParquetReaderFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.aws.s3.S3Constants;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.application.IServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+
+public class AwsS3DeltaReaderFactory extends AwsS3ParquetReaderFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
+ IWarningCollector warningCollector, IExternalFilterEvaluatorFactory filterEvaluatorFactory)
+ throws AlgebricksException, HyracksDataException {
+
+ Configuration conf = new Configuration();
+ conf.set(S3Constants.HADOOP_ACCESS_KEY_ID, configuration.get(S3Constants.ACCESS_KEY_ID_FIELD_NAME));
+ conf.set(S3Constants.HADOOP_SECRET_ACCESS_KEY, configuration.get(S3Constants.SECRET_ACCESS_KEY_FIELD_NAME));
+ conf.set(S3Constants.HADOOP_REGION, configuration.get(S3Constants.REGION_FIELD_NAME));
+ String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
+ if (serviceEndpoint != null) {
+ conf.set(S3Constants.HADOOP_SERVICE_END_POINT, serviceEndpoint);
+ }
+ String tableMetadataPath = S3Constants.HADOOP_S3_PROTOCOL + "://"
+ + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
+ + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
+
+ prepareDeltaTableFormat(configuration, conf, tableMetadataPath);
+ super.configure(serviceCtx, configuration, warningCollector, filterEvaluatorFactory);
+ }
+
+ @Override
+ public Set<String> getReaderSupportedFormats() {
+ return Collections.singleton(ExternalDataConstants.FORMAT_DELTA);
+ }
+
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
index f35dce4aa5..0360c15fb1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.external.provider;
+import static org.apache.asterix.external.util.ExternalDataUtils.isDeltaTable;
+
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
@@ -121,6 +123,10 @@ public class DatasourceFactoryProvider {
if (factories.containsKey(adaptorName)) {
Map<String, Class<?>> formatClassMap = factories.get(adaptorName);
String format = configuration.get(ExternalDataConstants.KEY_FORMAT);
+ if (isDeltaTable(configuration)) {
+ format = configuration.get(ExternalDataConstants.TABLE_FORMAT);
+ return getInstance(formatClassMap.getOrDefault(format, formatClassMap.get(DEFAULT_FORMAT)));
+ }
return getInstance(formatClassMap.getOrDefault(format, formatClassMap.get(DEFAULT_FORMAT)));
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 3139be7f70..d8f89c22d7 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -119,7 +119,7 @@ public class ExternalDataConstants {
// a string representing the format of the record (for adapters which produces records with additional information like pk or metadata)
public static final String KEY_RECORD_FORMAT = "record-format";
public static final String TABLE_FORMAT = "table-format";
- public static final String ICEBERG_METADATA_LOCATION = "metadata-path";
+ public static final String TABLE_METADATA_LOCATION = "metadata-path";
public static final int SUPPORTED_ICEBERG_FORMAT_VERSION = 1;
public static final String KEY_META_TYPE_NAME = "meta-type-name";
public static final String KEY_ADAPTER_NAME = "adapter-name";
@@ -205,6 +205,7 @@ public class ExternalDataConstants {
public static final String FORMAT_TSV = "tsv";
public static final String FORMAT_PARQUET = "parquet";
public static final String FORMAT_APACHE_ICEBERG = "apache-iceberg";
+ public static final String FORMAT_DELTA = "delta";
public static final Set<String> ALL_FORMATS;
public static final Set<String> TEXTUAL_FORMATS;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 104f301071..8d332610e8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -108,6 +108,10 @@ import org.apache.iceberg.Table;
import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.io.CloseableIterable;
+import io.delta.standalone.DeltaLog;
+import io.delta.standalone.Snapshot;
+import io.delta.standalone.actions.AddFile;
+
public class ExternalDataUtils {
private static final Map<ATypeTag, IValueParserFactory> valueParserFactoryMap = new EnumMap<>(ATypeTag.class);
private static final int DEFAULT_MAX_ARGUMENT_SZ = 1024 * 1024;
@@ -469,6 +473,10 @@ public class ExternalDataUtils {
}
if (configuration.containsKey(ExternalDataConstants.TABLE_FORMAT)) {
+ if (configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_DELTA)) {
+ configuration.put(ExternalDataConstants.KEY_PARSER, ExternalDataConstants.FORMAT_NOOP);
+ configuration.put(ExternalDataConstants.KEY_FORMAT, ExternalDataConstants.FORMAT_PARQUET);
+ }
prepareTableFormat(configuration);
}
}
@@ -478,68 +486,96 @@ public class ExternalDataUtils {
*
* @param configuration external data configuration
*/
- public static void prepareTableFormat(Map<String, String> configuration) throws AlgebricksException {
- // Apache Iceberg table format
- if (configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_APACHE_ICEBERG)) {
- Configuration conf = new Configuration();
-
- String metadata_path = configuration.get(ExternalDataConstants.ICEBERG_METADATA_LOCATION);
-
- // If the table is in S3
- if (configuration.get(ExternalDataConstants.KEY_READER)
- .equals(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3)) {
-
- conf.set(S3Constants.HADOOP_ACCESS_KEY_ID, configuration.get(S3Constants.ACCESS_KEY_ID_FIELD_NAME));
- conf.set(S3Constants.HADOOP_SECRET_ACCESS_KEY,
- configuration.get(S3Constants.SECRET_ACCESS_KEY_FIELD_NAME));
- metadata_path = S3Constants.HADOOP_S3_PROTOCOL + "://"
- + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
- + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
- } else if (configuration.get(ExternalDataConstants.KEY_READER).equals(ExternalDataConstants.READER_HDFS)) {
- conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_URI,
- configuration.get(ExternalDataConstants.KEY_HDFS_URL));
- metadata_path = configuration.get(ExternalDataConstants.KEY_HDFS_URL) + '/' + metadata_path;
- }
-
- HadoopTables tables = new HadoopTables(conf);
+ public static boolean isDeltaTable(Map<String, String> configuration) {
+ return configuration.containsKey(ExternalDataConstants.TABLE_FORMAT)
+ && configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_DELTA);
+ }
- Table icebergTable = tables.load(metadata_path);
+ public static void validateDeltaTableProperties(Map<String, String> configuration) throws CompilationException {
+ if (!(configuration.get(ExternalDataConstants.KEY_FORMAT) == null
+ || configuration.get(ExternalDataConstants.KEY_FORMAT).equals(ExternalDataConstants.FORMAT_PARQUET))) {
+ throw new CompilationException(ErrorCode.INVALID_DELTA_PARAMETER);
+ }
+ }
- if (icebergTable instanceof BaseTable) {
- BaseTable baseTable = (BaseTable) icebergTable;
+ public static void prepareDeltaTableFormat(Map<String, String> configuration, Configuration conf,
+ String tableMetadataPath) {
+ DeltaLog deltaLog = DeltaLog.forTable(conf, tableMetadataPath);
+ Snapshot snapshot = deltaLog.snapshot();
+ List<AddFile> dataFiles = snapshot.getAllFiles();
+ StringBuilder builder = new StringBuilder();
+ for (AddFile batchFile : dataFiles) {
+ builder.append(",");
+ String path = batchFile.getPath();
+ builder.append(tableMetadataPath).append('/').append(path);
+ }
+ if (builder.length() > 0) {
+ builder.deleteCharAt(0);
+ }
+ configuration.put(ExternalDataConstants.KEY_PATH, builder.toString());
+ }
- if (baseTable.operations().current()
- .formatVersion() != ExternalDataConstants.SUPPORTED_ICEBERG_FORMAT_VERSION) {
- throw new AsterixException(ErrorCode.UNSUPPORTED_ICEBERG_FORMAT_VERSION,
- "AsterixDB only supports Iceberg version up to "
- + ExternalDataConstants.SUPPORTED_ICEBERG_FORMAT_VERSION);
- }
+ public static void prepareIcebergTableFormat(Map<String, String> configuration, Configuration conf,
+ String tableMetadataPath) throws AlgebricksException {
+ HadoopTables tables = new HadoopTables(conf);
+ Table icebergTable = tables.load(tableMetadataPath);
- try (CloseableIterable<FileScanTask> fileScanTasks = baseTable.newScan().planFiles()) {
+ if (icebergTable instanceof BaseTable) {
+ BaseTable baseTable = (BaseTable) icebergTable;
- StringBuilder builder = new StringBuilder();
+ if (baseTable.operations().current()
+ .formatVersion() != ExternalDataConstants.SUPPORTED_ICEBERG_FORMAT_VERSION) {
+ throw new AsterixException(ErrorCode.UNSUPPORTED_ICEBERG_FORMAT_VERSION,
+ "AsterixDB only supports Iceberg version up to "
+ + ExternalDataConstants.SUPPORTED_ICEBERG_FORMAT_VERSION);
+ }
- for (FileScanTask task : fileScanTasks) {
- builder.append(",");
- String path = task.file().path().toString();
- builder.append(path);
- }
+ try (CloseableIterable<FileScanTask> fileScanTasks = baseTable.newScan().planFiles()) {
- if (builder.length() > 0) {
- builder.deleteCharAt(0);
- }
+ StringBuilder builder = new StringBuilder();
- configuration.put(ExternalDataConstants.KEY_PATH, builder.toString());
+ for (FileScanTask task : fileScanTasks) {
+ builder.append(",");
+ String path = task.file().path().toString();
+ builder.append(path);
+ }
- } catch (IOException e) {
- throw new AsterixException(ErrorCode.ERROR_READING_ICEBERG_METADATA, e);
+ if (builder.length() > 0) {
+ builder.deleteCharAt(0);
}
- } else {
- throw new AsterixException(ErrorCode.UNSUPPORTED_ICEBERG_TABLE,
- "Invalid iceberg base table. Please remove metadata specifiers");
+ configuration.put(ExternalDataConstants.KEY_PATH, builder.toString());
+
+ } catch (IOException e) {
+ throw new AsterixException(ErrorCode.ERROR_READING_ICEBERG_METADATA, e);
}
+ } else {
+ throw new AsterixException(ErrorCode.UNSUPPORTED_ICEBERG_TABLE,
+ "Invalid iceberg base table. Please remove metadata specifiers");
+ }
+ }
+
+ public static void prepareTableFormat(Map<String, String> configuration) throws AlgebricksException {
+ Configuration conf = new Configuration();
+ String tableMetadataPath = configuration.get(ExternalDataConstants.TABLE_METADATA_LOCATION);
+
+ // If the table is in S3
+ if (configuration.get(ExternalDataConstants.KEY_READER).equals(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3)) {
+
+ conf.set(S3Constants.HADOOP_ACCESS_KEY_ID, configuration.get(S3Constants.ACCESS_KEY_ID_FIELD_NAME));
+ conf.set(S3Constants.HADOOP_SECRET_ACCESS_KEY, configuration.get(S3Constants.SECRET_ACCESS_KEY_FIELD_NAME));
+ tableMetadataPath = S3Constants.HADOOP_S3_PROTOCOL + "://"
+ + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
+ + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
+ } else if (configuration.get(ExternalDataConstants.KEY_READER).equals(ExternalDataConstants.READER_HDFS)) {
+ conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_URI,
+ configuration.get(ExternalDataConstants.KEY_HDFS_URL));
+ tableMetadataPath = configuration.get(ExternalDataConstants.KEY_HDFS_URL) + '/' + tableMetadataPath;
+ }
+ // Apache Iceberg table format
+ if (configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_APACHE_ICEBERG)) {
+ prepareIcebergTableFormat(configuration, conf, tableMetadataPath);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index 6a16913258..891d7f3bfa 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -23,6 +23,8 @@ import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_I
import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED;
import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
+import static org.apache.asterix.external.util.ExternalDataUtils.isDeltaTable;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
import static org.apache.asterix.external.util.aws.s3.S3Constants.ACCESS_KEY_ID_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
@@ -272,9 +274,11 @@ public class S3Utils {
*/
public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc,
IWarningCollector collector) throws CompilationException {
-
+ if (isDeltaTable(configuration)) {
+ validateDeltaTableProperties(configuration);
+ }
// check if the format property is present
- if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+ else if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
}
diff --git a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
index 88f3fcb440..2c15b5a844 100644
--- a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
+++ b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
@@ -22,6 +22,7 @@ org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory
org.apache.asterix.external.input.record.reader.http.HttpServerRecordReaderFactory
org.apache.asterix.external.input.record.reader.aws.AwsS3ReaderFactory
org.apache.asterix.external.input.record.reader.aws.parquet.AwsS3ParquetReaderFactory
+org.apache.asterix.external.input.record.reader.aws.delta.AwsS3DeltaReaderFactory
org.apache.asterix.external.input.record.reader.gcs.GCSReaderFactory
org.apache.asterix.external.input.record.reader.azure.blob.AzureBlobReaderFactory
org.apache.asterix.external.input.record.reader.azure.datalake.AzureDataLakeReaderFactory
(asterixdb) 04/09: Merge branch 'gerrit/neo' into 'gerrit/trinity'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e49261f38ce9fadf0e86ade7edf81b0aed8850db
Merge: 5abe3c520b 041571d17e
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Wed Oct 23 19:26:26 2024 -0400
Merge branch 'gerrit/neo' into 'gerrit/trinity'
Ext-ref: MB-63696
Change-Id: I490626293e255e0193ca425900bbd24d0c25ec47
.../optimizerts/queries/query-ASTERIXDB-3512.sqlpp | 44 ++++++++++++++++++++++
.../optimizerts/results/query-ASTERIXDB-3512.plan | 28 ++++++++++++++
.../query-ASTERIXDB-3512.1.ddl.sqlpp | 34 +++++++++++++++++
.../query-ASTERIXDB-3512.2.update.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.query.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 ++++++--
8 files changed, 200 insertions(+), 4 deletions(-)
(asterixdb) 02/09: Merge branch 'gerrit/neo' into 'gerrit/trinity'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5abe3c520ba0a12ee82634fe4ac868af6b6ee58b
Merge: 9f5a8d08b9 6a8121408b
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Mon Oct 7 11:08:33 2024 -0400
Merge branch 'gerrit/neo' into 'gerrit/trinity'
Ext-ref: MB-63525
Change-Id: Ifde754e1e3d932cc1fb6107c55d9e3c93acbd0ed
.../ASTERIXDB-3502/ASTERIXDB-3502.01.ddl.sqlpp | 61 +++++++++++++++
.../ASTERIXDB-3502/ASTERIXDB-3502.02.update.sqlpp | 26 +++++++
.../ASTERIXDB-3502/ASTERIXDB-3502.03.query.sqlpp | 39 ++++++++++
.../join/ASTERIXDB-3502/ASTERIXDB-3502.03.adm | 0
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 ++
.../asterix/lang/common/util/FunctionUtil.java | 17 +---
.../lang/expression/CommutativeEqualsTest.java | 90 ++++++++++++++++++++--
7 files changed, 217 insertions(+), 21 deletions(-)
(asterixdb) 03/09: [ASTERIXDB-3512][COMP] Error with query if secondary indexes created
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 041571d17ed71620a42fc8ec688097a99e160088
Author: Peeyush Gupta <pe...@couchbase.com>
AuthorDate: Wed Oct 9 15:00:22 2024 -0700
[ASTERIXDB-3512][COMP] Error with query if secondary indexes created
- user model changes: no
- storage format changes: no
- interface changes: no
Ext-ref: MB-63696
Change-Id: Ie6b97d5436fe6f76a92548f2aa756ef5d6d20e49
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18928
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../optimizerts/queries/query-ASTERIXDB-3512.sqlpp | 44 ++++++++++++++++++++++
.../optimizerts/results/query-ASTERIXDB-3512.plan | 28 ++++++++++++++
.../query-ASTERIXDB-3512.1.ddl.sqlpp | 34 +++++++++++++++++
.../query-ASTERIXDB-3512.2.update.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.query.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 ++++++--
8 files changed, 200 insertions(+), 4 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/query-ASTERIXDB-3512.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/query-ASTERIXDB-3512.sqlpp
new file mode 100644
index 0000000000..d51df3bc07
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/query-ASTERIXDB-3512.sqlpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type dt1 as {id: int};
+
+create dataset collection1(dt1) PRIMARY KEY id;
+create dataset collection2(dt1) PRIMARY KEY id;
+
+CREATE INDEX idx1 ON collection2 (date: STRING, oid: STRING, sym: STRING, reg: BIGINT);
+
+with tmp1 AS (
+ SELECT S.sym
+ FROM collection1 AS S
+ WHERE S.date < print_date(current_date(), 'YYYY-MM-DD')),
+ tmp2 AS (
+ SELECT O.sym
+ FROM collection2 AS O
+ WHERE O.date < print_date(current_date(), 'YYYY-MM-DD'))
+SELECT sl1.sym
+FROM tmp1 AS sl1
+UNION ALL
+SELECT sl2.sym
+FROM tmp2 as sl2;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-ASTERIXDB-3512.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-ASTERIXDB-3512.plan
new file mode 100644
index 0000000000..9160a431c1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/query-ASTERIXDB-3512.plan
@@ -0,0 +1,28 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- UNION_ALL |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.collection1) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.collection2.collection2) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$117(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH (test.collection2.idx1) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.1.ddl.sqlpp
new file mode 100644
index 0000000000..980d4d459f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.1.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3512
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type dt1 as {id: int};
+
+create dataset collection1(dt1) PRIMARY KEY id;
+create dataset collection2(dt1) PRIMARY KEY id;
+
+CREATE INDEX idx1 ON collection2 (date: STRING, oid: STRING, sym: STRING, reg: BIGINT);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.2.update.sqlpp
new file mode 100644
index 0000000000..41879a7324
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.2.update.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+upsert into collection1
+[
+ {
+ "id":1,
+ "sym":10,
+ "date": "2024-01-01"
+ }
+];
+
+upsert into collection2
+[
+ {
+ "id":1,
+ "sym":10,
+ "date": "2024-01-01"
+ }
+];
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.query.sqlpp
new file mode 100644
index 0000000000..60f91ada79
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.query.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3512
+ */
+
+use test;
+
+with tmp1 AS (
+ SELECT S.sym
+ FROM collection1 AS S
+ WHERE S.date < print_date(current_date(), 'YYYY-MM-DD')),
+ tmp2 AS (
+ SELECT O.sym
+ FROM collection2 AS O
+ WHERE O.date < print_date(current_date(), 'YYYY-MM-DD'))
+SELECT sl1.sym
+FROM tmp1 AS sl1
+UNION ALL
+SELECT sl2.sym
+FROM tmp2 as sl2;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.adm
new file mode 100644
index 0000000000..9ea757d3be
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3512/query-ASTERIXDB-3512.3.adm
@@ -0,0 +1,2 @@
+{ "sym": 10 }
+{ "sym": 10 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index 717700f96d..09b9e2360e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -7310,6 +7310,11 @@
<output-dir compare="Text">query-ASTERIXDB-3316</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-3512">
+ <output-dir compare="Text">query-ASTERIXDB-3512</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="misc">
<compilation-unit name="remove_listify">
<output-dir compare="Text">remove_listify</output-dir>
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
index 956f0b7b67..84346fdbcd 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -42,6 +42,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogi
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
@@ -163,9 +164,15 @@ public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule
if (!canRemoveOperator(op, opInSubplan, parentOp)) {
break;
}
- op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
- opRef.setValue(op);
- assignVarsSetForThisOp = removeAssignVarFromConsideration(opRef);
+ if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+ op = new EmptyTupleSourceOperator();
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ opRef.setValue(op);
+ } else {
+ op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ opRef.setValue(op);
+ assignVarsSetForThisOp = removeAssignVarFromConsideration(opRef);
+ }
isTransformed = true;
}
@@ -445,7 +452,7 @@ public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule
private static boolean canRemoveOperator(ILogicalOperator op, boolean opInsideSubplan, ILogicalOperator parentOp) {
LogicalOperatorTag opTag = op.getOperatorTag();
- if (opTag == LogicalOperatorTag.AGGREGATE || opTag == LogicalOperatorTag.UNIONALL) {
+ if ((opTag == LogicalOperatorTag.AGGREGATE && opInsideSubplan) || opTag == LogicalOperatorTag.UNIONALL) {
return false;
}
if (!opInsideSubplan) {
(asterixdb) 06/09: Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8e36160d5b07c0cdeb35f8e4264bac7ba0edb1c7
Merge: 67bbe28bf3 e49261f38c
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Thu Oct 24 12:29:46 2024 -0400
Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
Ext-ref: MB-63819
Change-Id: Ib5168f1c571a673b52c18a13d787b6be11465e3b
.../optimizerts/queries/query-ASTERIXDB-3512.sqlpp | 44 ++++++++++++++++++++++
.../optimizerts/results/query-ASTERIXDB-3512.plan | 28 ++++++++++++++
.../query-ASTERIXDB-3512.1.ddl.sqlpp | 34 +++++++++++++++++
.../query-ASTERIXDB-3512.2.update.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.query.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 ++++++--
8 files changed, 200 insertions(+), 4 deletions(-)
(asterixdb) 07/09: Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1723d0c83525f03308d1d117079fa5298c7f8438
Merge: 3164da5f58 8e36160d5b
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Thu Oct 24 19:42:17 2024 +0000
Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish
.../optimizerts/queries/query-ASTERIXDB-3512.sqlpp | 44 ++++++++++++++++++++++
.../optimizerts/results/query-ASTERIXDB-3512.plan | 28 ++++++++++++++
.../query-ASTERIXDB-3512.1.ddl.sqlpp | 34 +++++++++++++++++
.../query-ASTERIXDB-3512.2.update.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.query.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 ++++++--
8 files changed, 200 insertions(+), 4 deletions(-)
(asterixdb) 09/09: Merge branch 'gerrit/goldfish' into 'master'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2f061939c01b05735a6a51f787a8d2a2f50585d0
Merge: 7156522359 7e7960c090
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Oct 25 11:51:32 2024 -0400
Merge branch 'gerrit/goldfish' into 'master'
Change-Id: I088ea2f4a99970b30cdd15f723f05e2db1da9277
.../optimizerts/queries/query-ASTERIXDB-3512.sqlpp | 44 ++++++++++++++++++++++
.../optimizerts/results/query-ASTERIXDB-3512.plan | 28 ++++++++++++++
.../query-ASTERIXDB-3512.1.ddl.sqlpp | 34 +++++++++++++++++
.../query-ASTERIXDB-3512.2.update.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.query.sqlpp | 38 +++++++++++++++++++
.../query-ASTERIXDB-3512.3.adm | 2 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++
.../runtimets/testsuite_external_dataset_s3.xml | 4 +-
.../asterix/common/exceptions/ErrorCode.java | 4 +-
.../src/main/resources/asx_errormsg/en.properties | 4 +-
.../rules/RemoveUnusedAssignAndAggregateRule.java | 15 ++++++--
11 files changed, 206 insertions(+), 10 deletions(-)
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 949c0a3584,2c7af49eb0..aadc42bc82
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@@ -105,29 -80,10 +105,29 @@@
<compilation-unit name="supported-adapter-format-compression">
<output-dir compare="Text">supported-adapter-format-compression</output-dir>
<expected-error>ASX1188: Unsupported writing adapter 'AZUREBLOB'. Supported adapters: [gcs, localfs, s3]</expected-error>
- <expected-error>ASX1189: Unsupported writing format 'csv'. Supported formats: [json]</expected-error>
- <expected-error>ASX1096: Unknown compression scheme rar. Supported schemes are [gzip]</expected-error>
+ <expected-error>ASX1189: Unsupported writing format 'csv'. Supported formats: [json, parquet]</expected-error>
+ <expected-error>ASX1202: Unsupported compression scheme rar. Supported schemes for json are [gzip]</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="parquet-error-checks">
+ <output-dir compare="Text">parquet-error-checks</output-dir>
+ <expected-error>ASX1079: Compilation error: TYPE() Expression is required for parquet format</expected-error>
+ <expected-error>ASX0037: Type mismatch: expected value of type integer, but got the value of type BINARY</expected-error>
+ <expected-error>HYR0132: Extra field in the result, field 'second' does not exist at 'nested' in the schema</expected-error>
+ <expected-error>HYR0131: Result does not follow the schema, group type expected but found primitive type at 'nested'</expected-error>
+ <expected-error>HYR0131: Result does not follow the schema, primitive type expected but found group type at 'name'</expected-error>
- <expected-error>ASX1201: Storage units expected for the field 'row-group-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
- <expected-error>ASX1201: Storage units expected for the field 'page-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
++ <expected-error>ASX1206: Storage units expected for the field 'row-group-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
++ <expected-error>ASX1206: Storage units expected for the field 'page-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
+ <expected-error>ASX1202: Unsupported compression scheme rar. Supported schemes for parquet are [gzip, snappy, zstd]</expected-error>
+ <expected-error>ASX1001: Syntax error</expected-error>
+ <expected-error>ASX1204: 'binary' type not supported in parquet format</expected-error>
+ <expected-error>ASX1205: Invalid Parquet Writer Version provided '3'. Supported values: [1, 2]</expected-error>
+ <expected-error>ASX1001: Syntax error</expected-error>
+ <expected-error>ASX1001: Syntax error</expected-error>
+ <expected-error>ASX1001: Syntax error</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="copy-to/negative">
<compilation-unit name="empty-over">
<output-dir compare="Text">empty-over</output-dir>
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 7c89bc91e6,091fb6774f..ea7cc3bae9
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@@ -303,12 -303,8 +303,12 @@@ public enum ErrorCode implements IErro
DUPLICATE_FIELD_IN_PRIMARY_KEY(1198),
INCOMPATIBLE_FIELDS_IN_PRIMARY_KEY(1199),
PREFIX_SHOULD_NOT_START_WITH_SLASH(1200),
- ILLEGAL_SIZE_PROVIDED(1201),
+ INVALID_DELTA_TABLE_FORMAT(1201),
-
+ UNSUPPORTED_WRITER_COMPRESSION_SCHEME(1202),
+ INVALID_PARQUET_SCHEMA(1203),
+ TYPE_UNSUPPORTED_PARQUET_WRITE(1204),
+ INVALID_PARQUET_WRITER_VERSION(1205),
- INVALID_DELTA_TABLE_FORMAT(1206),
++ ILLEGAL_SIZE_PROVIDED(1206),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
UTIL_DATAFLOW_UTILS_TUPLE_TOO_LARGE(3002),
diff --cc asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index d1e01f66f9,8e3ccffff3..3e2f124b27
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@@ -305,12 -305,7 +305,12 @@@
1198 = Duplicate field '%1$s' in primary key
1199 = Fields '%1$s' and '%2$s' are incompatible for primary key
1200 = Prefix should not start with "/". Prefix: '%1$s'
- 1201 = Storage units expected for the field '%1$s' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided '%2$s'
+ 1201 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was provided.
+1202 = Unsupported compression scheme %1$s. Supported schemes for %2$s are %3$s
+1203 = Invalid schema provided: '%1$s'
+1204 = '%1$s' type not supported in parquet format
+1205 = Invalid Parquet Writer Version provided '%1$s'. Supported values: %2$s
- 1206 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was provided.
++1206 = Storage units expected for the field '%1$s' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided '%2$s'
# Feed Errors
3001 = Illegal state.
(asterixdb) 08/09: [ASTERIXDB-3503][EXT] Deltalake invalid type error code change
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7e7960c090942a06046b04163c9a186463998c1c
Author: ayush.tripathi <ay...@couchbase.com>
AuthorDate: Fri Oct 11 15:35:20 2024 +0530
[ASTERIXDB-3503][EXT] Deltalake invalid type error code change
- user model changes: no
- storage format changes: no
- interface changes: no
Ext-ref: MB-63840
Change-Id: Ic0c12e32b85987cbc14d28e83fb2be644b922590
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18952
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ayush Tripathi <Ay...@couchbase.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Murtadha Hubail <mh...@apache.org>
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19008
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../common/deltalake-empty/deltalake-empty.00.ddl.sqlpp | 2 +-
.../deltalake-invalid-file-format.00.ddl.sqlpp} | 7 ++++---
.../deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp | 2 +-
.../deltalake-multiple_file_read.00.ddl.sqlpp | 2 +-
.../src/test/resources/runtimets/testsuite_external_dataset_s3.xml | 7 +++++++
.../main/java/org/apache/asterix/common/exceptions/ErrorCode.java | 2 +-
.../asterix-common/src/main/resources/asx_errormsg/en.properties | 2 +-
.../java/org/apache/asterix/external/util/ExternalDataUtils.java | 5 +++--
8 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
index f21f0131fc..2e77bebfdc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-empty/deltalake-empty.00.ddl.sqlpp
@@ -26,7 +26,7 @@
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter%
(
%template%,
("container"="playground"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
index 8adab7721c..98799e1be8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-invalid-file-format/deltalake-invalid-file-format.00.ddl.sqlpp
@@ -26,10 +26,11 @@ DROP DATAVERSE test IF EXISTS;
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter%
(
%template%,
("container"="playground"),
- ("definition"="delta-data/modified_delta_table"),
- ("table-format" = "delta")
+ ("definition"="delta-data/my-table-empty"),
+ ("table-format" = "delta"),
+ ("format" = "avro")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
index 8adab7721c..9238da6d8d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-modified-data/deltalake-modified-data.00.ddl.sqlpp
@@ -26,7 +26,7 @@ DROP DATAVERSE test IF EXISTS;
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter%
(
%template%,
("container"="playground"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
index 61c5c3ac38..9aa0380e81 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-multiple-file-read/deltalake-multiple_file_read.00.ddl.sqlpp
@@ -26,7 +26,7 @@ DROP DATAVERSE test IF EXISTS;
CREATE TYPE DeltalakeTableType as {
};
- CREATE EXTERNAL DATASET DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter%
(
%template%,
("container"="playground"),
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 4e94791e2d..2c7af49eb0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -320,6 +320,13 @@
<output-dir compare="Text">common/deltalake-modified-data</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/deltalake-invalid-file-format">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">none</output-dir>
+ <expected-error>Supported file format for 'delta' tables is 'parquet', but 'avro' was provided.</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="common/avro/avro-types/avro-map">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 985a4f699d..091fb6774f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -97,7 +97,6 @@ public enum ErrorCode implements IError {
UNSUPPORTED_COLUMN_TYPE(67),
INVALID_KEY_TYPE(68),
FAILED_TO_READ_KEY(69),
- INVALID_DELTA_PARAMETER(70),
UNSUPPORTED_JRE(100),
@@ -304,6 +303,7 @@ public enum ErrorCode implements IError {
DUPLICATE_FIELD_IN_PRIMARY_KEY(1198),
INCOMPATIBLE_FIELDS_IN_PRIMARY_KEY(1199),
PREFIX_SHOULD_NOT_START_WITH_SLASH(1200),
+ INVALID_DELTA_TABLE_FORMAT(1201),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index b5c06e3aa2..8e3ccffff3 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -41,7 +41,6 @@
1009 = A returning expression cannot contain dataset access
37,1091 = Type mismatch: expected value of type %1$s, but got the value of type %2$s
51 = Incomparable input types: %1$s and %2$s
-70 = Table Type 'delta' supports parquet file formats only.
# Data errors
6 = Invalid format for %1$s in %2$s
@@ -306,6 +305,7 @@
1198 = Duplicate field '%1$s' in primary key
1199 = Fields '%1$s' and '%2$s' are incompatible for primary key
1200 = Prefix should not start with "/". Prefix: '%1$s'
+1201 = Supported file format for 'delta' tables is 'parquet', but '%1$s' was provided.
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8d332610e8..b4314b99f0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -473,7 +473,7 @@ public class ExternalDataUtils {
}
if (configuration.containsKey(ExternalDataConstants.TABLE_FORMAT)) {
- if (configuration.get(ExternalDataConstants.TABLE_FORMAT).equals(ExternalDataConstants.FORMAT_DELTA)) {
+ if (isDeltaTable(configuration)) {
configuration.put(ExternalDataConstants.KEY_PARSER, ExternalDataConstants.FORMAT_NOOP);
configuration.put(ExternalDataConstants.KEY_FORMAT, ExternalDataConstants.FORMAT_PARQUET);
}
@@ -494,7 +494,8 @@ public class ExternalDataUtils {
public static void validateDeltaTableProperties(Map<String, String> configuration) throws CompilationException {
if (!(configuration.get(ExternalDataConstants.KEY_FORMAT) == null
|| configuration.get(ExternalDataConstants.KEY_FORMAT).equals(ExternalDataConstants.FORMAT_PARQUET))) {
- throw new CompilationException(ErrorCode.INVALID_DELTA_PARAMETER);
+ throw new CompilationException(ErrorCode.INVALID_DELTA_TABLE_FORMAT,
+ configuration.get(ExternalDataConstants.KEY_FORMAT));
}
}