You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by wy...@apache.org on 2021/09/03 21:39:53 UTC
[asterixdb] branch master updated: [ASTERIXDB-2933][COMP][EXT]
Pushdowns Part3: Enable pushdown
This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new c84b073 [ASTERIXDB-2933][COMP][EXT] Pushdowns Part3: Enable pushdown
c84b073 is described below
commit c84b073159f0f1de23abb6543238ea0f794dbb6a
Author: Wail Alkowaileet <wa...@gmail.com>
AuthorDate: Thu Sep 2 21:56:32 2021 -0700
[ASTERIXDB-2933][COMP][EXT] Pushdowns Part3: Enable pushdown
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Enable more pushdown when reading Parquet files
- Issue warning on type-mismatch between the expected schema
and the actual Parquet file's schema
- Add testcases
Change-Id: I411f0f5b98c1bc1e174473344cda666f96151e4a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13064
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Wael Alkowaileet <wa...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../asterix/optimizer/base/RuleCollections.java | 5 +-
.../PushFieldAccessToExternalDataScanRule.java | 228 ---------------------
.../data/hdfs/parquet/heterogeneous_1.json | 3 +
.../data/hdfs/parquet/heterogeneous_2.json | 3 +
.../external_dataset/ExternalDatasetTestUtils.java | 2 +
.../array-access-pushdown.01.ddl.sqlpp} | 22 +-
.../array-access-pushdown.02.query.sqlpp} | 7 +-
.../array-access-pushdown.03.query.sqlpp} | 6 +-
.../array-access-pushdown.04.query.sqlpp} | 7 +-
.../array-access-pushdown.05.query.sqlpp} | 6 +-
.../array-access-pushdown.06.query.sqlpp} | 8 +-
.../array-access-pushdown.07.query.sqlpp} | 7 +-
.../array-access-pushdown.08.query.sqlpp} | 8 +-
.../array-access-pushdown.09.query.sqlpp} | 7 +-
.../array-access-pushdown.10.query.sqlpp} | 9 +-
.../array-access-pushdown.11.query.sqlpp} | 8 +-
.../array-access-pushdown.12.query.sqlpp} | 9 +-
.../array-access-pushdown.13.query.sqlpp} | 8 +-
.../array-access-pushdown.14.query.sqlpp} | 9 +-
.../array-access-pushdown.15.query.sqlpp} | 8 +-
.../array-access-pushdown.16.query.sqlpp} | 9 +-
.../array-access-pushdown.17.query.sqlpp} | 8 +-
.../array-access-pushdown.18.query.sqlpp} | 7 +-
.../array-access-pushdown.19.query.sqlpp} | 6 +-
.../array-access-pushdown.20.query.sqlpp} | 7 +-
.../array-access-pushdown.21.query.sqlpp} | 6 +-
.../field-access-pushdown.01.ddl.sqlpp} | 2 +-
.../field-access-pushdown.02.query.sqlpp} | 1 +
.../field-access-pushdown.03.query.sqlpp} | 0
.../field-access-pushdown.04.query.sqlpp} | 1 +
.../field-access-pushdown.05.query.sqlpp} | 0
.../field-access-pushdown.06.query.sqlpp} | 1 +
.../field-access-pushdown.07.query.sqlpp} | 0
.../field-access-pushdown.08.query.sqlpp} | 1 +
.../field-access-pushdown.09.query.sqlpp} | 0
.../field-access-pushdown.10.query.sqlpp} | 1 +
.../field-access-pushdown.11.query.sqlpp} | 0
.../field-access-pushdown.12.query.sqlpp} | 1 +
.../field-access-pushdown.13.query.sqlpp} | 0
.../field-access-pushdown.14.query.sqlpp} | 3 +-
.../field-access-pushdown.15.query.sqlpp} | 2 +-
.../field-access-pushdown.16.query.sqlpp} | 3 +-
.../field-access-pushdown.17.query.sqlpp} | 2 +-
.../field-access-pushdown.18.query.sqlpp} | 10 +-
.../field-access-pushdown.19.query.sqlpp} | 9 +-
.../field-access-pushdown.20.query.sqlpp} | 10 +-
.../field-access-pushdown.21.query.sqlpp} | 9 +-
.../field-access-pushdown.22.query.sqlpp} | 0
.../heterogeneous-access-pushdown.1.ddl.sqlpp} | 22 +-
.../heterogeneous-access-pushdown.2.query.sqlpp} | 18 +-
.../heterogeneous-access-pushdown.3.query.sqlpp} | 17 +-
.../heterogeneous-access-pushdown.4.query.sqlpp} | 14 +-
.../heterogeneous-access-pushdown.5.query.sqlpp} | 13 +-
.../missing-fields/missing-fields.2.query.sqlpp | 1 +
.../object-concat/object-concat.2.query.sqlpp | 2 +
.../object-concat/object-concat.3.query.sqlpp | 1 +
.../object-concat/object-concat.4.query.sqlpp | 2 +
.../object-concat/object-concat.5.query.sqlpp | 2 +
.../pushdown-plans.01.ddl.sqlpp} | 26 +--
.../pushdown-plans.02.query.sqlpp} | 11 +-
.../pushdown-plans.03.query.sqlpp} | 11 +-
.../pushdown-plans.04.query.sqlpp} | 13 +-
.../pushdown-plans.05.query.sqlpp} | 13 +-
.../pushdown-plans.06.query.sqlpp} | 13 +-
.../pushdown-plans.07.query.sqlpp} | 19 +-
.../select-all-fields.2.query.sqlpp | 2 +
.../select-count-one-field.2.query.sqlpp | 4 +-
.../select-count-one-field.3.query.sqlpp | 2 +
.../select-count-one-field.4.query.sqlpp | 2 +
.../select-count-one-field.5.query.sqlpp | 2 +-
.../string-standard-utf8.2.query.sqlpp | 2 +
.../type-mismatch.1.ddl.sqlpp} | 22 +-
.../type-mismatch.2.query.sqlpp} | 13 +-
.../type-mismatch.3.query.sqlpp} | 13 +-
.../type-mismatch.4.query.sqlpp} | 11 +-
.../array-access-pushdown.02.adm | 2 +
.../array-access-pushdown.03.adm} | 20 +-
.../array-access-pushdown.04.adm | 2 +
.../array-access-pushdown.05.adm} | 20 +-
.../array-access-pushdown.06.adm | 1 +
.../array-access-pushdown.07.adm | 34 +++
.../array-access-pushdown.08.adm | 1 +
.../array-access-pushdown.09.adm | 34 +++
.../array-access-pushdown.10.adm | 1 +
.../array-access-pushdown.11.adm} | 26 +--
.../array-access-pushdown.12.adm | 1 +
.../array-access-pushdown.13.adm} | 26 +--
.../array-access-pushdown.14.adm | 1 +
.../array-access-pushdown.15.adm | 39 ++++
.../array-access-pushdown.16.adm | 1 +
.../array-access-pushdown.17.adm | 39 ++++
.../array-access-pushdown.18.adm | 2 +
.../array-access-pushdown.19.adm} | 20 +-
.../array-access-pushdown.20.adm | 2 +
.../array-access-pushdown.21.adm} | 20 +-
.../field-access-pushdown.02.adm} | 0
.../field-access-pushdown.03.adm} | 0
.../field-access-pushdown.04.adm} | 0
.../field-access-pushdown.05.adm} | 0
.../field-access-pushdown.06.adm} | 0
.../field-access-pushdown.07.adm} | 0
.../field-access-pushdown.08.adm} | 0
.../field-access-pushdown.09.adm} | 2 +-
.../field-access-pushdown.10.adm} | 0
.../field-access-pushdown.11.adm} | 0
.../field-access-pushdown.12.adm} | 0
.../field-access-pushdown.13.adm} | 4 +-
.../field-access-pushdown.14.adm} | 0
.../field-access-pushdown.15.adm} | 0
.../field-access-pushdown.16.adm} | 0
.../field-access-pushdown.17.adm} | 2 +-
.../field-access-pushdown.18.adm | 1 +
.../field-access-pushdown.19.adm | 18 ++
.../field-access-pushdown.20.adm | 1 +
.../field-access-pushdown.21.adm | 18 ++
.../field-access-pushdown.22.adm} | 2 +-
.../heterogeneous-access-pushdown.02.adm | 6 +
.../heterogeneous-access-pushdown.03.adm | 30 +++
.../heterogeneous-access-pushdown.04.adm | 6 +
.../heterogeneous-access-pushdown.05.adm | 30 +++
.../parquet/object-concat/object-concat.3.adm | 2 +-
.../parquet/object-concat/object-concat.5.adm | 2 +-
.../parquet/pushdown-plans/pushdown-plans.02.adm | 50 +++++
.../pushdown-plans.03.adm} | 20 +-
.../parquet/pushdown-plans/pushdown-plans.04.adm | 50 +++++
.../parquet/pushdown-plans/pushdown-plans.05.adm | 52 +++++
.../parquet/pushdown-plans/pushdown-plans.06.adm | 26 +++
.../parquet/pushdown-plans/pushdown-plans.07.adm | 113 ++++++++++
.../parquet/type-mismatch/type-mismatch.02.adm | 2 +
.../parquet/type-mismatch/type-mismatch.03.adm | 2 +
.../parquet/type-mismatch/type-mismatch.04.adm | 2 +
.../runtimets/testsuite_external_dataset_s3.xml | 31 ++-
.../external/input/HDFSDataSourceFactory.java | 24 ++-
.../aws/parquet/AwsS3ParquetReaderFactory.java | 2 +-
.../reader/hdfs/HDFSLookupReaderFactory.java | 2 +-
.../hdfs/parquet/AbstractComplexConverter.java | 28 +--
.../parquet/AsterixTypeToParquetTypeVisitor.java | 209 +++++++++++++++++++
.../reader/hdfs/parquet/MissingConverter.java} | 49 ++++-
.../hdfs/parquet/ParquetFileRecordReader.java | 16 +-
.../reader/hdfs/parquet/ParquetReadSupport.java | 81 +++-----
.../external/util/ExternalDataConstants.java | 5 +
.../asterix/external/util/ExternalDataUtils.java | 47 +++++
.../apache/asterix/external/util/HDFSUtils.java | 104 +++++++++-
.../metadata/declared/DatasetDataSource.java | 16 +-
.../declared/ExternalDataProjectionInfo.java | 104 ----------
.../projectedfieldnames/TestFieldNamesEquals.java | 98 ---------
146 files changed, 1486 insertions(+), 803 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index b6e287b..4e08d5c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -69,12 +69,12 @@ import org.apache.asterix.optimizer.rules.NestGroupByRule;
import org.apache.asterix.optimizer.rules.PushAggFuncIntoStandaloneAggregateRule;
import org.apache.asterix.optimizer.rules.PushAggregateIntoNestedSubplanRule;
import org.apache.asterix.optimizer.rules.PushFieldAccessRule;
-import org.apache.asterix.optimizer.rules.PushFieldAccessToExternalDataScanRule;
import org.apache.asterix.optimizer.rules.PushGroupByThroughProduct;
import org.apache.asterix.optimizer.rules.PushLimitIntoOrderByRule;
import org.apache.asterix.optimizer.rules.PushLimitIntoPrimarySearchRule;
import org.apache.asterix.optimizer.rules.PushProperJoinThroughProduct;
import org.apache.asterix.optimizer.rules.PushSimilarityFunctionsBelowJoin;
+import org.apache.asterix.optimizer.rules.PushValueAccessToExternalDataScanRule;
import org.apache.asterix.optimizer.rules.RemoveDuplicateFieldsRule;
import org.apache.asterix.optimizer.rules.RemoveLeftOuterUnnestForLeftOuterJoinRule;
import org.apache.asterix.optimizer.rules.RemoveRedundantListifyRule;
@@ -385,11 +385,12 @@ public final class RuleCollections {
// We are going to apply a constant folding rule again for this case.
physicalRewritesTopLevel.add(new ConstantFoldingRule(appCtx));
physicalRewritesTopLevel.add(new PushLimitIntoOrderByRule());
+ //Must run before PushLimitIntoPrimarySearchRule to ensure the select condition is inspected
+ physicalRewritesTopLevel.add(new PushValueAccessToExternalDataScanRule());
physicalRewritesTopLevel.add(new PushLimitIntoPrimarySearchRule());
// remove assigns that could become unused after PushLimitIntoPrimarySearchRule
physicalRewritesTopLevel.add(new RemoveUnusedAssignAndAggregateRule());
physicalRewritesTopLevel.add(new IntroduceProjectsRule());
- physicalRewritesTopLevel.add(new PushFieldAccessToExternalDataScanRule());
physicalRewritesTopLevel.add(new SetAsterixPhysicalOperatorsRule());
physicalRewritesTopLevel.add(new IntroduceRapidFrameFlushProjectAssignRule());
physicalRewritesTopLevel.add(new SetExecutionModeRule());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessToExternalDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessToExternalDataScanRule.java
deleted file mode 100644
index e0bd22e..0000000
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessToExternalDataScanRule.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.exceptions.CompilationException;
-import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.asterix.metadata.declared.DataSource;
-import org.apache.asterix.metadata.declared.DatasetDataSource;
-import org.apache.asterix.metadata.declared.ExternalDataProjectionInfo;
-import org.apache.asterix.metadata.declared.MetadataProvider;
-import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.utils.ConstantExpressionUtil;
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * TODO Use {@link PushValueAccessToExternalDataScanRule}
- * Will be removed in a follow up change
- */
-@Deprecated
-public class PushFieldAccessToExternalDataScanRule implements IAlgebraicRewriteRule {
- //Datasets payload variables
- private final List<LogicalVariable> recordVariables = new ArrayList<>();
- //Dataset scan operators' projection info
- private final List<ExternalDataProjectionInfo> projectionInfos = new ArrayList<>();
- //Final result live variables
- private final Set<LogicalVariable> projectedVariables = new HashSet<>();
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- final ILogicalOperator currentOp = opRef.getValue();
- final LogicalOperatorTag currentOpTag = currentOp.getOperatorTag();
- if (!context.getPhysicalOptimizationConfig().isExternalFieldPushdown()) {
- return false;
- }
- if (currentOpTag == LogicalOperatorTag.PROJECT) {
- ProjectOperator projectOp = (ProjectOperator) currentOp;
- projectedVariables.addAll(projectOp.getVariables());
- return false;
- }
-
- if (currentOpTag != LogicalOperatorTag.DATASOURCESCAN) {
- return false;
- }
-
- return setDatasetProperties(currentOp, (MetadataProvider) context.getMetadataProvider());
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- final ILogicalOperator op = opRef.getValue();
- if (!context.getPhysicalOptimizationConfig().isExternalFieldPushdown()
- || context.checkIfInDontApplySet(this, op) || projectionInfos.isEmpty()) {
- return false;
- }
-
- if (op.getOperatorTag() != LogicalOperatorTag.SELECT && op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
- return false;
- }
-
- if (op.getOperatorTag() == LogicalOperatorTag.SELECT) {
- final SelectOperator selectOp = (SelectOperator) op;
- pushFieldAccessExpression(selectOp.getCondition(), context);
- } else {
- final AssignOperator assignOp = (AssignOperator) op;
- pushFieldAccessExpression(assignOp.getExpressions(), context);
- }
-
- //Add to do not apply to avoid pushing the same expression twice when the plan contains REPLICATE
- context.addToDontApplySet(this, op);
-
- return false;
- }
-
- private void pushFieldAccessExpression(List<Mutable<ILogicalExpression>> exprList, IOptimizationContext context)
- throws AlgebricksException {
-
- for (Mutable<ILogicalExpression> exprRef : exprList) {
- pushFieldAccessExpression(exprRef, context);
- }
- }
-
- private void pushFieldAccessExpression(Mutable<ILogicalExpression> exprRef, IOptimizationContext context)
- throws AlgebricksException {
- final ILogicalExpression expr = exprRef.getValue();
- if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return;
- }
-
- final AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-
- //Only field access expressions are allowed
- if (!isFieldAccessByName(funcExpr)) {
- pushFieldAccessExpression(funcExpr.getArguments(), context);
- return;
- }
-
- //Get root expression input variable in case it is nested field access
- final LogicalVariable funcRootInputVar = getRootExpressionInputVariable(funcExpr);
- if (funcRootInputVar != null) {
- final int recordVarIndex = recordVariables.indexOf(funcRootInputVar);
- //Is funcRootInputVar originated from a data-scan operator?
- if (recordVarIndex >= 0) {
- final List<List<String>> projectedFieldNames = projectionInfos.get(recordVarIndex).getProjectionInfo();
- final List<String> fieldNames = new ArrayList<>();
- //Add fieldAccessExpr to field names list
- buildFieldNames(funcExpr, fieldNames);
- if (!fieldNames.isEmpty()) {
- projectedFieldNames.add(fieldNames);
- }
- }
- } else {
- //Descend to the arguments expressions to see if any can be pushed
- pushFieldAccessExpression(funcExpr.getArguments(), context);
- }
- }
-
- private boolean setDatasetProperties(ILogicalOperator op, MetadataProvider mp) throws AlgebricksException {
- final DataSourceScanOperator scan = (DataSourceScanOperator) op;
- final DataSource dataSource = (DataSource) scan.getDataSource();
-
- if (dataSource == null) {
- return false;
- }
- final DataverseName dataverse = dataSource.getId().getDataverseName();
- final String datasetName = dataSource.getId().getDatasourceName();
- final Dataset dataset = mp.findDataset(dataverse, datasetName);
-
- //Only external dataset can have pushed down expressions
- if (dataset == null || dataset.getDatasetType() == DatasetType.INTERNAL
- || dataset.getDatasetType() == DatasetType.EXTERNAL && !ExternalDataUtils
- .supportsPushdown(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties())) {
- return false;
- }
-
- boolean changed = false;
- final DatasetDataSource datasetDataSource = (DatasetDataSource) dataSource;
- final LogicalVariable recordVar = datasetDataSource.getDataRecordVariable(scan.getVariables());
- if (!projectedVariables.contains(recordVar) && scan.getProjectionInfo() == null) {
- //Do not push expressions to data scan if the whole record is needed
- recordVariables.add(recordVar);
- ExternalDataProjectionInfo projectionInfo = new ExternalDataProjectionInfo();
- scan.setProjectionInfo(projectionInfo);
- projectionInfos.add(projectionInfo);
- changed = true;
- }
- return changed;
- }
-
- private static LogicalVariable getRootExpressionInputVariable(AbstractFunctionCallExpression funcExpr) {
- ILogicalExpression currentExpr = funcExpr.getArguments().get(0).getValue();
- while (isFieldAccessByName(currentExpr)) {
- currentExpr = ((AbstractFunctionCallExpression) currentExpr).getArguments().get(0).getValue();
- }
-
- if (currentExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- return ((VariableReferenceExpression) currentExpr).getVariableReference();
- }
- return null;
- }
-
- private static boolean isFieldAccessByName(ILogicalExpression expression) {
- return expression.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL
- && BuiltinFunctions.FIELD_ACCESS_BY_NAME
- .equals(((AbstractFunctionCallExpression) expression).getFunctionIdentifier());
- }
-
- private static void buildFieldNames(ILogicalExpression expr, List<String> fieldNames) throws CompilationException {
- if (!isFieldAccessByName(expr)) {
- /*
- * We only push nested field-access expressions.
- * This is a sanity check if the previous checks have missed.
- */
- return;
- }
-
- AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
- ILogicalExpression objectExpr = funcExpr.getArguments().get(0).getValue();
- if (!isPayload(objectExpr)) {
- buildFieldNames(objectExpr, fieldNames);
- }
- fieldNames.add(ConstantExpressionUtil.getStringArgument(funcExpr, 1));
- }
-
- private static boolean isPayload(ILogicalExpression expr) {
- return expr.getExpressionTag() == LogicalExpressionTag.VARIABLE;
- }
-}
diff --git a/asterixdb/asterix-app/data/hdfs/parquet/heterogeneous_1.json b/asterixdb/asterix-app/data/hdfs/parquet/heterogeneous_1.json
new file mode 100644
index 0000000..69cefc0
--- /dev/null
+++ b/asterixdb/asterix-app/data/hdfs/parquet/heterogeneous_1.json
@@ -0,0 +1,3 @@
+{"id": 0, "arrayOrObject" : [{"text" : "1"}, {"text" : "2"}]}
+{"id": 1, "arrayOrObject" : [{"text" : "3"}, {"text" : "4"}]}
+{"id": 2, "arrayOrObject" : [{"text" : "5"}, {"text" : "6"}]}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/hdfs/parquet/heterogeneous_2.json b/asterixdb/asterix-app/data/hdfs/parquet/heterogeneous_2.json
new file mode 100644
index 0000000..4826af5
--- /dev/null
+++ b/asterixdb/asterix-app/data/hdfs/parquet/heterogeneous_2.json
@@ -0,0 +1,3 @@
+{"id": 3, "arrayOrObject" : {"text" : "7"}}
+{"id": 4, "arrayOrObject" : {"text" : "8"}}
+{"id": 5, "arrayOrObject" : {"text" : "9"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index d445057..f6b501a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -240,6 +240,8 @@ public class ExternalDatasetTestUtils {
loadData(dataBasePath, "", "id_age-string.parquet", definition, definitionSegment, false, false);
loadData(dataBasePath, "", "id_name.parquet", definition, definitionSegment, false, false);
loadData(dataBasePath, "", "id_name_comment.parquet", definition, definitionSegment, false, false);
+ loadData(dataBasePath, "", "heterogeneous_1.parquet", definition, definitionSegment, false, false);
+ loadData(dataBasePath, "", "heterogeneous_2.parquet", definition, definitionSegment, false, false);
}
private static void loadData(String fileBasePath, String filePathSegment, String filename, String definition,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.01.ddl.sqlpp
similarity index 69%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.01.ddl.sqlpp
index 491d789..4e21d51 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.01.ddl.sqlpp
@@ -17,13 +17,25 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
+* Description : Array access pushdown
* Expected Res : Success
-* Date : June 22nd 2020
+* Date : July 20th 2021
*/
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
USE test;
-SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.text)
-FROM ParquetDataset p;
\ No newline at end of file
+CREATE TYPE ParquetType as {
+};
+
+CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING %adapter%
+(
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/reviews"),
+ ("include"="*dummy_tweet.parquet"),
+ ("format" = "parquet")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.02.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.02.query.sqlpp
index d51e173..3b645f1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.02.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down get-item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "false";
-SELECT VALUE p
+
+SELECT p.entities.urls[0].display_url
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.03.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.03.query.sqlpp
index 18669ba..c268b2a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.03.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down get-item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "false";
EXPLAIN
-SELECT VALUE p
+SELECT p.entities.urls[0].display_url
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.04.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.04.query.sqlpp
index 824ea80..43a6c07 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.04.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down get-item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE p
+
+SELECT p.entities.urls[0].display_url
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.05.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.05.query.sqlpp
index 2aac7ff..8187033 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.05.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down get-item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "true";
EXPLAIN
-SELECT VALUE p
+SELECT p.entities.urls[0].display_url
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.06.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.06.query.sqlpp
index d51e173..9e3e02b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.06.query.sqlpp
@@ -17,15 +17,17 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down p.entities.urls
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "false";
-SELECT VALUE p
+
+SELECT p.entities.urls[*].display_url
FROM ParquetDataset p
+WHERE p.entities.urls IS NOT MISSING
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.07.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.07.query.sqlpp
index 18669ba..bf42c9a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.07.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down p.entities.urls
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,7 @@ USE test;
SET `compiler.external.field.pushdown` "false";
EXPLAIN
-SELECT VALUE p
+SELECT p.entities.urls[*].display_url
FROM ParquetDataset p
+WHERE p.entities.urls IS NOT MISSING
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.08.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.08.query.sqlpp
index 824ea80..0148449 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.08.query.sqlpp
@@ -17,15 +17,17 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down p.entities.urls
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE p
+
+SELECT p.entities.urls[*].display_url
FROM ParquetDataset p
+WHERE p.entities.urls IS NOT MISSING
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.09.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.09.query.sqlpp
index 2aac7ff..cbb85b3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.09.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down p.entities.urls
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,7 @@ USE test;
SET `compiler.external.field.pushdown` "true";
EXPLAIN
-SELECT VALUE p
+SELECT p.entities.urls[*].display_url
FROM ParquetDataset p
+WHERE p.entities.urls IS NOT MISSING
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.10.query.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.10.query.sqlpp
index d51e173..cab4c18 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.10.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down scan-collection from unnest
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "false";
-SELECT VALUE p
-FROM ParquetDataset p
+
+SELECT urls.display_url
+FROM ParquetDataset p, p.entities.urls urls
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.11.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.11.query.sqlpp
index 18669ba..131ae3d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.11.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down scan-collection from unnest
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "false";
EXPLAIN
-SELECT VALUE p
-FROM ParquetDataset p
+SELECT urls.display_url
+FROM ParquetDataset p, p.entities.urls urls
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.12.query.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.12.query.sqlpp
index f44c910..509806c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.12.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Push only common fields when requesting nested values
+ * Description : Push down scan-collection from unnest
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT p.user.id, p.user.name
-FROM ParquetDataset4 p
+
+SELECT urls.display_url
+FROM ParquetDataset p, p.entities.urls urls
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.13.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.13.query.sqlpp
index 2aac7ff..79d5558 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.13.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down scan-collection from unnest
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "true";
EXPLAIN
-SELECT VALUE p
-FROM ParquetDataset p
+SELECT urls.display_url
+FROM ParquetDataset p, p.entities.urls urls
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.14.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.14.query.sqlpp
index d51e173..811074e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.14.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down scan-collection from unnest in a subplan
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "false";
-SELECT VALUE p
+
+SELECT VALUE COUNT(*)
FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+WHERE (EVERY ht in p.entities.urls SATISFIES ht.display_url = "string");
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.15.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.15.query.sqlpp
index 18669ba..45ee208 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.15.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down scan-collection from unnest in a subplan
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "false";
EXPLAIN
-SELECT VALUE p
+SELECT VALUE COUNT(*)
FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+WHERE (EVERY ht in p.entities.urls SATISFIES ht.display_url = "string");
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.16.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.16.query.sqlpp
index 824ea80..108524f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.16.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down scan-collection from unnest in a subplan
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE p
+
+SELECT VALUE COUNT(*)
FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+WHERE (EVERY ht in p.entities.urls SATISFIES ht.display_url = "string");
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.17.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.17.query.sqlpp
index 2aac7ff..407fd74 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.17.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down scan-collection from unnest in a subplan
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "true";
EXPLAIN
-SELECT VALUE p
+SELECT VALUE COUNT(*)
FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+WHERE (EVERY ht in p.entities.urls SATISFIES ht.display_url = "string");
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.18.query.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.18.query.sqlpp
index d51e173..fd9b28b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.18.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down nested get_item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "false";
-SELECT VALUE p
+
+SELECT VALUE p.place.bounding_box.coordinates[0][0][0]
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.19.query.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.19.query.sqlpp
index 18669ba..e2fb357 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.19.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down nested get_item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "false";
EXPLAIN
-SELECT VALUE p
+SELECT VALUE p.place.bounding_box.coordinates[0][0][0]
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.20.query.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.20.query.sqlpp
index 824ea80..3c01e49 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.20.query.sqlpp
@@ -17,15 +17,16 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down nested get_item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE p
+
+SELECT VALUE p.place.bounding_box.coordinates[0][0][0]
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.21.query.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.21.query.sqlpp
index 2aac7ff..57435ed 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.21.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Push down nested get_item
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 20th 2021
*/
USE test;
@@ -27,6 +27,6 @@ USE test;
SET `compiler.external.field.pushdown` "true";
EXPLAIN
-SELECT VALUE p
+SELECT VALUE p.place.bounding_box.coordinates[0][0][0]
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.01.ddl.sqlpp
similarity index 97%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.01.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.01.ddl.sqlpp
index f1ab714..9db6efb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.01.ddl.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
/*
-* Description : Expression pushdown
+* Description : Field access pushdown
* Expected Res : Success
* Date : June 22nd 2020
*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.02.query.sqlpp
similarity index 98%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.02.query.sqlpp
index d51e173..d457e75 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.02.query.sqlpp
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "false";
+
SELECT VALUE p
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.03.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.03.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.04.query.sqlpp
similarity index 98%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.04.query.sqlpp
index 824ea80..4e31474 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.04.query.sqlpp
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "true";
+
SELECT VALUE p
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.05.query.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.05.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.06.query.sqlpp
similarity index 98%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.06.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.06.query.sqlpp
index a354189..127a914 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.06.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.06.query.sqlpp
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "false";
+
SELECT p1, p2.id
FROM ParquetDataset p1, ParquetDataset2 p2
WHERE p1.id = p2.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.07.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.07.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.07.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.08.query.sqlpp
similarity index 98%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.08.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.08.query.sqlpp
index 4b2003d..2bdfa95 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.08.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.08.query.sqlpp
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "true";
+
SELECT p1, p2.id
FROM ParquetDataset p1, ParquetDataset2 p2
WHERE p1.id = p2.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.09.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.09.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.09.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.10.query.sqlpp
similarity index 98%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.10.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.10.query.sqlpp
index 878875c..8dc44f3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.10.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.10.query.sqlpp
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "false";
+
SELECT p1.age, p2.name
FROM ParquetDataset p1, ParquetDataset3 p2
WHERE p1.id = p2.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.11.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.11.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.11.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.11.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.12.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.12.query.sqlpp
similarity index 98%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.12.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.12.query.sqlpp
index 627719e..99bb5fc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.12.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.12.query.sqlpp
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "true";
+
SELECT p1.age, p2.name
FROM ParquetDataset p1, ParquetDataset3 p2
WHERE p1.id = p2.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.13.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.13.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.13.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.13.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.14.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.14.query.sqlpp
similarity index 90%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.14.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.14.query.sqlpp
index 41f24d4..190d40d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.14.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.14.query.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
/*
- * Description : Push only common fields when requesting nested values
+ * Description : Push down children of common fields access when requesting nested values
* Expected Res : Success
* Date : June 22nd 2020
*/
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "false";
+
SELECT p.user.id, p.user.name
FROM ParquetDataset4 p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.15.query.sqlpp
similarity index 90%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.15.query.sqlpp
index aea2583..32aaf12 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.15.query.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
/*
- * Description : Push only common fields when requesting nested values
+ * Description : Push down children of common fields access when requesting nested values
* Expected Res : Success
* Date : June 22nd 2020
*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.16.query.sqlpp
similarity index 90%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.16.query.sqlpp
index f44c910..1213808 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.16.query.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
/*
- * Description : Push only common fields when requesting nested values
+ * Description : Push down children of common fields access when requesting nested values
* Expected Res : Success
* Date : June 22nd 2020
*/
@@ -26,6 +26,7 @@ USE test;
SET `compiler.external.field.pushdown` "true";
+
SELECT p.user.id, p.user.name
FROM ParquetDataset4 p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.17.query.sqlpp
similarity index 90%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.17.query.sqlpp
index f855121..2a47449 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.17.query.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
/*
- * Description : Push only common fields when requesting nested values
+ * Description : Push down children of common fields access when requesting nested values
* Expected Res : Success
* Date : June 22nd 2020
*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.18.query.sqlpp
similarity index 85%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.18.query.sqlpp
index d51e173..e42c9ca 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.18.query.sqlpp
@@ -17,15 +17,15 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Test SELECT COUNT(*)
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 25nd 2021
*/
USE test;
SET `compiler.external.field.pushdown` "false";
-SELECT VALUE p
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+
+SELECT VALUE COUNT(*)
+FROM ParquetDataset4 p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.19.query.sqlpp
similarity index 85%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.19.query.sqlpp
index 18669ba..71624af 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.19.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Test SELECT COUNT(*)
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 25nd 2021
*/
USE test;
@@ -27,6 +27,5 @@ USE test;
SET `compiler.external.field.pushdown` "false";
EXPLAIN
-SELECT VALUE p
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+SELECT VALUE COUNT(*)
+FROM ParquetDataset4 p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.20.query.sqlpp
similarity index 85%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.20.query.sqlpp
index 824ea80..f8d1312 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.20.query.sqlpp
@@ -17,15 +17,15 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Test SELECT COUNT(*). Should get empty records
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 25nd 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE p
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+
+SELECT VALUE COUNT(*)
+FROM ParquetDataset4 p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.21.query.sqlpp
similarity index 85%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.21.query.sqlpp
index 2aac7ff..c433eb5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.21.query.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
+ * Description : Test SELECT COUNT(*). Should get empty records
* Expected Res : Success
- * Date : June 22nd 2020
+ * Date : July 25nd 2021
*/
USE test;
@@ -27,6 +27,5 @@ USE test;
SET `compiler.external.field.pushdown` "true";
EXPLAIN
-SELECT VALUE p
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+SELECT VALUE COUNT(*)
+FROM ParquetDataset4 p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.18.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.22.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.18.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.22.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.1.ddl.sqlpp
similarity index 69%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.1.ddl.sqlpp
index 491d789..f1c4e81 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.1.ddl.sqlpp
@@ -17,13 +17,25 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
+* Description : Heterogeneous access DDL
* Expected Res : Success
-* Date : June 22nd 2020
+* Date : July 23th 2021
*/
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
USE test;
-SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.text)
-FROM ParquetDataset p;
\ No newline at end of file
+CREATE TYPE ParquetType as {
+};
+
+CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING %adapter%
+(
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/reviews"),
+ ("include"="*heterogeneous*"),
+ ("format" = "parquet")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.2.query.sqlpp
similarity index 77%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.2.query.sqlpp
index d51e173..1ffdf5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.2.query.sqlpp
@@ -17,15 +17,21 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
- * Expected Res : Success
- * Date : June 22nd 2020
- */
-
+* Description : Accessing a heterogeneous value
+* Expected Res : Success
+* Date : July 23th 2021
+*/
USE test;
SET `compiler.external.field.pushdown` "false";
-SELECT VALUE p
+
+SELECT VALUE (
+ CASE WHEN is_array(p.arrayOrObject) THEN
+ p.arrayOrObject[*].text
+ ELSE
+ p.arrayOrObject.text
+ END
+)
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.3.query.sqlpp
similarity index 77%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.3.query.sqlpp
index 18669ba..428c496 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.3.query.sqlpp
@@ -17,16 +17,21 @@
* under the License.
*/
/*
- * Description : Ignore Field Access pushdown when requesting all fields
- * Expected Res : Success
- * Date : June 22nd 2020
- */
-
+* Description : Accessing a heterogeneous value
+* Expected Res : Success
+* Date : July 23th 2021
+*/
USE test;
SET `compiler.external.field.pushdown` "false";
EXPLAIN
-SELECT VALUE p
+SELECT VALUE (
+ CASE WHEN is_array(p.arrayOrObject) THEN
+ p.arrayOrObject[*].text
+ ELSE
+ p.arrayOrObject.text
+ END
+)
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.4.query.sqlpp
similarity index 79%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.4.query.sqlpp
index a429b73..f75c1f2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.4.query.sqlpp
@@ -17,13 +17,21 @@
* under the License.
*/
/*
-* Description : Concat two objects after pushdown
+* Description : Accessing a heterogeneous value
* Expected Res : Success
-* Date : September 22nd 2020
+* Date : July 23th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE object_concat(p.coordinates, p.user).name
+
+
+SELECT VALUE (
+ CASE WHEN is_array(p.arrayOrObject) THEN
+ p.arrayOrObject[*].text
+ ELSE
+ p.arrayOrObject.text
+ END
+)
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.5.query.sqlpp
similarity index 79%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.5.query.sqlpp
index dd114e6..4336a0d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.5.query.sqlpp
@@ -17,14 +17,21 @@
* under the License.
*/
/*
-* Description : Concat two objects after pushdown
+* Description : Accessing a heterogeneous value
* Expected Res : Success
-* Date : September 22nd 2020
+* Date : July 23th 2021
*/
USE test;
SET `compiler.external.field.pushdown` "true";
+
EXPLAIN
-SELECT VALUE object_concat(p.coordinates, p.user).name
+SELECT VALUE (
+ CASE WHEN is_array(p.arrayOrObject) THEN
+ p.arrayOrObject[*].text
+ ELSE
+ p.arrayOrObject.text
+ END
+)
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/missing-fields/missing-fields.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/missing-fields/missing-fields.2.query.sqlpp
index 1715bba..8372ce6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/missing-fields/missing-fields.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/missing-fields/missing-fields.2.query.sqlpp
@@ -23,5 +23,6 @@
*/
USE test;
+SET `compiler.external.field.pushdown` "false";
SELECT p.not_a_field1 IS MISSING as f1, p.user.not_a_field2 IS MISSING as f2
FROM ParquetDataset p
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.2.query.sqlpp
index a429b73..daacd24 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.2.query.sqlpp
@@ -24,6 +24,8 @@
USE test;
SET `compiler.external.field.pushdown` "true";
+
+
SELECT VALUE object_concat(p.coordinates, p.user).name
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
index dd114e6..7463ab1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
@@ -24,6 +24,7 @@
USE test;
SET `compiler.external.field.pushdown` "true";
+
EXPLAIN
SELECT VALUE object_concat(p.coordinates, p.user).name
FROM ParquetDataset p
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.4.query.sqlpp
index 06bd91c..c3a7586 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.4.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.4.query.sqlpp
@@ -24,6 +24,8 @@
USE test;
SET `compiler.external.field.pushdown` "true";
+
+
SELECT VALUE object_concat(p.coordinates, p.user)
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.5.query.sqlpp
index 0b42622..cbbc793 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.5.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.5.query.sqlpp
@@ -24,6 +24,8 @@
USE test;
SET `compiler.external.field.pushdown` "true";
+
+
EXPLAIN
SELECT VALUE object_concat(p.coordinates, p.user)
FROM ParquetDataset p
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.01.ddl.sqlpp
similarity index 67%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.01.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.01.ddl.sqlpp
index f1ab714..1913848 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.01.ddl.sqlpp
@@ -17,9 +17,9 @@
* under the License.
*/
/*
-* Description : Expression pushdown
+* Description : Test complex plans while enabling the value access pushdown
* Expected Res : Success
-* Date : June 22nd 2020
+* Date : July 23th 2021
*/
DROP DATAVERSE test IF EXISTS;
@@ -31,12 +31,12 @@ USE test;
CREATE TYPE ParquetType as {
};
-CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING %adapter%
+CREATE EXTERNAL DATASET ParquetDataset1(ParquetType) USING %adapter%
(
%template%,
("container"="playground"),
("definition"="parquet-data/reviews"),
- ("include"="*id_age.parquet"),
+ ("include"="*dummy_tweet.parquet"),
("format" = "parquet")
);
@@ -45,24 +45,6 @@ CREATE EXTERNAL DATASET ParquetDataset2(ParquetType) USING %adapter%
%template%,
("container"="playground"),
("definition"="parquet-data/reviews"),
- ("include"="*id_age.parquet"),
- ("format" = "parquet")
-);
-
-CREATE EXTERNAL DATASET ParquetDataset3(ParquetType) USING %adapter%
-(
- %template%,
- ("container"="playground"),
- ("definition"="parquet-data/reviews"),
- ("include"="*id_name_comment.parquet"),
- ("format" = "parquet")
-);
-
-CREATE EXTERNAL DATASET ParquetDataset4(ParquetType) USING %adapter%
-(
- %template%,
- ("container"="playground"),
- ("definition"="parquet-data/reviews"),
("include"="*dummy_tweet.parquet"),
("format" = "parquet")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.02.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.02.query.sqlpp
index 491d789..dcaa9f3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.02.query.sqlpp
@@ -17,13 +17,16 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
+* Description : No pushdown on p1
* Expected Res : Success
-* Date : June 22nd 2020
+* Date : July 23th 2021
*/
+
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.text)
-FROM ParquetDataset p;
\ No newline at end of file
+EXPLAIN
+SELECT sum(object_length(p1))
+FROM ParquetDataset1 p1, ParquetDataset2 p2
+WHERE p2.id = p1.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.03.query.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.5.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.03.query.sqlpp
index 76ac0ce..0b08465 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.5.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.03.query.sqlpp
@@ -17,13 +17,16 @@
* under the License.
*/
/*
-* Description : Retrieve the number of users (with expression pushdown)
+* Description : No pushdown
* Expected Res : Success
-* Date : June 22nd 2020
+* Date : July 23th 2021
*/
+
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.user.name)
-FROM ParquetDataset p;
\ No newline at end of file
+EXPLAIN
+SELECT DISTINCT VALUE p1
+FROM ParquetDataset1 p1
+WHERE p1.id > 10;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.04.query.sqlpp
similarity index 76%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.04.query.sqlpp
index dd114e6..2e7bb09 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.04.query.sqlpp
@@ -17,14 +17,17 @@
* under the License.
*/
/*
-* Description : Concat two objects after pushdown
+* Description : Should only pushdown "p1.entities.hashtags"
* Expected Res : Success
-* Date : September 22nd 2020
+* Date : July 23th 2021
*/
+
USE test;
SET `compiler.external.field.pushdown` "true";
+
EXPLAIN
-SELECT VALUE object_concat(p.coordinates, p.user).name
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+SELECT text, sum(array_distinct(p1.entities.hashtags))
+FROM ParquetDataset1 p1, p1.entities.hashtags ht
+WHERE p1.id > 10 AND lowercase(ht.text) = "string"
+GROUP BY ht.text AS text;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.05.query.sqlpp
similarity index 76%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.05.query.sqlpp
index dd114e6..61d00b8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.05.query.sqlpp
@@ -17,14 +17,17 @@
* under the License.
*/
/*
-* Description : Concat two objects after pushdown
+* Description : Get only "indices" and "text" from "p1.entities.hashtags"
* Expected Res : Success
-* Date : September 22nd 2020
+* Date : July 23th 2021
*/
+
USE test;
SET `compiler.external.field.pushdown` "true";
+
EXPLAIN
-SELECT VALUE object_concat(p.coordinates, p.user).name
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+SELECT text, sum(array_sum(ht.indices))
+FROM ParquetDataset1 p1, p1.entities.hashtags ht
+WHERE p1.id > 10 AND lowercase(ht.text) = "string"
+GROUP BY ht.text AS text;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.query.sqlpp
index dd114e6..93b09d0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.query.sqlpp
@@ -17,14 +17,17 @@
* under the License.
*/
/*
-* Description : Concat two objects after pushdown
+* Description : Pushdown "p.entities.urls"
* Expected Res : Success
-* Date : September 22nd 2020
+* Date : July 23th 2021
*/
+
USE test;
SET `compiler.external.field.pushdown` "true";
+
EXPLAIN
-SELECT VALUE object_concat(p.coordinates, p.user).name
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+SELECT p.entities.urls[0].display_url
+FROM ParquetDataset1 p
+WHERE array_count(p.entities.urls) > 10
+LIMIT 10
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.07.query.sqlpp
similarity index 69%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.07.query.sqlpp
index dd114e6..b098547 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/object-concat/object-concat.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/pushdown-plans/pushdown-plans.07.query.sqlpp
@@ -17,14 +17,23 @@
* under the License.
*/
/*
-* Description : Concat two objects after pushdown
+* Description : Get "entities.hashtags.text" from "p1" and "p2"
+ and "user.name" from "p1"
* Expected Res : Success
-* Date : September 22nd 2020
+* Date : July 23th 2021
*/
+
USE test;
SET `compiler.external.field.pushdown` "true";
+
EXPLAIN
-SELECT VALUE object_concat(p.coordinates, p.user).name
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+SELECT uname, COUNT(*) as cnt
+FROM ParquetDataset1 p1, p1.entities.hashtags ht1
+WHERE ht1.text IN (SELECT VALUE ht2.text
+ FROM ParquetDataset2 p2, p2.entities.hashtags ht2)
+GROUP BY p1.user.name AS uname
+ORDER BY cnt DESC
+LIMIT 10;
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-all-fields/select-all-fields.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-all-fields/select-all-fields.2.query.sqlpp
index 51a7b3e..3a0e80d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-all-fields/select-all-fields.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-all-fields/select-all-fields.2.query.sqlpp
@@ -23,6 +23,8 @@
*/
USE test;
+SET `compiler.external.field.pushdown` "false";
+
SELECT VALUE p
FROM ParquetDataset p
ORDER BY p.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
index 491d789..7b022fe 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
@@ -17,13 +17,13 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
+* Description : Retrieve the number of texts in all tweets
* Expected Res : Success
* Date : June 22nd 2020
*/
USE test;
-SET `compiler.external.field.pushdown` "true";
+SET `compiler.external.field.pushdown` "false";
SELECT VALUE count(p.text)
FROM ParquetDataset p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.3.query.sqlpp
index 84639bf..9ec9468 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.3.query.sqlpp
@@ -23,5 +23,7 @@
*/
USE test;
+SET `compiler.external.field.pushdown` "true";
+
SELECT VALUE count(p.text)
FROM ParquetDataset p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.4.query.sqlpp
index dddb9b3..a1e72c2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.4.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.4.query.sqlpp
@@ -23,5 +23,7 @@
*/
USE test;
+SET `compiler.external.field.pushdown` "false";
+
SELECT VALUE count(p.user.name)
FROM ParquetDataset p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.5.query.sqlpp
index 76ac0ce..0a0e193 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.5.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.5.query.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
/*
-* Description : Retrieve the number of users (with expression pushdown)
+* Description : Retrieve the number of users
* Expected Res : Success
* Date : June 22nd 2020
*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/string-standard-utf8/string-standard-utf8.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/string-standard-utf8/string-standard-utf8.2.query.sqlpp
index f849a1f..a957ac5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/string-standard-utf8/string-standard-utf8.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/string-standard-utf8/string-standard-utf8.2.query.sqlpp
@@ -23,6 +23,8 @@
*/
USE test;
+SET `compiler.external.field.pushdown` "false";
+
SELECT VALUE array_count(split(trim(p.comment),"𩸽"))
FROM ParquetDataset p
WHERE contains(p.comment, "𩸽");
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.1.ddl.sqlpp
similarity index 69%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.1.ddl.sqlpp
index 491d789..ab202b7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.1.ddl.sqlpp
@@ -17,13 +17,25 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
+* Description : Type mismatch DDL
* Expected Res : Success
-* Date : June 22nd 2020
+* Date : July 20th 2021
*/
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
USE test;
-SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.text)
-FROM ParquetDataset p;
\ No newline at end of file
+CREATE TYPE ParquetType as {
+};
+
+CREATE EXTERNAL DATASET ParquetDataset(ParquetType) USING %adapter%
+(
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/reviews"),
+ ("include"="*dummy_tweet.parquet"),
+ ("format" = "parquet")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.2.query.sqlpp
similarity index 80%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.2.query.sqlpp
index 491d789..29008c8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.2.query.sqlpp
@@ -17,13 +17,16 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
-* Expected Res : Success
-* Date : June 22nd 2020
+* Description : Accessing an array as an object
+* Expected Res : Warning
+* Date : July 20th 2021
*/
+
+-- param max-warnings:json=1000
+
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.text)
-FROM ParquetDataset p;
\ No newline at end of file
+SELECT VALUE p.entities.urls.display_url IS MISSING
+FROM ParquetDataset p
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.3.query.sqlpp
similarity index 80%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.3.query.sqlpp
index 491d789..65f6bdf 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.3.query.sqlpp
@@ -17,13 +17,16 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
-* Expected Res : Success
-* Date : June 22nd 2020
+* Description : Accessing an object as an array
+* Expected Res : Warning
+* Date : July 20th 2021
*/
+
+-- param max-warnings:json=1000
+
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.text)
-FROM ParquetDataset p;
\ No newline at end of file
+SELECT VALUE p.entities[*].display_url IS MISSING
+FROM ParquetDataset p
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.4.query.sqlpp
similarity index 78%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.4.query.sqlpp
index 491d789..26ccbcb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/select-count-one-field/select-count-one-field.2.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/type-mismatch/type-mismatch.4.query.sqlpp
@@ -17,13 +17,16 @@
* under the License.
*/
/*
-* Description : Retrieve the number of texts in all tweets (with expression pushdown)
-* Expected Res : Success
-* Date : June 22nd 2020
+* Description : Accessing an array as an object
+* Expected Res : Warning
+* Date : July 20th 2021
*/
+
+-- param max-warnings:json=1000
+
USE test;
SET `compiler.external.field.pushdown` "true";
-SELECT VALUE count(p.text)
+SELECT VALUE p.place.bounding_box.coordinates[0][0].not_object IS MISSING
FROM ParquetDataset p;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.02.adm
new file mode 100644
index 0000000..18f3275
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.02.adm
@@ -0,0 +1,2 @@
+{ "display_url": "string" }
+{ }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.03.adm
similarity index 62%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.03.adm
index a5f69e0..f0fad1e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.03.adm
@@ -1,26 +1,26 @@
-distribute result [$$15]
+distribute result [$$17]
-- DISTRIBUTE_RESULT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$15])
+ project ([$$17])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$15] <- [object-concat($$18, $$19)]
+ assign [$$17] <- [{"display_url": $$20}]
-- ASSIGN |PARTITIONED|
- project ([$$18, $$19])
+ project ([$$20])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
- order (ASC, $$17)
- -- STABLE_SORT [$$17(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$19(ASC) ] |PARTITIONED|
+ order (ASC, $$19)
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$18, $$19, $$17])
+ project ([$$20, $$19])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$19, $$18, $$17] <- [$$p.getField("user"), $$p.getField("coordinates"), $$p.getField("id")]
+ assign [$$20, $$19] <- [get-item($$p.getField("entities").getField("urls"), 0).getField("display_url"), $$p.getField("id")]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset project (user, coordinates, id)
+ data-scan []<-[$$p] <- test.ParquetDataset
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.04.adm
new file mode 100644
index 0000000..18f3275
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.04.adm
@@ -0,0 +1,2 @@
+{ "display_url": "string" }
+{ }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.05.adm
similarity index 61%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.05.adm
index a5f69e0..2a24b09 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.05.adm
@@ -1,26 +1,26 @@
-distribute result [$$15]
+distribute result [$$17]
-- DISTRIBUTE_RESULT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$15])
+ project ([$$17])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$15] <- [object-concat($$18, $$19)]
+ assign [$$17] <- [{"display_url": $$20}]
-- ASSIGN |PARTITIONED|
- project ([$$18, $$19])
+ project ([$$20])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
- order (ASC, $$17)
- -- STABLE_SORT [$$17(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$19(ASC) ] |PARTITIONED|
+ order (ASC, $$19)
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$18, $$19, $$17])
+ project ([$$20, $$19])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$19, $$18, $$17] <- [$$p.getField("user"), $$p.getField("coordinates"), $$p.getField("id")]
+ assign [$$20, $$19] <- [get-item($$p.getField("entities").getField("urls"), 0).getField("display_url"), $$p.getField("id")]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset project (user, coordinates, id)
+ data-scan []<-[$$p] <- test.ParquetDataset project ({entities:{urls:[{display_url:any}]},id:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.06.adm
new file mode 100644
index 0000000..695240b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.06.adm
@@ -0,0 +1 @@
+{ "display_url": [ "string" ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.07.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.07.adm
new file mode 100644
index 0000000..1ed4eed
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.07.adm
@@ -0,0 +1,34 @@
+distribute result [$$21]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$21])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$21] <- [{"display_url": $$25}]
+ -- ASSIGN |PARTITIONED|
+ project ([$$25])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED|
+ order (ASC, $$24)
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$25, $$24])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$25] <- [array-star($$23).getField("display_url")]
+ -- ASSIGN |PARTITIONED|
+ select (not(is-missing($$23)))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$23, $$24])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$23, $$24] <- [$$p.getField("entities").getField("urls"), $$p.getField("id")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.08.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.08.adm
new file mode 100644
index 0000000..695240b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.08.adm
@@ -0,0 +1 @@
+{ "display_url": [ "string" ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.09.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.09.adm
new file mode 100644
index 0000000..2302d35
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.09.adm
@@ -0,0 +1,34 @@
+distribute result [$$21]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$21])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$21] <- [{"display_url": $$25}]
+ -- ASSIGN |PARTITIONED|
+ project ([$$25])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$24(ASC) ] |PARTITIONED|
+ order (ASC, $$24)
+ -- STABLE_SORT [$$24(ASC)] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$25, $$24])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$25] <- [array-star($$23).getField("display_url")]
+ -- ASSIGN |PARTITIONED|
+ select (not(is-missing($$23)))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$23, $$24])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$23, $$24] <- [$$p.getField("entities").getField("urls"), $$p.getField("id")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset project ({entities:{urls:[{display_url:any}]},id:any})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.10.adm
new file mode 100644
index 0000000..41c14f5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.10.adm
@@ -0,0 +1 @@
+{ "display_url": "string" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.11.adm
similarity index 59%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.11.adm
index dc8c103..a9ee166 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.11.adm
@@ -1,30 +1,30 @@
-distribute result [$$17]
+distribute result [$$26]
-- DISTRIBUTE_RESULT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$17])
+ project ([$$26])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$17] <- [{"id": $$21, "name": $$22}]
+ assign [$$26] <- [{"display_url": $$urls.getField("display_url")}]
-- ASSIGN |PARTITIONED|
- project ([$$21, $$22])
+ project ([$$urls])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$20(ASC) ] |PARTITIONED|
- order (ASC, $$20)
- -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$29(ASC) ] |PARTITIONED|
+ order (ASC, $$29)
+ -- STABLE_SORT [$$29(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$21, $$22, $$20])
+ project ([$$urls, $$29])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$22, $$21] <- [$$19.getField("name"), $$19.getField("id")]
- -- ASSIGN |PARTITIONED|
- project ([$$19, $$20])
+ unnest $$urls <- scan-collection($$28)
+ -- UNNEST |PARTITIONED|
+ project ([$$28, $$29])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$19, $$20] <- [$$p.getField("user"), $$p.getField("id")]
+ assign [$$28, $$29] <- [$$p.getField("entities").getField("urls"), $$p.getField("id")]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset4
+ data-scan []<-[$$p] <- test.ParquetDataset
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.12.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.12.adm
new file mode 100644
index 0000000..41c14f5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.12.adm
@@ -0,0 +1 @@
+{ "display_url": "string" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.13.adm
similarity index 57%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.13.adm
index d812b1f..a29ef6c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.13.adm
@@ -1,30 +1,30 @@
-distribute result [$$17]
+distribute result [$$26]
-- DISTRIBUTE_RESULT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$17])
+ project ([$$26])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$17] <- [{"id": $$21, "name": $$22}]
+ assign [$$26] <- [{"display_url": $$urls.getField("display_url")}]
-- ASSIGN |PARTITIONED|
- project ([$$21, $$22])
+ project ([$$urls])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$20(ASC) ] |PARTITIONED|
- order (ASC, $$20)
- -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$29(ASC) ] |PARTITIONED|
+ order (ASC, $$29)
+ -- STABLE_SORT [$$29(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$21, $$22, $$20])
+ project ([$$urls, $$29])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$22, $$21] <- [$$19.getField("name"), $$19.getField("id")]
- -- ASSIGN |PARTITIONED|
- project ([$$19, $$20])
+ unnest $$urls <- scan-collection($$28)
+ -- UNNEST |PARTITIONED|
+ project ([$$28, $$29])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$19, $$20] <- [$$p.getField("user"), $$p.getField("id")]
+ assign [$$28, $$29] <- [$$p.getField("entities").getField("urls"), $$p.getField("id")]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset4 project (user, id)
+ data-scan []<-[$$p] <- test.ParquetDataset project ({entities:{urls:[{display_url:any}]},id:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.14.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.14.adm
new file mode 100644
index 0000000..0cfbf08
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.14.adm
@@ -0,0 +1 @@
+2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.15.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.15.adm
new file mode 100644
index 0000000..de214d0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.15.adm
@@ -0,0 +1,39 @@
+distribute result [$$46]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ aggregate [$$46] <- [agg-sql-sum($$52)]
+ -- AGGREGATE |UNPARTITIONED|
+ exchange
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ aggregate [$$52] <- [agg-sql-count(1)]
+ -- AGGREGATE |PARTITIONED|
+ select ($$39)
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$39])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$39] <- [empty-stream()]
+ -- AGGREGATE |LOCAL|
+ select (not(if-missing-or-null(eq($$48, "string"), FALSE)))
+ -- STREAM_SELECT |LOCAL|
+ assign [$$48] <- [$$ht.getField("display_url")]
+ -- ASSIGN |LOCAL|
+ unnest $$ht <- scan-collection($$47)
+ -- UNNEST |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ project ([$$47])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$47] <- [$$p.getField("entities").getField("urls")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.16.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.16.adm
new file mode 100644
index 0000000..0cfbf08
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.16.adm
@@ -0,0 +1 @@
+2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.17.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.17.adm
new file mode 100644
index 0000000..9f122c6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.17.adm
@@ -0,0 +1,39 @@
+distribute result [$$46]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ aggregate [$$46] <- [agg-sql-sum($$52)]
+ -- AGGREGATE |UNPARTITIONED|
+ exchange
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ aggregate [$$52] <- [agg-sql-count(1)]
+ -- AGGREGATE |PARTITIONED|
+ select ($$39)
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$39])
+ -- STREAM_PROJECT |PARTITIONED|
+ subplan {
+ aggregate [$$39] <- [empty-stream()]
+ -- AGGREGATE |LOCAL|
+ select (not(if-missing-or-null(eq($$48, "string"), FALSE)))
+ -- STREAM_SELECT |LOCAL|
+ assign [$$48] <- [$$ht.getField("display_url")]
+ -- ASSIGN |LOCAL|
+ unnest $$ht <- scan-collection($$47)
+ -- UNNEST |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SUBPLAN |PARTITIONED|
+ project ([$$47])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$47] <- [$$p.getField("entities").getField("urls")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset project ({entities:{urls:[{display_url:any}]}})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.18.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.18.adm
new file mode 100644
index 0000000..15eecd2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.18.adm
@@ -0,0 +1,2 @@
+1.1
+1.1
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.19.adm
similarity index 62%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.19.adm
index a5f69e0..37849ca 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.19.adm
@@ -1,26 +1,26 @@
-distribute result [$$15]
+distribute result [$$18]
-- DISTRIBUTE_RESULT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$15])
+ project ([$$18])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$15] <- [object-concat($$18, $$19)]
+ assign [$$18] <- [get-item(get-item(get-item($$21, 0), 0), 0)]
-- ASSIGN |PARTITIONED|
- project ([$$18, $$19])
+ project ([$$21])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
- order (ASC, $$17)
- -- STABLE_SORT [$$17(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$20(ASC) ] |PARTITIONED|
+ order (ASC, $$20)
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$18, $$19, $$17])
+ project ([$$21, $$20])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$19, $$18, $$17] <- [$$p.getField("user"), $$p.getField("coordinates"), $$p.getField("id")]
+ assign [$$21, $$20] <- [$$p.getField("place").getField("bounding_box").getField("coordinates"), $$p.getField("id")]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset project (user, coordinates, id)
+ data-scan []<-[$$p] <- test.ParquetDataset
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.20.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.20.adm
new file mode 100644
index 0000000..15eecd2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.20.adm
@@ -0,0 +1,2 @@
+1.1
+1.1
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.21.adm
similarity index 60%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.21.adm
index a5f69e0..ee3a128 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/array-access-pushdown/array-access-pushdown.21.adm
@@ -1,26 +1,26 @@
-distribute result [$$15]
+distribute result [$$18]
-- DISTRIBUTE_RESULT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$15])
+ project ([$$18])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$15] <- [object-concat($$18, $$19)]
+ assign [$$18] <- [get-item(get-item(get-item($$21, 0), 0), 0)]
-- ASSIGN |PARTITIONED|
- project ([$$18, $$19])
+ project ([$$21])
-- STREAM_PROJECT |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
- order (ASC, $$17)
- -- STABLE_SORT [$$17(ASC)] |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$20(ASC) ] |PARTITIONED|
+ order (ASC, $$20)
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$18, $$19, $$17])
+ project ([$$21, $$20])
-- STREAM_PROJECT |PARTITIONED|
- assign [$$19, $$18, $$17] <- [$$p.getField("user"), $$p.getField("coordinates"), $$p.getField("id")]
+ assign [$$21, $$20] <- [$$p.getField("place").getField("bounding_box").getField("coordinates"), $$p.getField("id")]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset project (user, coordinates, id)
+ data-scan []<-[$$p] <- test.ParquetDataset project ({place:{bounding_box:{coordinates:[[[any]]]}},id:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.02.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.04.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.02.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.03.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.05.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.03.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.04.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.04.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.05.adm
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.05.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.08.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.06.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.08.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.06.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.07.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.07.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.07.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.07.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.08.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.06.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.08.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.09.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.09.adm
similarity index 98%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.09.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.09.adm
index d22d967..2e7bd2b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.09.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.09.adm
@@ -38,7 +38,7 @@ distribute result [$$28]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p2] <- test.ParquetDataset2 project (id)
+ data-scan []<-[$$p2] <- test.ParquetDataset2 project ({id:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.12.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.10.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.12.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.10.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.11.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.11.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.11.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.12.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.10.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.12.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.13.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.13.adm
similarity index 97%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.13.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.13.adm
index afd33c4..12e1b6b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.13.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.13.adm
@@ -28,7 +28,7 @@ distribute result [$$29]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p1] <- test.ParquetDataset project (age, id)
+ data-scan []<-[$$p1] <- test.ParquetDataset project ({id:any,age:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -42,7 +42,7 @@ distribute result [$$29]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p2] <- test.ParquetDataset3 project (name, id)
+ data-scan []<-[$$p2] <- test.ParquetDataset3 project ({name:any,id:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.14.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.16.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.14.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.15.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.15.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.15.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.14.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.16.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.14.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.16.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.17.adm
similarity index 96%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.17.adm
index d812b1f..1ffc9a7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.17.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.17.adm
@@ -24,7 +24,7 @@ distribute result [$$17]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset4 project (user, id)
+ data-scan []<-[$$p] <- test.ParquetDataset4 project ({id:any,user:{name:any,id:any}})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.18.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.18.adm
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.18.adm
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.19.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.19.adm
new file mode 100644
index 0000000..229e53a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.19.adm
@@ -0,0 +1,18 @@
+distribute result [$$31]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ aggregate [$$31] <- [agg-sql-sum($$32)]
+ -- AGGREGATE |UNPARTITIONED|
+ exchange
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ aggregate [$$32] <- [agg-sql-count(1)]
+ -- AGGREGATE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset4
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.20.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.20.adm
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.20.adm
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.21.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.21.adm
new file mode 100644
index 0000000..6937d90
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.21.adm
@@ -0,0 +1,18 @@
+distribute result [$$31]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ aggregate [$$31] <- [agg-sql-sum($$32)]
+ -- AGGREGATE |UNPARTITIONED|
+ exchange
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ aggregate [$$32] <- [agg-sql-count(1)]
+ -- AGGREGATE |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset4 project ({})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.18.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.22.adm
similarity index 96%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.18.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.22.adm
index d812b1f..1ffc9a7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.18.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/field-access-pushdown/field-access-pushdown.22.adm
@@ -24,7 +24,7 @@ distribute result [$$17]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset4 project (user, id)
+ data-scan []<-[$$p] <- test.ParquetDataset4 project ({id:any,user:{name:any,id:any}})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.02.adm
new file mode 100644
index 0000000..1ebe993
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.02.adm
@@ -0,0 +1,6 @@
+[ "1", "2" ]
+[ "3", "4" ]
+[ "5", "6" ]
+"7"
+"8"
+"9"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.03.adm
new file mode 100644
index 0000000..4a602ef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.03.adm
@@ -0,0 +1,30 @@
+distribute result [$$20]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$20])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$20] <- [switch-case(TRUE, is-array($$22), $$24, $$25)]
+ -- ASSIGN |PARTITIONED|
+ project ([$$22, $$24, $$25])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED|
+ order (ASC, $$23)
+ -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$24, $$25] <- [array-star($$22).getField("text"), $$22.getField("text")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$22, $$23])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$22, $$23] <- [$$p.getField("arrayOrObject"), $$p.getField("id")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.04.adm
new file mode 100644
index 0000000..1ebe993
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.04.adm
@@ -0,0 +1,6 @@
+[ "1", "2" ]
+[ "3", "4" ]
+[ "5", "6" ]
+"7"
+"8"
+"9"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.05.adm
new file mode 100644
index 0000000..56dd2ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/heterogeneous-access-pushdown/heterogeneous-access-pushdown.05.adm
@@ -0,0 +1,30 @@
+distribute result [$$20]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$20])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$20] <- [switch-case(TRUE, is-array($$22), $$24, $$25)]
+ -- ASSIGN |PARTITIONED|
+ project ([$$22, $$24, $$25])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$23(ASC) ] |PARTITIONED|
+ order (ASC, $$23)
+ -- STABLE_SORT [$$23(ASC)] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$24, $$25] <- [array-star($$22).getField("text"), $$22.getField("text")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$22, $$23])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$22, $$23] <- [$$p.getField("arrayOrObject"), $$p.getField("id")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset project ({arrayOrObject:<[{text:any}],{text:any}>,id:any})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.3.adm
index a097e74..365218f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.3.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.3.adm
@@ -16,7 +16,7 @@ distribute result [$$16]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset project (coordinates, user, id)
+ data-scan []<-[$$p] <- test.ParquetDataset project ({coordinates:any,id:any,user:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
index a5f69e0..4533f63 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/object-concat/object-concat.5.adm
@@ -20,7 +20,7 @@ distribute result [$$15]
-- ASSIGN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset project (user, coordinates, id)
+ data-scan []<-[$$p] <- test.ParquetDataset project ({coordinates:any,id:any,user:any})
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.02.adm
new file mode 100644
index 0000000..b020ac9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.02.adm
@@ -0,0 +1,50 @@
+distribute result [$$48]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ project ([$$48])
+ -- STREAM_PROJECT |UNPARTITIONED|
+ assign [$$48] <- [{"$1": $$51}]
+ -- ASSIGN |UNPARTITIONED|
+ aggregate [$$51] <- [agg-global-sql-sum($$53)]
+ -- AGGREGATE |UNPARTITIONED|
+ exchange
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ aggregate [$$53] <- [agg-local-sql-sum($$46)]
+ -- AGGREGATE |PARTITIONED|
+ project ([$$46])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$46] <- [object-length($$p1)]
+ -- ASSIGN |PARTITIONED|
+ project ([$$p1])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ join (eq($$49, $$50))
+ -- HYBRID_HASH_JOIN [$$50][$$49] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$50] |PARTITIONED|
+ assign [$$50] <- [$$p1.getField("id")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p1] <- test.ParquetDataset1
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$49] |PARTITIONED|
+ project ([$$49])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$49] <- [$$p2.getField("id")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p2] <- test.ParquetDataset2 project ({id:any})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.03.adm
similarity index 50%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.03.adm
index 5260a0a..47348c2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/expression-pushdown/expression-pushdown.03.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.03.adm
@@ -1,20 +1,20 @@
-distribute result [$$p]
+distribute result [$$p1]
-- DISTRIBUTE_RESULT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$p])
- -- STREAM_PROJECT |PARTITIONED|
+ distinct ([$$p1])
+ -- PRE_SORTED_DISTINCT_BY |PARTITIONED|
exchange
- -- SORT_MERGE_EXCHANGE [$$13(ASC) ] |PARTITIONED|
- order (ASC, $$13)
- -- STABLE_SORT [$$13(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ order (ASC, $$p1)
+ -- STABLE_SORT [$$p1(ASC)] |PARTITIONED|
exchange
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$13] <- [$$p.getField("id")]
- -- ASSIGN |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$p1] |PARTITIONED|
+ select (gt($$p1.getField("id"), 10))
+ -- STREAM_SELECT |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$p] <- test.ParquetDataset
+ data-scan []<-[$$p1] <- test.ParquetDataset1
-- DATASOURCE_SCAN |PARTITIONED|
exchange
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.04.adm
new file mode 100644
index 0000000..d22cd15
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.04.adm
@@ -0,0 +1,50 @@
+distribute result [$$65]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$65])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$65] <- [{"text": $$text, "$1": $$68}]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ group by ([$$text := $$76]) decor ([]) {
+ aggregate [$$68] <- [agg-global-sql-sum($$75)]
+ -- AGGREGATE |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SORT_GROUP_BY[$$76] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$76] |PARTITIONED|
+ group by ([$$76 := $$67]) decor ([]) {
+ aggregate [$$75] <- [agg-local-sql-sum(array-distinct($$70))]
+ -- AGGREGATE |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SORT_GROUP_BY[$$67] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(lowercase($$67), "string"))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$70, $$67])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$67] <- [$$ht.getField("text")]
+ -- ASSIGN |PARTITIONED|
+ unnest $$ht <- scan-collection($$70)
+ -- UNNEST |PARTITIONED|
+ project ([$$70])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$70] <- [$$p1.getField("entities").getField("hashtags")]
+ -- ASSIGN |PARTITIONED|
+ select (gt($$p1.getField("id"), 10))
+ -- STREAM_SELECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p1] <- test.ParquetDataset1 project ({entities:{hashtags:any},id:any})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.05.adm
new file mode 100644
index 0000000..8ad73f9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.05.adm
@@ -0,0 +1,52 @@
+distribute result [$$64]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$64])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$64] <- [{"text": $$text, "$1": $$67}]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ group by ([$$text := $$74]) decor ([]) {
+ aggregate [$$67] <- [agg-global-sql-sum($$73)]
+ -- AGGREGATE |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SORT_GROUP_BY[$$74] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$74] |PARTITIONED|
+ group by ([$$74 := $$66]) decor ([]) {
+ aggregate [$$73] <- [agg-local-sql-sum(sql-sum($$70))]
+ -- AGGREGATE |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SORT_GROUP_BY[$$66] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(lowercase($$66), "string"))
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$70, $$66])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$70, $$66] <- [$$ht.getField("indices"), $$ht.getField("text")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$ht])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$ht <- scan-collection($$69)
+ -- UNNEST |PARTITIONED|
+ project ([$$69])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$69] <- [$$p1.getField("entities").getField("hashtags")]
+ -- ASSIGN |PARTITIONED|
+ select (gt($$p1.getField("id"), 10))
+ -- STREAM_SELECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p1] <- test.ParquetDataset1 project ({entities:{hashtags:[{indices:any,text:any}]},id:any})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.adm
new file mode 100644
index 0000000..cac9db7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.06.adm
@@ -0,0 +1,26 @@
+distribute result [$$21]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ limit 10
+ -- STREAM_LIMIT |UNPARTITIONED|
+ exchange
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ project ([$$21])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$21] <- [{"display_url": get-item($$22, 0).getField("display_url")}]
+ -- ASSIGN |PARTITIONED|
+ limit 10
+ -- STREAM_LIMIT |PARTITIONED|
+ project ([$$22])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$22] <- [$$p.getField("entities").getField("urls")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p] <- test.ParquetDataset1 condition (gt(sql-count($$p.getField("entities").getField("urls")), 10)) limit 10 project ({entities:{urls:any}})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.07.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.07.adm
new file mode 100644
index 0000000..30d5ac3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/pushdown-plans/pushdown-plans.07.adm
@@ -0,0 +1,113 @@
+distribute result [$$94]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ limit 10
+ -- STREAM_LIMIT |UNPARTITIONED|
+ project ([$$94])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$94] <- [{"uname": $$uname, "cnt": $$96}]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$96(DESC) ] |PARTITIONED|
+ limit 10
+ -- STREAM_LIMIT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ order (topK: 10) (DESC, $$96)
+ -- STABLE_SORT [topK: 10] [$$96(DESC)] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ group by ([$$uname := $$107]) decor ([]) {
+ aggregate [$$96] <- [agg-sql-sum($$106)]
+ -- AGGREGATE |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SORT_GROUP_BY[$$107] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$107] |PARTITIONED|
+ group by ([$$107 := $$95]) decor ([]) {
+ aggregate [$$106] <- [agg-sql-count(1)]
+ -- AGGREGATE |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- SORT_GROUP_BY[$$95] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$95])
+ -- STREAM_PROJECT |PARTITIONED|
+ select ($$85)
+ -- STREAM_SELECT |PARTITIONED|
+ project ([$$85, $$95])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ group by ([$$105 := $$103]) decor ([$$95]) {
+ aggregate [$$85] <- [non-empty-stream()]
+ -- AGGREGATE |LOCAL|
+ select (not(is-missing($$104)))
+ -- STREAM_SELECT |LOCAL|
+ nested tuple source
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- PRE_CLUSTERED_GROUP_BY[$$103] |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ order (ASC, $$103)
+ -- STABLE_SORT [$$103(ASC)] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$103] |PARTITIONED|
+ project ([$$95, $$104, $$103])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ left outer join (eq($$97, $$81))
+ -- HYBRID_HASH_JOIN [$$97][$$81] |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$97] |PARTITIONED|
+ running-aggregate [$$103] <- [create-query-uid()]
+ -- RUNNING_AGGREGATE |PARTITIONED|
+ project ([$$95, $$97])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$97] <- [$$ht1.getField("text")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$95, $$ht1])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$ht1 <- scan-collection($$98)
+ -- UNNEST |PARTITIONED|
+ project ([$$98, $$95])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$98, $$95] <- [$$p1.getField("entities").getField("hashtags"), $$p1.getField("user").getField("name")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p1] <- test.ParquetDataset1 project ({entities:{hashtags:[{text:any}]},user:{name:any}})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange
+ -- HASH_PARTITION_EXCHANGE [$$81] |PARTITIONED|
+ project ([$$104, $$81])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$104, $$81] <- [TRUE, $$ht2.getField("text")]
+ -- ASSIGN |PARTITIONED|
+ project ([$$ht2])
+ -- STREAM_PROJECT |PARTITIONED|
+ unnest $$ht2 <- scan-collection($$99)
+ -- UNNEST |PARTITIONED|
+ project ([$$99])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$99] <- [$$p2.getField("entities").getField("hashtags")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$p2] <- test.ParquetDataset2 project ({entities:{hashtags:[{text:any}]}})
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.02.adm
new file mode 100644
index 0000000..0be5d98
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.02.adm
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.03.adm
new file mode 100644
index 0000000..0be5d98
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.03.adm
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.04.adm
new file mode 100644
index 0000000..0be5d98
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/parquet/type-mismatch/type-mismatch.04.adm
@@ -0,0 +1,2 @@
+true
+true
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index c461722..6178282 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -96,9 +96,15 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
- <compilation-unit name="common/parquet/expression-pushdown">
+ <compilation-unit name="common/parquet/field-access-pushdown">
<placeholder name="adapter" value="S3" />
- <output-dir compare="Text">common/parquet/expression-pushdown</output-dir>
+ <output-dir compare="Text">common/parquet/field-access-pushdown</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/parquet/array-access-pushdown">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/parquet/array-access-pushdown</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
@@ -119,6 +125,27 @@
<output-dir compare="Text">common/parquet/object-concat</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset" check-warnings="true">
+ <compilation-unit name="common/parquet/type-mismatch">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/parquet/type-mismatch</output-dir>
+ <expected-warn>ASX0002: Type mismatch: function field-access-by-name expects its 1st input parameter to be of type object, but the actual input type is array (in line 31, at column 29)</expected-warn>
+ <expected-warn>ASX0002: Type mismatch: function array-star expects its 1st input parameter to be of type array, but the actual input type is object (in line 31, at column 24)</expected-warn>
+ <expected-warn>ASX0002: Type mismatch: function field-access-by-name expects its 1st input parameter to be of type object, but the actual input type is array (in line 31, at column 52)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/parquet/heterogeneous-access-pushdown">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/parquet/heterogeneous-access-pushdown</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="common/parquet/pushdown-plans">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">common/parquet/pushdown-plans</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/s3">
<compilation-unit name="parquet-anonymous-access">
<output-dir compare="Text">parquet-anonymous-access</output-dir>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
index 469677c..f15f735 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
@@ -86,16 +86,16 @@ public class HDFSDataSourceFactory implements IRecordReaderFactory<Object>, IInd
@Override
public void configure(IServiceContext serviceCtx, Map<String, String> configuration,
IWarningCollector warningCollector) throws AlgebricksException, HyracksDataException {
- JobConf hdfsConf = createHdfsConf(serviceCtx, configuration);
+ JobConf hdfsConf = createHdfsConf(serviceCtx, configuration, warningCollector.shouldWarn());
configureHdfsConf(hdfsConf, configuration);
}
- protected JobConf createHdfsConf(IServiceContext serviceCtx, Map<String, String> configuration)
+ protected JobConf createHdfsConf(IServiceContext serviceCtx, Map<String, String> configuration, boolean shouldWarn)
throws HyracksDataException {
this.serviceCtx = serviceCtx;
this.configuration = configuration;
init((ICCServiceContext) serviceCtx);
- return HDFSUtils.configureHDFSJobConf(configuration);
+ return HDFSUtils.configureHDFSJobConf(configuration, shouldWarn);
}
protected void configureHdfsConf(JobConf conf, Map<String, String> configuration) throws AlgebricksException {
@@ -229,7 +229,19 @@ public class HDFSDataSourceFactory implements IRecordReaderFactory<Object>, IInd
}
}
restoreConfig(ctx);
- return createRecordReader(configuration, read, inputSplits, readSchedule, nodeName, conf, files, indexer);
+ JobConf readerConf = conf;
+ if (ctx.getWarningCollector().shouldWarn()
+ && configuration.get(ExternalDataConstants.KEY_INPUT_FORMAT.trim())
+ .equals(ExternalDataConstants.INPUT_FORMAT_PARQUET)) {
+ /*
+ * JobConf is used to pass warnings from the ParquetReadSupport to ParquetReader. As multiple
+ * partitions can issue different warnings, we might have a race condition on JobConf. Thus, we
+ * should create a copy when warnings are enabled.
+ */
+ readerConf = confFactory.getConf();
+ }
+ return createRecordReader(configuration, read, inputSplits, readSchedule, nodeName, readerConf, files,
+ indexer, ctx.getWarningCollector());
} catch (Exception e) {
throw HyracksDataException.create(e);
}
@@ -257,10 +269,10 @@ public class HDFSDataSourceFactory implements IRecordReaderFactory<Object>, IInd
private static IRecordReader<? extends Object> createRecordReader(Map<String, String> configuration, boolean[] read,
InputSplit[] inputSplits, String[] readSchedule, String nodeName, JobConf conf, List<ExternalFile> files,
- IExternalIndexer indexer) throws IOException {
+ IExternalIndexer indexer, IWarningCollector warningCollector) throws IOException {
if (configuration.get(ExternalDataConstants.KEY_INPUT_FORMAT.trim())
.equals(ExternalDataConstants.INPUT_FORMAT_PARQUET)) {
- return new ParquetFileRecordReader<>(read, inputSplits, readSchedule, nodeName, conf);
+ return new ParquetFileRecordReader<>(read, inputSplits, readSchedule, nodeName, conf, warningCollector);
} else {
return new HDFSRecordReader<>(read, inputSplits, readSchedule, nodeName, conf, files, indexer);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
index 803e657..85d8671 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
@@ -50,7 +50,7 @@ public class AwsS3ParquetReaderFactory extends HDFSDataSourceFactory {
putS3ConfToHadoopConf(configuration, path);
//Configure Hadoop S3 input splits
- JobConf conf = createHdfsConf(serviceCtx, configuration);
+ JobConf conf = createHdfsConf(serviceCtx, configuration, warningCollector.shouldWarn());
int numberOfPartitions = getPartitionConstraint().getLocations().length;
ExternalDataUtils.AwsS3.configureAwsS3HdfsJobConf(conf, configuration, numberOfPartitions);
configureHdfsConf(conf, configuration);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java
index 75d431d..8fbe5c4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/HDFSLookupReaderFactory.java
@@ -65,7 +65,7 @@ public class HDFSLookupReaderFactory<T> implements ILookupReaderFactory<T> {
IWarningCollector warningCollector) throws AsterixException {
this.serviceCtx = serviceCtx;
this.configuration = configuration;
- JobConf conf = HDFSUtils.configureHDFSJobConf(configuration);
+ JobConf conf = HDFSUtils.configureHDFSJobConf(configuration, warningCollector.shouldWarn());
try {
confFactory = new ConfFactory(conf);
} catch (HyracksDataException e) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AbstractComplexConverter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AbstractComplexConverter.java
index 81b61f8..363d2d2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AbstractComplexConverter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AbstractComplexConverter.java
@@ -26,11 +26,11 @@ import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.parquet.io.api.Converter;
import org.apache.parquet.io.api.GroupConverter;
import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Type.Repetition;
-abstract class AbstractComplexConverter extends GroupConverter implements IFieldValue {
+public abstract class AbstractComplexConverter extends GroupConverter implements IFieldValue {
protected final AbstractComplexConverter parent;
private final IValueReference fieldName;
private final int index;
@@ -38,12 +38,11 @@ abstract class AbstractComplexConverter extends GroupConverter implements IField
protected final ParserContext context;
protected IMutableValueStorage tempStorage;
- public AbstractComplexConverter(AbstractComplexConverter parent, int index, GroupType parquetType,
- ParserContext context) {
+ AbstractComplexConverter(AbstractComplexConverter parent, int index, GroupType parquetType, ParserContext context) {
this(parent, null, index, parquetType, context);
}
- public AbstractComplexConverter(AbstractComplexConverter parent, IValueReference fieldName, int index,
+ AbstractComplexConverter(AbstractComplexConverter parent, IValueReference fieldName, int index,
GroupType parquetType, ParserContext context) {
this.parent = parent;
this.fieldName = fieldName;
@@ -52,9 +51,11 @@ abstract class AbstractComplexConverter extends GroupConverter implements IField
converters = new Converter[parquetType.getFieldCount()];
for (int i = 0; i < parquetType.getFieldCount(); i++) {
final Type type = parquetType.getType(i);
- if (type.isPrimitive()) {
+ if (type == AsterixTypeToParquetTypeVisitor.MISSING) {
+ converters[i] = MissingConverter.INSTANCE;
+ } else if (type.isPrimitive()) {
converters[i] = createAtomicConverter(parquetType, i);
- } else if (type.getOriginalType() == OriginalType.LIST) {
+ } else if (LogicalTypeAnnotation.listType().equals(type.getLogicalTypeAnnotation())) {
converters[i] = createArrayConverter(parquetType, i);
} else if (type.getRepetition() == Repetition.REPEATED) {
converters[i] = createRepeatedConverter(parquetType, i);
@@ -86,7 +87,6 @@ abstract class AbstractComplexConverter extends GroupConverter implements IField
*
* In Avro:
* optional group urls (LIST) {
- * // if number of fields > 1, then should be treated as array of objects
* repeated group array {
* optional binary display_url (UTF8);
* optional binary expanded_url (UTF8);
@@ -106,12 +106,13 @@ abstract class AbstractComplexConverter extends GroupConverter implements IField
*
* @formatter:on
*/
- public AbstractComplexConverter createRepeatedConverter(GroupType type, int index) {
- final GroupType arrayType = type.getType(index).asGroupType();
- if (arrayType.getFieldCount() == 1) {
- return new RepeatedConverter(this, index, arrayType, context);
+ private AbstractComplexConverter createRepeatedConverter(GroupType type, int index) {
+ GroupType repeatedType = type.getType(index).asGroupType();
+ //The name "array" is used by Avro to represent group element (array of objects)
+ if (repeatedType.getFieldCount() > 1 || "array".equals(repeatedType.getName())) {
+ return new ObjectConverter(this, index, repeatedType, context);
}
- return new ObjectConverter(this, index, arrayType, context);
+ return new RepeatedConverter(this, index, repeatedType, context);
}
@Override
@@ -149,5 +150,4 @@ abstract class AbstractComplexConverter extends GroupConverter implements IField
}
parent.addValue(this);
}
-
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
new file mode 100644
index 0000000..d596493
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/AsterixTypeToParquetTypeVisitor.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.hdfs.parquet;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
+import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+
+/**
+ * This visitor clips the filesSchema stored in Parquet using the expected type
+ */
+public class AsterixTypeToParquetTypeVisitor implements IATypeVisitor<Type, Type> {
+ public static final MessageType EMPTY_PARQUET_MESSAGE = Types.buildMessage().named("EMPTY");
+ public static final PrimitiveType MISSING =
+ Types.optional(PrimitiveType.PrimitiveTypeName.BOOLEAN).named("MISSING");
+
+ private final List<Warning> warnings;
+ private Map<String, FunctionCallInformation> funcInfo;
+
+ public AsterixTypeToParquetTypeVisitor(boolean shouldWarn) {
+ warnings = shouldWarn ? new ArrayList<>() : null;
+ }
+
+ public MessageType clipType(ARecordType rootType, MessageType fileSchema,
+ Map<String, FunctionCallInformation> funcInfo) {
+ if (rootType == DataProjectionInfo.EMPTY_TYPE) {
+ return EMPTY_PARQUET_MESSAGE;
+ } else if (rootType == DataProjectionInfo.ALL_FIELDS_TYPE) {
+ return fileSchema;
+ }
+ Types.MessageTypeBuilder builder = Types.buildMessage();
+ this.funcInfo = funcInfo;
+ clipObjectChildren(builder, rootType, fileSchema);
+ return builder.named(fileSchema.getName());
+ }
+
+ public List<Warning> getWarnings() {
+ return warnings;
+ }
+
+ @Override
+ public Type visit(ARecordType recordType, Type arg) {
+ //No LogicalTypeAnnotation for Object types
+ if (isNotCompatibleType(arg, recordType)) {
+ return MISSING;
+ }
+ Types.GroupBuilder<GroupType> builder = Types.buildGroup(arg.getRepetition());
+ if (clipObjectChildren(builder, recordType, arg) == 0) {
+ //If no fields where added, add MISSING as parquet does not allow empty objects.
+ builder.addField(MISSING);
+ }
+ return builder.named(arg.getName());
+ }
+
+ /**
+ * There are two ways for representing arrays using ProtoBuf schema see the example in
+ * {@link org.apache.asterix.external.input.record.reader.hdfs.parquet.AbstractComplexConverter} for more
+ * information.
+ */
+ @Override
+ public Type visit(AbstractCollectionType collectionType, Type arg) {
+ if (isNotCompatibleType(arg, collectionType)) {
+ return MISSING;
+ }
+ GroupType arrayType = arg.asGroupType();
+ //There is only one child
+ Type childType = arrayType.getType(0);
+ if ("array".equals(childType.getName()) || childType.asGroupType().getFieldCount() > 1) {
+ //Handle Avro-like schema
+ return handleHandleAvroArray(collectionType, arrayType);
+ }
+ //Handling spark-like schema
+ Types.ListBuilder<GroupType> builder = Types.list(arg.getRepetition());
+ //LIST items always wrapped in a group called list
+ childType = childType.asGroupType().getType(0);
+ Type requestedChildType = collectionType.getItemType().accept(this, childType);
+ //setElementType will wrap childType again in a group type
+ builder.setElementType(requestedChildType);
+ return builder.named(arg.getName());
+ }
+
+ private int clipObjectChildren(Types.GroupBuilder<?> builder, ARecordType recordType, Type arg) {
+ GroupType groupType = arg.asGroupType();
+ String[] fieldNames = recordType.getFieldNames();
+ IAType[] fieldTypes = recordType.getFieldTypes();
+ int numberOfAddedFields = 0;
+ for (int i = 0; i < fieldNames.length; i++) {
+ Type type = getType(groupType, fieldNames[i]);
+ Type childType = fieldTypes[i].accept(this, type);
+ if (childType != MISSING) {
+ //We only add non-MISSING children
+ builder.addField(childType);
+ numberOfAddedFields++;
+ }
+ }
+ return numberOfAddedFields;
+ }
+
+ private Type handleHandleAvroArray(AbstractCollectionType collectionType, GroupType groupType) {
+ Types.GroupBuilder<GroupType> builder =
+ Types.buildGroup(groupType.getRepetition()).as(groupType.getLogicalTypeAnnotation());
+ //There is only one child
+ Type type = groupType.getType(0);
+ Type childType = collectionType.getItemType().accept(this, type);
+ builder.addField(childType);
+ return builder.named(groupType.getName());
+ }
+
+ @Override
+ public Type visit(AUnionType unionType, Type arg) {
+ if (arg.getLogicalTypeAnnotation() == LogicalTypeAnnotation.listType()) {
+ //Currently, only array type is supported
+ return unionType.getType(ATypeTag.ARRAY).accept(this, arg);
+ } else {
+ return unionType.getType(ATypeTag.OBJECT).accept(this, arg);
+ }
+ }
+
+ @Override
+ public Type visitFlat(IAType node, Type arg) {
+ return arg;
+ }
+
+ private boolean isNotCompatibleType(Type type, IAType node) {
+ if (type == MISSING) {
+ return true;
+ }
+ ATypeTag actualType = mapType(type);
+ ATypeTag expectedType = node.getTypeTag();
+
+ boolean isNotExpected = actualType != expectedType;
+ if (warnings != null && isNotExpected) {
+ //typeName is unique
+ FunctionCallInformation info = funcInfo.get(node.getTypeName());
+ //If no warning is created, then it means it has been reported
+ Warning warning = info.createTypeMismatchWarning(expectedType, actualType);
+ if (warning != null) {
+ //New warning that we saw for the first time. We should report it.
+ warnings.add(warning);
+ }
+ }
+ return isNotExpected;
+ }
+
+ private static ATypeTag mapType(Type parquetType) {
+ LogicalTypeAnnotation typeAnnotation = parquetType.getLogicalTypeAnnotation();
+ if (!parquetType.isPrimitive() && typeAnnotation == null) {
+ return ATypeTag.OBJECT;
+ } else if (typeAnnotation == LogicalTypeAnnotation.listType()) {
+ return ATypeTag.ARRAY;
+ } else if (typeAnnotation == LogicalTypeAnnotation.stringType()) {
+ return ATypeTag.STRING;
+ } else {
+ //Check other primitive types
+ PrimitiveType.PrimitiveTypeName primitiveTypeName = parquetType.asPrimitiveType().getPrimitiveTypeName();
+ switch (primitiveTypeName) {
+ case BOOLEAN:
+ return ATypeTag.BOOLEAN;
+ case INT32:
+ case INT64:
+ return ATypeTag.BIGINT;
+ case FLOAT:
+ case DOUBLE:
+ return ATypeTag.DOUBLE;
+ default:
+ throw new IllegalStateException("Unsupported type " + parquetType);
+ }
+ }
+ }
+
+ private static Type getType(GroupType groupType, String fieldName) {
+ if (groupType.containsField(fieldName)) {
+ return groupType.getType(fieldName);
+ }
+ return MISSING;
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/MissingConverter.java
similarity index 50%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
rename to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/MissingConverter.java
index d51e173..e38056b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/parquet/expression-pushdown/expression-pushdown.02.query.sqlpp
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/MissingConverter.java
@@ -16,16 +16,45 @@
* specific language governing permissions and limitations
* under the License.
*/
-/*
- * Description : Ignore Field Access pushdown when requesting all fields
- * Expected Res : Success
- * Date : June 22nd 2020
- */
+package org.apache.asterix.external.input.record.reader.hdfs.parquet;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.PrimitiveConverter;
+
+class MissingConverter extends PrimitiveConverter {
+ protected static final MissingConverter INSTANCE = new MissingConverter();
+
+ private MissingConverter() {
+ }
+
+ @Override
+ public void addBinary(Binary value) {
+ //NoOp
+ }
+
+ @Override
+ public void addBoolean(boolean value) {
+ //NoOp
+ }
+
+ @Override
+ public void addFloat(float value) {
+ //NoOp
+ }
+
+ @Override
+ public void addDouble(double value) {
+ //NoOp
+ }
-USE test;
+ @Override
+ public void addInt(int value) {
+ //NoOp
+ }
-SET `compiler.external.field.pushdown` "false";
+ @Override
+ public void addLong(long value) {
+ //NoOp
+ }
-SELECT VALUE p
-FROM ParquetDataset p
-ORDER BY p.id;
\ No newline at end of file
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetFileRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetFileRecordReader.java
index e208240..b06afd3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetFileRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetFileRecordReader.java
@@ -22,10 +22,12 @@ import java.io.IOException;
import org.apache.asterix.external.input.record.ValueReferenceRecord;
import org.apache.asterix.external.input.record.reader.hdfs.AbstractHDFSRecordReader;
+import org.apache.asterix.external.util.HDFSUtils;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.data.std.api.IValueReference;
/**
@@ -33,10 +35,12 @@ import org.apache.hyracks.data.std.api.IValueReference;
* The reader returns records in ADM format.
*/
public class ParquetFileRecordReader<V extends IValueReference> extends AbstractHDFSRecordReader<Void, V> {
+ private final IWarningCollector warningCollector;
public ParquetFileRecordReader(boolean[] read, InputSplit[] inputSplits, String[] readSchedule, String nodeName,
- JobConf conf) {
+ JobConf conf, IWarningCollector warningCollector) {
super(read, inputSplits, readSchedule, nodeName, new ValueReferenceRecord<>(), conf);
+ this.warningCollector = warningCollector;
}
@Override
@@ -44,6 +48,15 @@ public class ParquetFileRecordReader<V extends IValueReference> extends Abstract
return false;
}
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (warningCollector.shouldWarn()) {
+ //report warnings
+ HDFSUtils.issueWarnings(warningCollector, conf);
+ }
+ }
+
@SuppressWarnings("unchecked")
@Override
protected RecordReader<Void, V> getRecordReader(int splitIndex) throws IOException {
@@ -53,5 +66,4 @@ public class ParquetFileRecordReader<V extends IValueReference> extends Abstract
}
return reader;
}
-
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetReadSupport.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetReadSupport.java
index 814da3c..b2a5eeb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetReadSupport.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/ParquetReadSupport.java
@@ -18,33 +18,28 @@
*/
package org.apache.asterix.external.input.record.reader.hdfs.parquet;
+import java.io.IOException;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.HDFSUtils;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.parquet.hadoop.api.InitContext;
import org.apache.parquet.hadoop.api.ReadSupport;
import org.apache.parquet.io.api.GroupConverter;
import org.apache.parquet.io.api.RecordMaterializer;
-import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
-import org.apache.parquet.schema.Type;
-import org.apache.parquet.schema.Type.Repetition;
-import org.apache.parquet.schema.Types;
-import org.apache.parquet.schema.Types.GroupBuilder;
-import org.apache.parquet.schema.Types.MessageTypeBuilder;
public class ParquetReadSupport extends ReadSupport<IValueReference> {
- private static final PrimitiveType NULL = Types.optional(PrimitiveTypeName.BOOLEAN).named("NULL");
-
@Override
public ReadContext init(InitContext context) {
- final String requestedSchemaString = context.getConfiguration().get(ExternalDataConstants.KEY_REQUESTED_FIELDS);
- final MessageType requestedSchema = getRequestedSchema(requestedSchemaString, context.getFileSchema());
+ MessageType requestedSchema = getRequestedSchema(context);
return new ReadContext(requestedSchema, Collections.emptyMap());
}
@@ -54,6 +49,30 @@ public class ParquetReadSupport extends ReadSupport<IValueReference> {
return new ADMRecordMaterializer(readContext);
}
+ private static MessageType getRequestedSchema(InitContext context) {
+ Configuration configuration = context.getConfiguration();
+ MessageType fileSchema = context.getFileSchema();
+ boolean shouldWarn = configuration.getBoolean(ExternalDataConstants.KEY_HADOOP_ASTERIX_WARNINGS_ENABLED, false);
+ AsterixTypeToParquetTypeVisitor visitor = new AsterixTypeToParquetTypeVisitor(shouldWarn);
+ try {
+ ARecordType expectedType = HDFSUtils.getExpectedType(configuration);
+ Map<String, FunctionCallInformation> functionCallInformationMap =
+ HDFSUtils.getFunctionCallInformationMap(configuration);
+ MessageType requestedType = visitor.clipType(expectedType, fileSchema, functionCallInformationMap);
+ List<Warning> warnings = visitor.getWarnings();
+
+ if (shouldWarn && !warnings.isEmpty()) {
+ //New warnings were created, set the warnings in hadoop configuration to be reported
+ HDFSUtils.setWarnings(warnings, configuration);
+ //Update the reported warnings so that we do not report the same warning again
+ HDFSUtils.setFunctionCallInformationMap(functionCallInformationMap, configuration);
+ }
+ return requestedType;
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
private static class ADMRecordMaterializer extends RecordMaterializer<IValueReference> {
private final RootConverter rootConverter;
@@ -70,43 +89,5 @@ public class ParquetReadSupport extends ReadSupport<IValueReference> {
public GroupConverter getRootConverter() {
return rootConverter;
}
-
- }
-
- private static MessageType getRequestedSchema(String requestedSchemaString, MessageType fileSchema) {
- if ("*".equals(requestedSchemaString)) {
- return fileSchema;
- }
-
- final MessageTypeBuilder builder = Types.buildMessage();
- final String[] paths = requestedSchemaString.split(",");
- for (int i = 0; i < paths.length; i++) {
- buildRequestedType(paths[i].trim().split("[.]"), builder, fileSchema, 0);
- }
-
- return builder.named("asterix");
-
- }
-
- private static void buildRequestedType(String[] fieldNames, GroupBuilder<?> builder, GroupType groupType,
- int start) {
- final String fieldName = fieldNames[start].trim();
-
- Type type = getType(groupType, fieldName);
- if (type != NULL && start < fieldNames.length - 1) {
- final GroupBuilder<GroupType> innerFieldBuilder = Types.buildGroup(Repetition.OPTIONAL);
- buildRequestedType(fieldNames, innerFieldBuilder, type.asGroupType(), start + 1);
- builder.addField(innerFieldBuilder.named(fieldName));
- } else {
- builder.addField(type);
- }
}
-
- private static Type getType(GroupType groupType, String fieldName) {
- if (groupType.containsField(fieldName)) {
- return groupType.getType(fieldName);
- }
- return NULL;
- }
-
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 3f58bec..243b87c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -60,6 +60,11 @@ public class ExternalDataConstants {
public static final String KEY_HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit";
public static final String KEY_HADOOP_SOCKET_PATH = "dfs.domain.socket.path";
public static final String KEY_HADOOP_BUFFER_SIZE = "io.file.buffer.size";
+ public static final String KEY_HADOOP_ASTERIX_WARNINGS_ENABLED = "org.apache.asterix.warnings.enabled";
+ //Base64 encoded warnings issued from Hadoop
+ public static final String KEY_HADOOP_ASTERIX_WARNINGS_LIST = "org.apache.asterix.warnings.list";
+ //Base64 encoded function call information
+ public static final String KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION = "org.apache.asterix.function.info";
public static final String KEY_SOURCE_DATATYPE = "type-name";
public static final String KEY_DELIMITER = "delimiter";
public static final String KEY_PARSER_FACTORY = "parser-factory";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 08d803e..dd44436 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -54,6 +54,8 @@ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_
import static org.apache.asterix.runtime.evaluators.functions.StringEvaluatorUtils.RESERVED_REGEX_CHARS;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
@@ -62,6 +64,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Base64;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
@@ -91,6 +94,8 @@ import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.runtime.evaluators.common.NumberUtils;
+import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
@@ -807,6 +812,48 @@ public class ExternalDataUtils {
|| ExternalDataConstants.FORMAT_PARQUET.equals(properties.get(ExternalDataConstants.KEY_FORMAT));
}
+ public static void setExternalDataProjectionInfo(DataProjectionInfo projectionInfo, Map<String, String> properties)
+ throws IOException {
+ properties.put(ExternalDataConstants.KEY_REQUESTED_FIELDS,
+ serializeExpectedTypeToString(projectionInfo.getProjectionInfo()));
+ properties.put(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION,
+ serializeFunctionCallInfoToString(projectionInfo.getFunctionCallInfoMap()));
+ }
+
+ /**
+ * Serialize {@link ARecordType} as Base64 string to pass it to {@link org.apache.hadoop.conf.Configuration}
+ *
+ * @param expectedType expected type
+ * @return the expected type as Base64 string
+ */
+ private static String serializeExpectedTypeToString(ARecordType expectedType) throws IOException {
+ if (expectedType == DataProjectionInfo.EMPTY_TYPE || expectedType == DataProjectionInfo.ALL_FIELDS_TYPE) {
+ //Return the type name of EMPTY_TYPE and ALL_FIELDS_TYPE
+ return expectedType.getTypeName();
+ }
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+ Base64.Encoder encoder = Base64.getEncoder();
+ DataProjectionInfo.writeTypeField(expectedType, dataOutputStream);
+ return encoder.encodeToString(byteArrayOutputStream.toByteArray());
+ }
+
+ /**
+ * Serialize {@link FunctionCallInformation} map as Base64 string to pass it to
+ * {@link org.apache.hadoop.conf.Configuration}
+ *
+ * @param functionCallInfoMap function information map
+ * @return function information map as Base64 string
+ */
+ static String serializeFunctionCallInfoToString(Map<String, FunctionCallInformation> functionCallInfoMap)
+ throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+ Base64.Encoder encoder = Base64.getEncoder();
+ DataProjectionInfo.writeFunctionCallInformationMapField(functionCallInfoMap, dataOutputStream);
+ return encoder.encodeToString(byteArrayOutputStream.toByteArray());
+ }
+
public static class AwsS3 {
private AwsS3() {
throw new AssertionError("do not instantiate");
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
index f6d9d5a..e774b40 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java
@@ -18,9 +18,14 @@
*/
package org.apache.asterix.external.util;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.List;
import java.util.Map;
@@ -35,6 +40,10 @@ import org.apache.asterix.external.indexing.RecordId.RecordIdType;
import org.apache.asterix.external.input.record.reader.hdfs.parquet.MapredParquetInputFormat;
import org.apache.asterix.external.input.record.reader.hdfs.parquet.ParquetReadSupport;
import org.apache.asterix.external.input.stream.HDFSInputStream;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.DataProjectionInfo;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -50,6 +59,8 @@ import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.context.ICCContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.hdfs.scheduler.Scheduler;
import org.apache.parquet.hadoop.ParquetInputFormat;
@@ -190,11 +201,11 @@ public class HDFSUtils {
}
}
- public static JobConf configureHDFSJobConf(Map<String, String> configuration) {
+ public static JobConf configureHDFSJobConf(Map<String, String> configuration, boolean shouldWarn) {
JobConf conf = new JobConf();
String localShortCircuitSocketPath = configuration.get(ExternalDataConstants.KEY_LOCAL_SOCKET_PATH);
String formatClassName = HDFSUtils.getInputFormatClassName(configuration);
- final String url = configuration.get(ExternalDataConstants.KEY_HDFS_URL);
+ String url = configuration.get(ExternalDataConstants.KEY_HDFS_URL);
//Allow hdfs adapter to read from local-files. However, this only works in a single-node configuration.
if (url != null && url.trim().startsWith("hdfs")) {
@@ -215,9 +226,22 @@ public class HDFSUtils {
if (ExternalDataConstants.CLASS_NAME_PARQUET_INPUT_FORMAT.equals(formatClassName)) {
//Parquet configurations
conf.set(ParquetInputFormat.READ_SUPPORT_CLASS, ParquetReadSupport.class.getName());
- //Set the requested fields. Default * which means all fields
- final String requestedFields = configuration.get(ExternalDataConstants.KEY_REQUESTED_FIELDS);
- conf.set(ExternalDataConstants.KEY_REQUESTED_FIELDS, requestedFields != null ? requestedFields : "*");
+ //Get requested values
+ String requestedValues = configuration.get(ExternalDataConstants.KEY_REQUESTED_FIELDS);
+ if (requestedValues == null) {
+ //No value is requested, return the entire record
+ requestedValues = DataProjectionInfo.ALL_FIELDS_TYPE.getTypeName();
+ } else {
+ //Subset of the values were requested, set the functionCallInformation
+ conf.set(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION,
+ configuration.get(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION));
+ /*
+ * Allows Parquet to issue warnings in case we found type mismatches (if warnings are enabled).
+ * Warnings will be issued during the type matching of Parquet's schema with the requested schema
+ */
+ conf.setBoolean(ExternalDataConstants.KEY_HADOOP_ASTERIX_WARNINGS_ENABLED, shouldWarn);
+ }
+ conf.set(ExternalDataConstants.KEY_REQUESTED_FIELDS, requestedValues);
}
return conf;
@@ -242,4 +266,74 @@ public class HDFSUtils {
return null;
}
}
+
+ public static ARecordType getExpectedType(Configuration configuration) throws IOException {
+ String encoded = configuration.get(ExternalDataConstants.KEY_REQUESTED_FIELDS, "");
+ if (encoded.isEmpty() || encoded.equals(DataProjectionInfo.ALL_FIELDS_TYPE.getTypeName())) {
+ //By default, return the entire records
+ return DataProjectionInfo.ALL_FIELDS_TYPE;
+ } else if (encoded.equals(DataProjectionInfo.EMPTY_TYPE.getTypeName())) {
+ //No fields were requested
+ return DataProjectionInfo.EMPTY_TYPE;
+ }
+ //A subset of the fields was requested
+ Base64.Decoder decoder = Base64.getDecoder();
+ byte[] typeBytes = decoder.decode(encoded);
+ DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(typeBytes));
+ return DataProjectionInfo.createTypeField(dataInputStream);
+ }
+
+ public static void setFunctionCallInformationMap(Map<String, FunctionCallInformation> funcCallInfoMap,
+ Configuration conf) throws IOException {
+ String stringFunctionCallInfoMap = ExternalDataUtils.serializeFunctionCallInfoToString(funcCallInfoMap);
+ conf.set(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION, stringFunctionCallInfoMap);
+ }
+
+ public static Map<String, FunctionCallInformation> getFunctionCallInformationMap(Configuration conf)
+ throws IOException {
+ String encoded = conf.get(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION, "");
+ if (!encoded.isEmpty()) {
+ Base64.Decoder decoder = Base64.getDecoder();
+ byte[] functionCallInfoMapBytes = decoder.decode(encoded);
+ DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(functionCallInfoMapBytes));
+ return DataProjectionInfo.createFunctionCallInformationMap(dataInputStream);
+ }
+ return null;
+ }
+
+ public static void setWarnings(List<Warning> warnings, Configuration conf) throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+
+ StringBuilder stringBuilder = new StringBuilder();
+ Base64.Encoder encoder = Base64.getEncoder();
+ for (int i = 0; i < warnings.size(); i++) {
+ Warning warning = warnings.get(i);
+ warning.writeFields(dataOutputStream);
+ stringBuilder.append(encoder.encodeToString(byteArrayOutputStream.toByteArray()));
+ //Warnings are separated by ','
+ stringBuilder.append(',');
+ byteArrayOutputStream.reset();
+ }
+ conf.set(ExternalDataConstants.KEY_HADOOP_ASTERIX_WARNINGS_LIST, stringBuilder.toString());
+ }
+
+ public static void issueWarnings(IWarningCollector warningCollector, Configuration conf) throws IOException {
+ String warnings = conf.get(ExternalDataConstants.KEY_HADOOP_ASTERIX_WARNINGS_LIST, "");
+ if (!warnings.isEmpty() && warningCollector.shouldWarn()) {
+ String[] encodedWarnings = warnings.split(",");
+ Base64.Decoder decoder = Base64.getDecoder();
+ for (int i = 0; i < encodedWarnings.length; i++) {
+ /*
+ * This should create a small number of objects as warnings are reported only once by AsterixDB's
+ * hadoop readers
+ */
+ byte[] warningBytes = decoder.decode(encodedWarnings[i]);
+ DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(warningBytes));
+ warningCollector.warn(Warning.create(dataInputStream));
+ }
+ //Remove reported warnings
+ conf.unset(ExternalDataConstants.KEY_HADOOP_ASTERIX_WARNINGS_LIST);
+ }
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 17f91b4..06f4f73 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -20,6 +20,7 @@ package org.apache.asterix.metadata.declared;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
+import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -27,6 +28,7 @@ import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.api.ITypedAdapterFactory;
+import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.entities.Dataset;
@@ -36,6 +38,7 @@ import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.projection.DataProjectionInfo;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -114,7 +117,7 @@ public class DatasetDataSource extends DataSource {
ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
int externalScanBufferSize = physicalOptimizationConfig.getExternalScanBufferSize();
- Map<String, String> properties = addProjectionInfo(projectionInfo, edd.getProperties());
+ Map<String, String> properties = addExternalProjectionInfo(projectionInfo, edd.getProperties());
properties.put(KEY_EXTERNAL_SCAN_BUFFER_SIZE, String.valueOf(externalScanBufferSize));
ITypedAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
edd.getAdapter(), properties, (ARecordType) itemType, null, context.getWarningCollector());
@@ -138,13 +141,18 @@ public class DatasetDataSource extends DataSource {
}
}
- private Map<String, String> addProjectionInfo(IProjectionInfo<?> projectionInfo, Map<String, String> properties) {
+ private Map<String, String> addExternalProjectionInfo(IProjectionInfo<?> projectionInfo,
+ Map<String, String> properties) {
Map<String, String> propertiesCopy = properties;
if (projectionInfo != null) {
//properties could be cached and reused, so we make a copy per query
propertiesCopy = new HashMap<>(properties);
- ExternalDataProjectionInfo fieldNamesInfo = (ExternalDataProjectionInfo) projectionInfo;
- fieldNamesInfo.addToProperties(propertiesCopy);
+ try {
+ DataProjectionInfo externalProjectionInfo = (DataProjectionInfo) projectionInfo;
+ ExternalDataUtils.setExternalDataProjectionInfo(externalProjectionInfo, propertiesCopy);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
}
return propertiesCopy;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ExternalDataProjectionInfo.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ExternalDataProjectionInfo.java
deleted file mode 100644
index 9e8247e..0000000
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ExternalDataProjectionInfo.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.declared;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionInfo;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-/**
- * TODO Use {@link org.apache.asterix.runtime.projection.DataProjectionInfo}
- * Will be removed in a follow up change
- */
-@Deprecated
-public class ExternalDataProjectionInfo implements IProjectionInfo<List<List<String>>> {
- private final List<List<String>> projectedFieldNames;
-
- public ExternalDataProjectionInfo() {
- projectedFieldNames = new ArrayList<>();
- }
-
- private ExternalDataProjectionInfo(List<List<String>> projectedFieldNames) {
- this.projectedFieldNames = new ArrayList<>();
- for (List<String> path : projectedFieldNames) {
- List<String> newPath = new ArrayList<>(path);
- this.projectedFieldNames.add(newPath);
- }
- }
-
- @Override
- public List<List<String>> getProjectionInfo() {
- return projectedFieldNames;
- }
-
- @Override
- public IProjectionInfo<List<List<String>>> createCopy() {
- return new ExternalDataProjectionInfo(projectedFieldNames);
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof ExternalDataProjectionInfo)) {
- return false;
- }
- List<List<String>> otherProjectedFieldNames = ((ExternalDataProjectionInfo) other).projectedFieldNames;
- return projectedFieldNames.size() == otherProjectedFieldNames.size()
- && VariableUtilities.varListEqualUnordered(projectedFieldNames, otherProjectedFieldNames);
- }
-
- @Override
- public String toString() {
- if (projectedFieldNames.isEmpty()) {
- return "";
- }
- final StringBuilder fieldNamesBuilder = new StringBuilder();
- append(fieldNamesBuilder);
- return fieldNamesBuilder.toString();
- }
-
- /**
- * Append projected field names to the external dataset properties
- */
- public void addToProperties(Map<String, String> properties) {
- final String pushedFieldNames = toString();
- if (!pushedFieldNames.isEmpty()) {
- properties.put(ExternalDataConstants.KEY_REQUESTED_FIELDS, toString());
- }
- }
-
- private void append(StringBuilder builder) {
- appendFieldNames(projectedFieldNames.get(0), builder);
- for (int i = 1; i < projectedFieldNames.size(); i++) {
- builder.append(", ");
- appendFieldNames(projectedFieldNames.get(i), builder);
- }
- }
-
- private void appendFieldNames(List<String> fieldNames, StringBuilder builder) {
- builder.append(fieldNames.get(0));
- for (int i = 1; i < fieldNames.size(); i++) {
- builder.append('.').append(fieldNames.get(i));
- }
- }
-
-}
diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/projectedfieldnames/TestFieldNamesEquals.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/projectedfieldnames/TestFieldNamesEquals.java
deleted file mode 100644
index d19a8bb..0000000
--- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/projectedfieldnames/TestFieldNamesEquals.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.metadata.projectedfieldnames;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.asterix.metadata.declared.ExternalDataProjectionInfo;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestFieldNamesEquals {
-
- @Test
- public void testEqual() {
- ExternalDataProjectionInfo p1 = new ExternalDataProjectionInfo();
- ExternalDataProjectionInfo p2 = new ExternalDataProjectionInfo();
- setFieldNames(p1, new String[] { "a.b.c", "d.e.f", "g.h.i" });
- setFieldNames(p2, new String[] { "a.b.c", "d.e.f", "g.h.i" });
- Assert.assertEquals(p1, p2);
- }
-
- @Test
- public void testReversed() {
- ExternalDataProjectionInfo p1 = new ExternalDataProjectionInfo();
- ExternalDataProjectionInfo p2 = new ExternalDataProjectionInfo();
- setFieldNames(p1, new String[] { "a.b.c", "d.e.f", "g.h.i" });
- setFieldNames(p2, new String[] { "g.h.i", "d.e.f", "a.b.c" });
- Assert.assertEquals(p1, p2);
- }
-
- @Test
- public void testDifferentPermutations() {
- ExternalDataProjectionInfo p1 = new ExternalDataProjectionInfo();
- ExternalDataProjectionInfo p2 = new ExternalDataProjectionInfo();
- setFieldNames(p1, new String[] { "a.b.c", "d.e.f", "g.h.i" });
- setFieldNames(p2, new String[] { "d.e.f", "g.h.i", "a.b.c" });
- Assert.assertEquals(p1, p2);
-
- p1 = new ExternalDataProjectionInfo();
- p2 = new ExternalDataProjectionInfo();
- setFieldNames(p1, new String[] { "a.b.c", "d.e.f", "g.h.i" });
- setFieldNames(p2, new String[] { "g.h.i", "d.e.f", "a.b.c" });
- Assert.assertEquals(p1, p2);
-
- p1 = new ExternalDataProjectionInfo();
- p2 = new ExternalDataProjectionInfo();
- setFieldNames(p1, new String[] { "a.b.c", "d.e.f", "g.h.i" });
- setFieldNames(p2, new String[] { "g.h.i", "a.b.c", "d.e.f" });
- Assert.assertEquals(p1, p2);
- }
-
- @Test
- public void testDifferentLengths() {
- ExternalDataProjectionInfo p1 = new ExternalDataProjectionInfo();
- ExternalDataProjectionInfo p2 = new ExternalDataProjectionInfo();
- setFieldNames(p1, new String[] { "a.b.c", "d.e.f" });
- setFieldNames(p2, new String[] { "d.e.f", "a.b.c", "g" });
- Assert.assertNotEquals(p1, p2);
- }
-
- @Test
- public void testEqualSubPath() {
- ExternalDataProjectionInfo p1 = new ExternalDataProjectionInfo();
- ExternalDataProjectionInfo p2 = new ExternalDataProjectionInfo();
- setFieldNames(p1, new String[] { "a.b.c", "d.e.f.g" });
- setFieldNames(p2, new String[] { "d.e.f", "a.b.c" });
- Assert.assertNotEquals(p1, p2);
- }
-
- private static void setFieldNames(ExternalDataProjectionInfo p, String[] fieldNames) {
- List<List<String>> fieldNamesList = p.getProjectionInfo();
- for (String fnString : fieldNames) {
- List<String> fnList = new ArrayList<>();
- String[] fn = fnString.split("[.]");
- Collections.addAll(fnList, fn);
- fieldNamesList.add(fnList);
- }
- }
-
-}