You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ht...@apache.org on 2020/06/16 11:11:20 UTC
[asterixdb] branch master updated (64ae834 -> 51c5b05)
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 64ae834 [ASTERIXDB-2730][STO] Optimize flush in GVBC
new 0163af6 [ASTERIXDB-2741][EXT] Support include/exclude filters for S3 external datasets
new 7e74c73 Merge commit '0163af68d0' from 'gerrit/mad-hatter'
new 51c5b05 Merge "Merge commit '0163af68d0' from 'gerrit/mad-hatter'"
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../asterix/app/translator/QueryTranslator.java | 14 +-
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 6 +-
.../aws/AwsS3ExternalDatasetTest.java | 154 ++++++++++++++----
.../external_dataset.000.ddl.sqlpp | 2 +-
.../bad-name-1/test.000.ddl.sqlpp} | 8 +-
.../bad-name-1}/test.099.ddl.sqlpp | 0
.../bad-name-2/test.000.ddl.sqlpp} | 8 +-
.../bad-name-2}/test.099.ddl.sqlpp | 0
.../bad-name-3/test.000.ddl.sqlpp} | 8 +-
.../bad-name-3}/test.099.ddl.sqlpp | 0
.../both/test.000.ddl.sqlpp} | 8 +-
.../both}/test.099.ddl.sqlpp | 0
.../exclude-1/test.000.ddl.sqlpp} | 7 +-
.../exclude-1/test.001.query.sqlpp} | 18 +--
.../exclude-1}/test.099.ddl.sqlpp | 0
.../exclude-2/test.000.ddl.sqlpp} | 9 +-
.../exclude-2/test.001.query.sqlpp} | 18 +--
.../exclude-2}/test.099.ddl.sqlpp | 0
.../exclude-3/test.000.ddl.sqlpp} | 8 +-
.../exclude-3/test.001.query.sqlpp} | 18 +--
.../exclude-3}/test.099.ddl.sqlpp | 0
.../exclude-4/test.000.ddl.sqlpp} | 8 +-
.../exclude-4/test.001.query.sqlpp} | 18 +--
.../exclude-4}/test.099.ddl.sqlpp | 0
.../exclude-5/test.000.ddl.sqlpp} | 8 +-
.../exclude-5/test.001.query.sqlpp} | 18 +--
.../exclude-5}/test.099.ddl.sqlpp | 0
.../exclude-6/test.000.ddl.sqlpp} | 7 +-
.../exclude-6/test.001.query.sqlpp} | 18 +--
.../exclude-6}/test.099.ddl.sqlpp | 0
.../exclude-all/test.000.ddl.sqlpp} | 7 +-
.../exclude-all/test.001.query.sqlpp} | 18 +--
.../exclude-all}/test.099.ddl.sqlpp | 0
.../include-1/test.000.ddl.sqlpp} | 7 +-
.../include-1/test.001.query.sqlpp} | 18 +--
.../include-1}/test.099.ddl.sqlpp | 0
.../include-10/test.000.ddl.sqlpp} | 8 +-
.../include-10/test.001.query.sqlpp} | 18 +--
.../include-10}/test.099.ddl.sqlpp | 0
.../include-2/test.000.ddl.sqlpp} | 7 +-
.../include-2/test.001.query.sqlpp} | 18 +--
.../include-2/test.002.query.sqlpp} | 18 +--
.../include-2}/test.099.ddl.sqlpp | 0
.../include-3/test.000.ddl.sqlpp} | 10 +-
.../include-3/test.001.query.sqlpp} | 18 +--
.../include-3}/test.099.ddl.sqlpp | 0
.../include-4/test.000.ddl.sqlpp} | 10 +-
.../include-4/test.001.query.sqlpp} | 18 +--
.../include-4}/test.099.ddl.sqlpp | 0
.../include-5/test.000.ddl.sqlpp} | 7 +-
.../include-5/test.001.query.sqlpp} | 18 +--
.../include-5}/test.099.ddl.sqlpp | 0
.../include-6/test.000.ddl.sqlpp} | 7 +-
.../include-6/test.001.query.sqlpp} | 18 +--
.../include-6}/test.099.ddl.sqlpp | 0
.../include-7/test.000.ddl.sqlpp} | 8 +-
.../include-7/test.001.query.sqlpp} | 18 +--
.../include-7}/test.099.ddl.sqlpp | 0
.../include-8/test.000.ddl.sqlpp} | 8 +-
.../include-8/test.001.query.sqlpp} | 18 +--
.../include-8}/test.099.ddl.sqlpp | 0
.../include-9/test.000.ddl.sqlpp} | 8 +-
.../include-9/test.001.query.sqlpp} | 18 +--
.../include-9}/test.099.ddl.sqlpp | 0
.../include-all/test.000.ddl.sqlpp} | 7 +-
.../include-all/test.001.query.sqlpp} | 18 +--
.../include-all}/test.099.ddl.sqlpp | 0
.../s3/include-exclude/exclude-1/result.001.adm | 1 +
.../s3/include-exclude/exclude-2/result.001.adm | 1 +
.../s3/include-exclude/exclude-3/result.001.adm | 1 +
.../s3/include-exclude/exclude-4/result.001.adm | 1 +
.../s3/include-exclude/exclude-5/result.001.adm | 1 +
.../s3/include-exclude/exclude-6/result.001.adm | 1 +
.../exclude-all}/result.001.adm | 0
.../s3/include-exclude/include-1/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.002.adm | 4 +
.../s3/include-exclude/include-3/result.001.adm | 1 +
.../s3/include-exclude/include-4/result.001.adm | 1 +
.../s3/include-exclude/include-5/result.001.adm | 1 +
.../s3/include-exclude/include-6/result.001.adm | 1 +
.../s3/include-exclude/include-7/result.001.adm | 1 +
.../s3/include-exclude/include-8/result.001.adm | 1 +
.../s3/include-exclude/include-9/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 118 ++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 22 +--
.../asterix/common/exceptions/ErrorCode.java | 18 ++-
.../src/main/resources/asx_errormsg/en.properties | 18 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 113 ++++++-------
.../external/util/ExternalDataConstants.java | 2 +
.../asterix/external/util/ExternalDataUtils.java | 174 ++++++++++++++++++++-
91 files changed, 658 insertions(+), 517 deletions(-)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/bad-name-1/test.000.ddl.sqlpp} (89%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/bad-name-1}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/bad-name-2/test.000.ddl.sqlpp} (89%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/bad-name-2}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/bad-name-3/test.000.ddl.sqlpp} (89%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/bad-name-3}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/both/test.000.ddl.sqlpp} (90%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/both}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-1/test.000.ddl.sqlpp} (91%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-1/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/exclude-1}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-2/test.000.ddl.sqlpp} (84%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-2/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/exclude-2}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-3/test.000.ddl.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-3/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/exclude-3}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-4/test.000.ddl.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-4/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/exclude-4}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-5/test.000.ddl.sqlpp} (86%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-5/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/exclude-5}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-6/test.000.ddl.sqlpp} (90%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-6/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/exclude-6}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-all/test.000.ddl.sqlpp} (91%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/exclude-all/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/exclude-all}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-1/test.000.ddl.sqlpp} (91%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-1/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-1}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-10/test.000.ddl.sqlpp} (90%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-10/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-10}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-2/test.000.ddl.sqlpp} (91%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-2/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-2/test.002.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-2}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-3/test.000.ddl.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-3/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-3}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-4/test.000.ddl.sqlpp} (88%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-4/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-4}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-5/test.000.ddl.sqlpp} (91%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-5/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-5}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-6/test.000.ddl.sqlpp} (90%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-6/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-6}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-7/test.000.ddl.sqlpp} (86%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-7/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-7}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-8/test.000.ddl.sqlpp} (89%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-8/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-8}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-9/test.000.ddl.sqlpp} (89%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-9/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-9}/test.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-all/test.000.ddl.sqlpp} (91%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{empty-string-definition/external_dataset.000.ddl.sqlpp => include-exclude/include-all/test.001.query.sqlpp} (67%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/{invalid-endpoint => include-exclude/include-all}/test.099.ddl.sqlpp (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm
copy asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/{definition-does-not-exist => include-exclude/exclude-all}/result.001.adm (100%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm
[asterixdb] 02/03: Merge commit '0163af68d0' from
'gerrit/mad-hatter'
Posted by ht...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7e74c73fd93aa5c5aaae3db2494acd1486ac9650
Merge: 047c9e0 0163af6
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Fri Jun 12 14:31:21 2020 +0300
Merge commit '0163af68d0' from 'gerrit/mad-hatter'
Change-Id: I5b53384961e42fe070d2e1b885e762428f6f8c45
.../asterix/app/translator/QueryTranslator.java | 14 +-
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 6 +-
.../aws/AwsS3ExternalDatasetTest.java | 154 ++++++++++++++----
.../external_dataset.000.ddl.sqlpp | 2 +-
.../bad-name-1/test.000.ddl.sqlpp} | 8 +-
.../bad-name-1/test.099.ddl.sqlpp} | 19 +--
.../bad-name-2/test.000.ddl.sqlpp} | 8 +-
.../bad-name-2/test.099.ddl.sqlpp} | 19 +--
.../bad-name-3/test.000.ddl.sqlpp} | 8 +-
.../bad-name-3/test.099.ddl.sqlpp} | 19 +--
.../both/test.000.ddl.sqlpp} | 8 +-
.../both/test.099.ddl.sqlpp} | 19 +--
.../exclude-1/test.000.ddl.sqlpp} | 7 +-
.../exclude-1/test.001.query.sqlpp} | 18 +--
.../exclude-1/test.099.ddl.sqlpp} | 19 +--
.../exclude-2/test.000.ddl.sqlpp} | 9 +-
.../exclude-2/test.001.query.sqlpp} | 18 +--
.../exclude-2/test.099.ddl.sqlpp} | 19 +--
.../exclude-3/test.000.ddl.sqlpp} | 8 +-
.../exclude-3/test.001.query.sqlpp} | 18 +--
.../exclude-3/test.099.ddl.sqlpp} | 19 +--
.../exclude-4/test.000.ddl.sqlpp} | 8 +-
.../exclude-4/test.001.query.sqlpp} | 18 +--
.../exclude-4/test.099.ddl.sqlpp} | 19 +--
.../exclude-5/test.000.ddl.sqlpp} | 8 +-
.../exclude-5/test.001.query.sqlpp} | 18 +--
.../exclude-5/test.099.ddl.sqlpp} | 19 +--
.../exclude-6/test.000.ddl.sqlpp} | 7 +-
.../exclude-6/test.001.query.sqlpp} | 18 +--
.../exclude-6/test.099.ddl.sqlpp} | 19 +--
.../exclude-all/test.000.ddl.sqlpp} | 7 +-
.../exclude-all/test.001.query.sqlpp} | 18 +--
.../exclude-all/test.099.ddl.sqlpp} | 19 +--
.../include-1/test.000.ddl.sqlpp} | 7 +-
.../include-1/test.001.query.sqlpp} | 18 +--
.../include-1/test.099.ddl.sqlpp} | 19 +--
.../include-10/test.000.ddl.sqlpp} | 8 +-
.../include-10/test.001.query.sqlpp} | 18 +--
.../include-10/test.099.ddl.sqlpp} | 19 +--
.../include-2/test.000.ddl.sqlpp} | 7 +-
.../include-2/test.001.query.sqlpp} | 18 +--
.../include-2/test.002.query.sqlpp} | 18 +--
.../include-2/test.099.ddl.sqlpp} | 19 +--
.../include-3/test.000.ddl.sqlpp} | 10 +-
.../include-3/test.001.query.sqlpp} | 18 +--
.../include-3/test.099.ddl.sqlpp} | 19 +--
.../include-4/test.000.ddl.sqlpp} | 10 +-
.../include-4/test.001.query.sqlpp} | 18 +--
.../include-4/test.099.ddl.sqlpp} | 19 +--
.../include-5/test.000.ddl.sqlpp} | 7 +-
.../include-5/test.001.query.sqlpp} | 18 +--
.../include-5/test.099.ddl.sqlpp} | 19 +--
.../include-6/test.000.ddl.sqlpp} | 7 +-
.../include-6/test.001.query.sqlpp} | 18 +--
.../include-6/test.099.ddl.sqlpp} | 19 +--
.../include-7/test.000.ddl.sqlpp} | 8 +-
.../include-7/test.001.query.sqlpp} | 18 +--
.../include-7/test.099.ddl.sqlpp} | 19 +--
.../include-8/test.000.ddl.sqlpp} | 8 +-
.../include-8/test.001.query.sqlpp} | 18 +--
.../include-8/test.099.ddl.sqlpp} | 19 +--
.../include-9/test.000.ddl.sqlpp} | 8 +-
.../include-9/test.001.query.sqlpp} | 18 +--
.../include-9/test.099.ddl.sqlpp} | 19 +--
.../include-all/test.000.ddl.sqlpp} | 7 +-
.../include-all/test.001.query.sqlpp} | 18 +--
.../include-all/test.099.ddl.sqlpp} | 19 +--
.../s3/include-exclude/exclude-1/result.001.adm | 1 +
.../s3/include-exclude/exclude-2/result.001.adm | 1 +
.../s3/include-exclude/exclude-3/result.001.adm | 1 +
.../s3/include-exclude/exclude-4/result.001.adm | 1 +
.../s3/include-exclude/exclude-5/result.001.adm | 1 +
.../s3/include-exclude/exclude-6/result.001.adm | 1 +
.../s3/include-exclude/exclude-all/result.001.adm | 1 +
.../s3/include-exclude/include-1/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.002.adm | 4 +
.../s3/include-exclude/include-3/result.001.adm | 1 +
.../s3/include-exclude/include-4/result.001.adm | 1 +
.../s3/include-exclude/include-5/result.001.adm | 1 +
.../s3/include-exclude/include-6/result.001.adm | 1 +
.../s3/include-exclude/include-7/result.001.adm | 1 +
.../s3/include-exclude/include-8/result.001.adm | 1 +
.../s3/include-exclude/include-9/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 118 ++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 22 +--
.../asterix/common/exceptions/ErrorCode.java | 18 ++-
.../src/main/resources/asx_errormsg/en.properties | 18 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 113 ++++++-------
.../external/util/ExternalDataConstants.java | 2 +
.../asterix/external/util/ExternalDataUtils.java | 174 ++++++++++++++++++++-
91 files changed, 681 insertions(+), 913 deletions(-)
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index deb8503,f127fbd..a052f99
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@@ -2862,23 -2837,6 +2862,23 @@@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="aggregate-subclause">
+ <test-case FilePath="aggregate-subclause">
+ <compilation-unit name="agg_filter_01">
+ <output-dir compare="Text">agg_filter_01</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate-subclause">
+ <compilation-unit name="agg_filter_02_neg">
+ <output-dir compare="Text">agg_filter_01</output-dir>
- <expected-error>ASX1116: Illegal use of aggregate FILTER clause</expected-error>
- <expected-error>ASX1116: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
+ <expected-error>ASX1073: Cannot resolve alias reference for undefined identifier r</expected-error>
- <expected-error>ASX1116: Illegal use of aggregate FILTER clause</expected-error>
- <expected-error>ASX1116: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
++ <expected-error>ASX1120: Illegal use of aggregate FILTER clause</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="array_fun">
<test-case FilePath="array_fun">
<compilation-unit name="array_prepend">
@@@ -5611,29 -5405,6 +5611,29 @@@
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-1">
+ <output-dir compare="Text">grouping-sets-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-2">
+ <output-dir compare="Text">grouping-sets-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="grouping-sets-3-negative">
+ <output-dir compare="Text">grouping-sets-2</output-dir>
- <expected-error>ASX1115: Unexpected alias: v21</expected-error>
- <expected-error>ASX1115: Unexpected alias: v22</expected-error>
- <expected-error>ASX1115: Unexpected alias: v23</expected-error>
++ <expected-error>ASX1119: Unexpected alias: v21</expected-error>
++ <expected-error>ASX1119: Unexpected alias: v22</expected-error>
++ <expected-error>ASX1119: Unexpected alias: v23</expected-error>
+ <expected-error>ASX1087: Invalid number of arguments for function grouping</expected-error>
- <expected-error>ASX1114: Invalid argument to grouping() function</expected-error>
- <expected-error>ASX1114: Invalid argument to grouping() function</expected-error>
- <expected-error>ASX1114: Invalid argument to grouping() function</expected-error>
- <expected-error>ASX1113: Too many grouping sets in group by clause: 512. Maximum allowed: 128.</expected-error>
++ <expected-error>ASX1118: Invalid argument to grouping() function</expected-error>
++ <expected-error>ASX1118: Invalid argument to grouping() function</expected-error>
++ <expected-error>ASX1118: Invalid argument to grouping() function</expected-error>
++ <expected-error>ASX1117: Too many grouping sets in group by clause: 512. Maximum allowed: 128.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
<compilation-unit name="having">
<output-dir compare="Text">core-02</output-dir>
</compilation-unit>
diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 2bb62b1,46a6228..503ff6e
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@@ -199,13 -198,10 +199,17 @@@ public class ErrorCode
public static final int UNEXPECTED_HINT = 1107;
public static final int EXTERNAL_SOURCE_ERROR = 1108;
public static final int EXTERNAL_SOURCE_CONTAINER_NOT_FOUND = 1109;
- public static final int SYNONYM_EXISTS = 1110;
- public static final int UNKNOWN_SYNONYM = 1111;
- public static final int UNKNOWN_LIBRARY = 1112;
- public static final int COMPILATION_GROUPING_SETS_OVERFLOW = 1113;
- public static final int COMPILATION_GROUPING_OPERATION_INVALID_ARG = 1114;
- public static final int COMPILATION_UNEXPECTED_ALIAS = 1115;
- public static final int COMPILATION_ILLEGAL_USE_OF_FILTER_CLAUSE = 1116;
+ public static final int PARAMETERS_NOT_ALLOWED_AT_SAME_TIME = 1110;
+ public static final int PROPERTY_INVALID_VALUE_TYPE = 1111;
+ public static final int INVALID_PROPERTY_FORMAT = 1112;
+ public static final int INVALID_REGEX_PATTERN = 1113;
++ public static final int SYNONYM_EXISTS = 1114;
++ public static final int UNKNOWN_SYNONYM = 1115;
++ public static final int UNKNOWN_LIBRARY = 1116;
++ public static final int COMPILATION_GROUPING_SETS_OVERFLOW = 1117;
++ public static final int COMPILATION_GROUPING_OPERATION_INVALID_ARG = 1118;
++ public static final int COMPILATION_UNEXPECTED_ALIAS = 1119;
++ public static final int COMPILATION_ILLEGAL_USE_OF_FILTER_CLAUSE = 1120;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --cc asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index fa0f469,b070552..e612503
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@@ -194,13 -193,10 +194,17 @@@
1107 = Unexpected hint: %1$s. %2$s expected at this location
1108 = External source error. %1$s
1109 = External source container %1$s not found
- 1110 = A synonym with this name %1$s already exists
- 1111 = Cannot find synonym with name %1$s
- 1112 = Unknown library %1$s
- 1113 = Too many grouping sets in group by clause: %1$s. Maximum allowed: %2$s.
- 1114 = Invalid argument to grouping() function
- 1115 = Unexpected alias: %1$s
- 1116 = Illegal use of aggregate FILTER clause
+ 1110 = The parameters \"%1$s\" and \"%2$s\" cannot be provided at the same time
+ 1111 = Property \"%1$s\" expects value(s) of type %2$s
+ 1112 = Invalid format for property \"%1$s\"
+ 1113 = Invalid pattern %1$s
++1114 = A synonym with this name %1$s already exists
++1115 = Cannot find synonym with name %1$s
++1116 = Unknown library %1$s
++1117 = Too many grouping sets in group by clause: %1$s. Maximum allowed: %2$s.
++1118 = Invalid argument to grouping() function
++1119 = Unexpected alias: %1$s
++1120 = Illegal use of aggregate FILTER clause
# Feed Errors
3001 = Illegal state.
[asterixdb] 03/03: Merge "Merge commit '0163af68d0' from
'gerrit/mad-hatter'"
Posted by ht...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 51c5b056662c8f260b2d477c699a67146e01ee7b
Merge: 64ae834 7e74c73
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Tue Jun 16 10:33:05 2020 +0000
Merge "Merge commit '0163af68d0' from 'gerrit/mad-hatter'"
.../asterix/app/translator/QueryTranslator.java | 14 +-
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 6 +-
.../aws/AwsS3ExternalDatasetTest.java | 154 ++++++++++++++----
.../external_dataset.000.ddl.sqlpp | 2 +-
.../bad-name-1/test.000.ddl.sqlpp} | 8 +-
.../bad-name-1/test.099.ddl.sqlpp} | 19 +--
.../bad-name-2/test.000.ddl.sqlpp} | 8 +-
.../bad-name-2/test.099.ddl.sqlpp} | 19 +--
.../bad-name-3/test.000.ddl.sqlpp} | 8 +-
.../bad-name-3/test.099.ddl.sqlpp} | 19 +--
.../both/test.000.ddl.sqlpp} | 8 +-
.../both/test.099.ddl.sqlpp} | 19 +--
.../exclude-1/test.000.ddl.sqlpp} | 7 +-
.../exclude-1/test.001.query.sqlpp} | 18 +--
.../exclude-1/test.099.ddl.sqlpp} | 19 +--
.../exclude-2/test.000.ddl.sqlpp} | 9 +-
.../exclude-2/test.001.query.sqlpp} | 18 +--
.../exclude-2/test.099.ddl.sqlpp} | 19 +--
.../exclude-3/test.000.ddl.sqlpp} | 8 +-
.../exclude-3/test.001.query.sqlpp} | 18 +--
.../exclude-3/test.099.ddl.sqlpp} | 19 +--
.../exclude-4/test.000.ddl.sqlpp} | 8 +-
.../exclude-4/test.001.query.sqlpp} | 18 +--
.../exclude-4/test.099.ddl.sqlpp} | 19 +--
.../exclude-5/test.000.ddl.sqlpp} | 8 +-
.../exclude-5/test.001.query.sqlpp} | 18 +--
.../exclude-5/test.099.ddl.sqlpp} | 19 +--
.../exclude-6/test.000.ddl.sqlpp} | 7 +-
.../exclude-6/test.001.query.sqlpp} | 18 +--
.../exclude-6/test.099.ddl.sqlpp} | 19 +--
.../exclude-all/test.000.ddl.sqlpp} | 7 +-
.../exclude-all/test.001.query.sqlpp} | 18 +--
.../exclude-all/test.099.ddl.sqlpp} | 19 +--
.../include-1/test.000.ddl.sqlpp} | 7 +-
.../include-1/test.001.query.sqlpp} | 18 +--
.../include-1/test.099.ddl.sqlpp} | 19 +--
.../include-10/test.000.ddl.sqlpp} | 8 +-
.../include-10/test.001.query.sqlpp} | 18 +--
.../include-10/test.099.ddl.sqlpp} | 19 +--
.../include-2/test.000.ddl.sqlpp} | 7 +-
.../include-2/test.001.query.sqlpp} | 18 +--
.../include-2/test.002.query.sqlpp} | 18 +--
.../include-2/test.099.ddl.sqlpp} | 19 +--
.../include-3/test.000.ddl.sqlpp} | 10 +-
.../include-3/test.001.query.sqlpp} | 18 +--
.../include-3/test.099.ddl.sqlpp} | 19 +--
.../include-4/test.000.ddl.sqlpp} | 10 +-
.../include-4/test.001.query.sqlpp} | 18 +--
.../include-4/test.099.ddl.sqlpp} | 19 +--
.../include-5/test.000.ddl.sqlpp} | 7 +-
.../include-5/test.001.query.sqlpp} | 18 +--
.../include-5/test.099.ddl.sqlpp} | 19 +--
.../include-6/test.000.ddl.sqlpp} | 7 +-
.../include-6/test.001.query.sqlpp} | 18 +--
.../include-6/test.099.ddl.sqlpp} | 19 +--
.../include-7/test.000.ddl.sqlpp} | 8 +-
.../include-7/test.001.query.sqlpp} | 18 +--
.../include-7/test.099.ddl.sqlpp} | 19 +--
.../include-8/test.000.ddl.sqlpp} | 8 +-
.../include-8/test.001.query.sqlpp} | 18 +--
.../include-8/test.099.ddl.sqlpp} | 19 +--
.../include-9/test.000.ddl.sqlpp} | 8 +-
.../include-9/test.001.query.sqlpp} | 18 +--
.../include-9/test.099.ddl.sqlpp} | 19 +--
.../include-all/test.000.ddl.sqlpp} | 7 +-
.../include-all/test.001.query.sqlpp} | 18 +--
.../include-all/test.099.ddl.sqlpp} | 19 +--
.../s3/include-exclude/exclude-1/result.001.adm | 1 +
.../s3/include-exclude/exclude-2/result.001.adm | 1 +
.../s3/include-exclude/exclude-3/result.001.adm | 1 +
.../s3/include-exclude/exclude-4/result.001.adm | 1 +
.../s3/include-exclude/exclude-5/result.001.adm | 1 +
.../s3/include-exclude/exclude-6/result.001.adm | 1 +
.../s3/include-exclude/exclude-all/result.001.adm | 1 +
.../s3/include-exclude/include-1/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.002.adm | 4 +
.../s3/include-exclude/include-3/result.001.adm | 1 +
.../s3/include-exclude/include-4/result.001.adm | 1 +
.../s3/include-exclude/include-5/result.001.adm | 1 +
.../s3/include-exclude/include-6/result.001.adm | 1 +
.../s3/include-exclude/include-7/result.001.adm | 1 +
.../s3/include-exclude/include-8/result.001.adm | 1 +
.../s3/include-exclude/include-9/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 118 ++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 22 +--
.../asterix/common/exceptions/ErrorCode.java | 18 ++-
.../src/main/resources/asx_errormsg/en.properties | 18 ++-
.../record/reader/aws/AwsS3InputStreamFactory.java | 113 ++++++-------
.../external/util/ExternalDataConstants.java | 2 +
.../asterix/external/util/ExternalDataUtils.java | 174 ++++++++++++++++++++-
91 files changed, 681 insertions(+), 913 deletions(-)
[asterixdb] 01/03: [ASTERIXDB-2741][EXT] Support include/exclude
filters for S3 external datasets
Posted by ht...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0163af68d0cd1d79d1dc95655f274fced2da48ca
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Mon Jun 8 16:38:53 2020 +0300
[ASTERIXDB-2741][EXT] Support include/exclude filters for S3 external datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added support for include/exclude filters.
- Added test cases for include/exclude.
Change-Id: I21dbc5c08456f95a2cad1a8fd9147472fad15376
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6604
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../asterix/app/translator/QueryTranslator.java | 14 +-
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 6 +-
.../aws/AwsS3ExternalDatasetTest.java | 154 ++++++++++++++----
.../external_dataset.000.ddl.sqlpp | 2 +-
.../bad-name-1/test.000.ddl.sqlpp} | 8 +-
.../bad-name-1/test.099.ddl.sqlpp} | 19 +--
.../bad-name-2/test.000.ddl.sqlpp} | 8 +-
.../bad-name-2/test.099.ddl.sqlpp} | 19 +--
.../bad-name-3/test.000.ddl.sqlpp} | 8 +-
.../bad-name-3/test.099.ddl.sqlpp} | 19 +--
.../both/test.000.ddl.sqlpp} | 8 +-
.../both/test.099.ddl.sqlpp} | 19 +--
.../exclude-1/test.000.ddl.sqlpp} | 7 +-
.../exclude-1/test.001.query.sqlpp} | 18 +--
.../exclude-1/test.099.ddl.sqlpp} | 19 +--
.../exclude-2/test.000.ddl.sqlpp} | 9 +-
.../exclude-2/test.001.query.sqlpp} | 18 +--
.../exclude-2/test.099.ddl.sqlpp} | 19 +--
.../exclude-3/test.000.ddl.sqlpp} | 8 +-
.../exclude-3/test.001.query.sqlpp} | 18 +--
.../exclude-3/test.099.ddl.sqlpp} | 19 +--
.../exclude-4/test.000.ddl.sqlpp} | 8 +-
.../exclude-4/test.001.query.sqlpp} | 18 +--
.../exclude-4/test.099.ddl.sqlpp} | 19 +--
.../exclude-5/test.000.ddl.sqlpp} | 8 +-
.../exclude-5/test.001.query.sqlpp} | 18 +--
.../exclude-5/test.099.ddl.sqlpp} | 19 +--
.../exclude-6/test.000.ddl.sqlpp} | 7 +-
.../exclude-6/test.001.query.sqlpp} | 18 +--
.../exclude-6/test.099.ddl.sqlpp} | 19 +--
.../exclude-all/test.000.ddl.sqlpp} | 7 +-
.../exclude-all/test.001.query.sqlpp} | 18 +--
.../exclude-all/test.099.ddl.sqlpp} | 19 +--
.../include-1/test.000.ddl.sqlpp} | 7 +-
.../include-1/test.001.query.sqlpp} | 18 +--
.../include-1/test.099.ddl.sqlpp} | 19 +--
.../include-10/test.000.ddl.sqlpp} | 8 +-
.../include-10/test.001.query.sqlpp} | 18 +--
.../include-10/test.099.ddl.sqlpp} | 19 +--
.../include-2/test.000.ddl.sqlpp} | 7 +-
.../include-2/test.001.query.sqlpp} | 18 +--
.../include-2/test.002.query.sqlpp} | 18 +--
.../include-2/test.099.ddl.sqlpp} | 19 +--
.../include-3/test.000.ddl.sqlpp} | 10 +-
.../include-3/test.001.query.sqlpp} | 18 +--
.../include-3/test.099.ddl.sqlpp} | 19 +--
.../include-4/test.000.ddl.sqlpp} | 10 +-
.../include-4/test.001.query.sqlpp} | 18 +--
.../include-4/test.099.ddl.sqlpp} | 19 +--
.../include-5/test.000.ddl.sqlpp} | 7 +-
.../include-5/test.001.query.sqlpp} | 18 +--
.../include-5/test.099.ddl.sqlpp} | 19 +--
.../include-6/test.000.ddl.sqlpp} | 7 +-
.../include-6/test.001.query.sqlpp} | 18 +--
.../include-6/test.099.ddl.sqlpp} | 19 +--
.../include-7/test.000.ddl.sqlpp} | 8 +-
.../include-7/test.001.query.sqlpp} | 18 +--
.../include-7/test.099.ddl.sqlpp} | 19 +--
.../include-8/test.000.ddl.sqlpp} | 8 +-
.../include-8/test.001.query.sqlpp} | 18 +--
.../include-8/test.099.ddl.sqlpp} | 19 +--
.../include-9/test.000.ddl.sqlpp} | 8 +-
.../include-9/test.001.query.sqlpp} | 18 +--
.../include-9/test.099.ddl.sqlpp} | 19 +--
.../include-all/test.000.ddl.sqlpp} | 7 +-
.../include-all/test.001.query.sqlpp} | 18 +--
.../include-all/test.099.ddl.sqlpp} | 19 +--
.../s3/include-exclude/exclude-1/result.001.adm | 1 +
.../s3/include-exclude/exclude-2/result.001.adm | 1 +
.../s3/include-exclude/exclude-3/result.001.adm | 1 +
.../s3/include-exclude/exclude-4/result.001.adm | 1 +
.../s3/include-exclude/exclude-5/result.001.adm | 1 +
.../s3/include-exclude/exclude-6/result.001.adm | 1 +
.../s3/include-exclude/exclude-all/result.001.adm | 1 +
.../s3/include-exclude/include-1/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.002.adm | 4 +
.../s3/include-exclude/include-3/result.001.adm | 1 +
.../s3/include-exclude/include-4/result.001.adm | 1 +
.../s3/include-exclude/include-5/result.001.adm | 1 +
.../s3/include-exclude/include-6/result.001.adm | 1 +
.../s3/include-exclude/include-7/result.001.adm | 1 +
.../s3/include-exclude/include-8/result.001.adm | 1 +
.../s3/include-exclude/include-9/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 118 ++++++++++++++
.../asterix/common/exceptions/ErrorCode.java | 4 +
.../src/main/resources/asx_errormsg/en.properties | 4 +
.../record/reader/aws/AwsS3InputStreamFactory.java | 113 ++++++-------
.../external/util/ExternalDataConstants.java | 2 +
.../asterix/external/util/ExternalDataUtils.java | 174 ++++++++++++++++++++-
90 files changed, 656 insertions(+), 888 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index b814b0a..1be3894 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -3240,16 +3240,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
protected void validateExternalDatasetProperties(ExternalDetailsDecl externalDetails,
Map<String, String> properties, SourceLocation srcLoc, MetadataTransactionContext mdTxnCtx)
throws AlgebricksException, HyracksDataException {
+ // Validate adapter specific properties
String adapter = externalDetails.getAdapter();
- // "format" parameter is needed for "S3" data source
- if (ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3.equals(adapter)
- && properties.get(ExternalDataConstants.KEY_FORMAT) == null) {
- throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
- }
-
Map<String, String> details = new HashMap<>(properties);
details.put(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE, adapter);
- validateExternalSourceContainer(details);
+ validateAdapterSpecificProperties(details, srcLoc);
}
/**
@@ -3257,7 +3252,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
*
* @param configuration external source properties
*/
- protected void validateExternalSourceContainer(Map<String, String> configuration) throws CompilationException {
- ExternalDataUtils.validateExternalSourceContainer(configuration);
+ protected void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc)
+ throws CompilationException {
+ ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc);
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
index bd433f1..48e966e 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
@@ -43,8 +43,9 @@ public class AwsS3ExternalDatasetOnePartitionTest extends AwsS3ExternalDatasetTe
SUITE_TESTS = "testsuite_external_dataset_one_partition.xml";
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
- PREPARE_S3_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
+ PREPARE_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareFixedDataBucket;
+ PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareMixedDataBucket;
return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@@ -53,4 +54,7 @@ public class AwsS3ExternalDatasetOnePartitionTest extends AwsS3ExternalDatasetTe
private static void prepareFixedDataBucket() {
}
+
+ private static void prepareMixedDataBucket() {
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 23bf3bd..8ec311d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -83,25 +83,28 @@ public class AwsS3ExternalDatasetTest {
static String SUITE_TESTS;
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
- static Runnable PREPARE_S3_BUCKET;
+ static Runnable PREPARE_BUCKET;
static Runnable PREPARE_FIXED_DATA_BUCKET;
+ static Runnable PREPARE_MIXED_DATA_BUCKET;
// Base directory paths for data files
private static final String JSON_DATA_PATH = joinPath("data", "json");
private static final String CSV_DATA_PATH = joinPath("data", "csv");
private static final String TSV_DATA_PATH = joinPath("data", "tsv");
+ private static final String MIXED_DATA_PATH = joinPath("data", "mixed");
// Service endpoint
- private static final int S3_MOCK_SERVER_PORT = 8001;
- private static final String S3_MOCK_SERVER_HOSTNAME = "http://localhost:" + S3_MOCK_SERVER_PORT;
+ private static final int MOCK_SERVER_PORT = 8001;
+ private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
// Region, bucket and definitions
- private static final String S3_MOCK_SERVER_REGION = "us-west-2";
- private static final String S3_MOCK_SERVER_BUCKET = "playground";
- private static final String S3_MOCK_SERVER_FIXED_DATA_BUCKET = "fixed-data-bucket"; // Do not use, has fixed data
- private static final String S3_MOCK_SERVER_BUCKET_JSON_DEFINITION = "json-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_BUCKET_CSV_DEFINITION = "csv-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_BUCKET_TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
+ private static final String MOCK_SERVER_REGION = "us-west-2";
+ private static final String MOCK_SERVER_BUCKET = "playground";
+ private static final String FIXED_DATA_BUCKET = "fixed-data"; // Do not use, has fixed data
+ private static final String INCLUDE_EXCLUDE_BUCKET = "include-exclude"; // include & exclude bucket
+ private static final String JSON_DEFINITION = "json-data/reviews/"; // data resides here
+ private static final String CSV_DEFINITION = "csv-data/reviews/"; // data resides here
+ private static final String TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
// This is used for a test to generate over 1000 number of files
private static final String OVER_1000_OBJECTS_PATH = "over-1000-objects";
@@ -115,7 +118,9 @@ public class AwsS3ExternalDatasetTest {
// IMPORTANT: The following values must be used in the AWS S3 test case
private static S3Mock s3MockServer;
private static S3Client client;
- private static PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET);
+ private static final PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(MOCK_SERVER_BUCKET);
+ private static final PutObjectRequest.Builder includeExcludeBuilder =
+ PutObjectRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET);
protected TestCaseContext tcCtx;
@@ -151,8 +156,9 @@ public class AwsS3ExternalDatasetTest {
SUITE_TESTS = "testsuite_external_dataset.xml";
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
- PREPARE_S3_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
+ PREPARE_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareFixedDataBucket;
+ PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareMixedDataBucket;
return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@@ -180,30 +186,31 @@ public class AwsS3ExternalDatasetTest {
private static void startAwsS3MockServer() {
// Starting S3 mock server to be used instead of real S3 server
LOGGER.info("Starting S3 mock server");
- s3MockServer = new S3Mock.Builder().withPort(S3_MOCK_SERVER_PORT).withInMemoryBackend().build();
+ s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
s3MockServer.start();
LOGGER.info("S3 mock server started successfully");
// Create a client and add some files to the S3 mock server
LOGGER.info("Creating S3 client to load initial files to S3 mock server");
S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(S3_MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
- builder.region(Region.of(S3_MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+ URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
.endpointOverride(endpoint);
client = builder.build();
LOGGER.info("Client created successfully");
// Create the bucket and upload some json files
- PREPARE_S3_BUCKET.run();
+ PREPARE_BUCKET.run();
PREPARE_FIXED_DATA_BUCKET.run();
+ PREPARE_MIXED_DATA_BUCKET.run();
}
/**
* Creates a bucket and fills it with some files for testing purpose.
*/
private static void prepareS3Bucket() {
- LOGGER.info("creating bucket " + S3_MOCK_SERVER_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(S3_MOCK_SERVER_BUCKET).build());
+ LOGGER.info("creating bucket " + MOCK_SERVER_BUCKET);
+ client.createBucket(CreateBucketRequest.builder().bucket(MOCK_SERVER_BUCKET).build());
LOGGER.info("bucket created successfully");
LOGGER.info("Adding JSON files to the bucket");
@@ -228,24 +235,24 @@ public class AwsS3ExternalDatasetTest {
* changed, the test case will fail and its result will need to be updated each time
*/
private static void prepareFixedDataBucket() {
- LOGGER.info("creating bucket " + S3_MOCK_SERVER_FIXED_DATA_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).build());
- LOGGER.info("bucket " + S3_MOCK_SERVER_FIXED_DATA_BUCKET + " created successfully");
+ LOGGER.info("creating bucket " + FIXED_DATA_BUCKET);
+ client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_BUCKET).build());
+ LOGGER.info("bucket " + FIXED_DATA_BUCKET + " created successfully");
- LOGGER.info("Loading fixed data to " + S3_MOCK_SERVER_FIXED_DATA_BUCKET);
+ LOGGER.info("Loading fixed data to " + FIXED_DATA_BUCKET);
// Files data
RequestBody requestBody = RequestBody.fromFile(Paths.get(JSON_DATA_PATH, "single-line", "20-records.json"));
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("1.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("2.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("lvl1/3.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("lvl1/4.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("lvl1/lvl2/5.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("1.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("2.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/3.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/4.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/lvl2/5.json").build(), requestBody);
}
private static void loadJsonFiles() {
String dataBasePath = JSON_DATA_PATH;
- String definition = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION;
+ String definition = JSON_DEFINITION;
// Normal format
String definitionSegment = "json";
@@ -279,7 +286,7 @@ public class AwsS3ExternalDatasetTest {
private static void loadCsvFiles() {
String dataBasePath = CSV_DATA_PATH;
- String definition = S3_MOCK_SERVER_BUCKET_CSV_DEFINITION;
+ String definition = CSV_DEFINITION;
// Normal format
String definitionSegment = "csv";
@@ -301,7 +308,7 @@ public class AwsS3ExternalDatasetTest {
private static void loadTsvFiles() {
String dataBasePath = TSV_DATA_PATH;
- String definition = S3_MOCK_SERVER_BUCKET_TSV_DEFINITION;
+ String definition = TSV_DEFINITION;
// Normal format
String definitionSegment = "tsv";
@@ -399,6 +406,95 @@ public class AwsS3ExternalDatasetTest {
}
}
+ /**
+ * Loads a combination of different file formats in the same path
+ */
+ private static void prepareMixedDataBucket() {
+ LOGGER.info("creating bucket " + INCLUDE_EXCLUDE_BUCKET);
+ client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET).build());
+ LOGGER.info("bucket " + INCLUDE_EXCLUDE_BUCKET + " created successfully");
+
+ // JSON
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+
+ // CSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2018.csv").build(),
+ RequestBody.fromString("7,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2019.csv").build(),
+ RequestBody.fromString("8,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2020.csv").build(),
+ RequestBody.fromString("{9,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2018.csv").build(),
+ RequestBody.fromString("10,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2019.csv").build(),
+ RequestBody.fromString("11,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2020.csv").build(),
+ RequestBody.fromString("12,\"good\""));
+
+ // TSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2018.tsv").build(),
+ RequestBody.fromString("13\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2019.tsv").build(),
+ RequestBody.fromString("14\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2020.tsv").build(),
+ RequestBody.fromString("15\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2018.tsv").build(),
+ RequestBody.fromString("16\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2019.tsv").build(),
+ RequestBody.fromString("17\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2020.tsv").build(),
+ RequestBody.fromString("18\t\"good\""));
+
+ // JSON no extension
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+ }
+
static class AwsTestExecutor extends TestExecutor {
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
index bb95094..a3bfd53 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
+("container"="fixed-data"),
("definition"=""),
("format"="json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.000.ddl.sqlpp
index bb95094..1b463bf 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.json"),
+("exclude1"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.000.ddl.sqlpp
index bb95094..fe3922d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.json"),
+("exclude#"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.000.ddl.sqlpp
index bb95094..efeffc4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.json"),
+("exclude#hello"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.000.ddl.sqlpp
index bb95094..542046d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*.json"),
+("exclude"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.000.ddl.sqlpp
index bb95094..6e84301 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.?sv")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.000.ddl.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.000.ddl.sqlpp
index bb95094..0ce88b7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.000.ddl.sqlpp
@@ -31,7 +31,10 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#1"="data/mixed/json/extension*"),
+("exclude#100"="data/mixed/json/EXTENSION*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.000.ddl.sqlpp
index bb95094..8c05fa6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#100"="data/mixed/json*201?*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.000.ddl.sqlpp
index bb95094..e1a50b8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#1"="data/mixed/json*bye*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.000.ddl.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.000.ddl.sqlpp
index bb95094..32e115a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#1"="data/mixed/json/extension/hello-world-2018.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.000.ddl.sqlpp
index bb95094..78a8310 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.[abct][abcs][abcv]")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.000.ddl.sqlpp
index bb95094..f09079a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.000.ddl.sqlpp
index bb95094..1b12acb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*2018*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.000.ddl.sqlpp
index bb95094..32c57b7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*[abc][.*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.000.ddl.sqlpp
index bb95094..e129135 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*201?*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.002.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.002.query.sqlpp
index bb95094..9de4a37 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.002.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select value test from test order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.000.ddl.sqlpp
index bb95094..4203999 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.000.ddl.sqlpp
@@ -23,6 +23,8 @@ use test;
drop type test if exists;
create type test as open {
+f1: int,
+f2: string
};
drop dataset test if exists;
@@ -31,7 +33,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*201?*.csv")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.000.ddl.sqlpp
index bb95094..92fed55 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.000.ddl.sqlpp
@@ -23,6 +23,8 @@ use test;
drop type test if exists;
create type test as open {
+f1: int,
+f2: string
};
drop dataset test if exists;
@@ -31,7 +33,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="tsv"),
+("header"=false),
+("include"="*201?*.tsv")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.000.ddl.sqlpp
index bb95094..2d38a8c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*bye*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.000.ddl.sqlpp
index bb95094..5b4344a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="data/mixed/json/*EXTENSION*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.000.ddl.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.000.ddl.sqlpp
index bb95094..e4e2050 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="data/mixed/json/NO-EXTENSION*"),
+("include#0"="data/mixed/json/EXTENSION*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.000.ddl.sqlpp
index bb95094..7d093b2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*.[!xyt][!xyz][!xyz]")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.000.ddl.sqlpp
index bb95094..9b0e7ff 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*.[a-c][a-z][a-z]")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.000.ddl.sqlpp
index bb95094..5545e38 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm
new file mode 100644
index 0000000..207f123
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm
@@ -0,0 +1 @@
+{ "count": 12 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm
new file mode 100644
index 0000000..8ac9199
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm
@@ -0,0 +1 @@
+{ "count": 11 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm
new file mode 100644
index 0000000..207f123
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm
@@ -0,0 +1 @@
+{ "count": 12 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-all/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-all/result.001.adm
new file mode 100644
index 0000000..c1a0ea2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-all/result.001.adm
@@ -0,0 +1 @@
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm
new file mode 100644
index 0000000..df462fe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm
@@ -0,0 +1 @@
+{ "count": 2 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm
new file mode 100644
index 0000000..9b5f94a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm
@@ -0,0 +1,4 @@
+{ "id": 1 }
+{ "id": 2 }
+{ "id": 4 }
+{ "id": 5 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm
new file mode 100644
index 0000000..8637a38
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm
@@ -0,0 +1 @@
+{ "count": 3 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 7c1aa48..7c4cf37 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -100,4 +100,122 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="include-exclude">
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/bad-name-1">
+ <output-dir compare="Text">aws/s3/include-exclude/bad-name-1</output-dir>
+ <expected-error>Invalid format for property "exclude1"</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/bad-name-2">
+ <output-dir compare="Text">aws/s3/include-exclude/bad-name-2</output-dir>
+ <expected-error>Invalid format for property "exclude#"</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/bad-name-3">
+ <output-dir compare="Text">aws/s3/include-exclude/bad-name-3</output-dir>
+ <expected-error>Invalid format for property "exclude#hello"</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/both">
+ <output-dir compare="Text">aws/s3/include-exclude/both</output-dir>
+ <expected-error>The parameters "include" and "exclude" cannot be provided at the same time</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-all">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-all</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-1">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-2">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-3">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-4">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-5">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-6">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-6</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-all">
+ <output-dir compare="Text">aws/s3/include-exclude/include-all</output-dir>
+ <expected-error>Malformed input stream</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-1">
+ <output-dir compare="Text">aws/s3/include-exclude/include-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-2">
+ <output-dir compare="Text">aws/s3/include-exclude/include-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-3">
+ <output-dir compare="Text">aws/s3/include-exclude/include-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-4">
+ <output-dir compare="Text">aws/s3/include-exclude/include-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-5">
+ <output-dir compare="Text">aws/s3/include-exclude/include-5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-6">
+ <output-dir compare="Text">aws/s3/include-exclude/include-6</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-7">
+ <output-dir compare="Text">aws/s3/include-exclude/include-7</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-8">
+ <output-dir compare="Text">aws/s3/include-exclude/include-8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-9">
+ <output-dir compare="Text">aws/s3/include-exclude/include-9</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-10">
+ <output-dir compare="Text">aws/s3/include-exclude/include-10</output-dir>
+ <expected-error>Invalid pattern *[abc][.*</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
</test-suite>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 6496c94..46a6228 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -198,6 +198,10 @@ public class ErrorCode {
public static final int UNEXPECTED_HINT = 1107;
public static final int EXTERNAL_SOURCE_ERROR = 1108;
public static final int EXTERNAL_SOURCE_CONTAINER_NOT_FOUND = 1109;
+ public static final int PARAMETERS_NOT_ALLOWED_AT_SAME_TIME = 1110;
+ public static final int PROPERTY_INVALID_VALUE_TYPE = 1111;
+ public static final int INVALID_PROPERTY_FORMAT = 1112;
+ public static final int INVALID_REGEX_PATTERN = 1113;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 7e75a51..b070552 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -193,6 +193,10 @@
1107 = Unexpected hint: %1$s. %2$s expected at this location
1108 = External source error. %1$s
1109 = External source container %1$s not found
+1110 = The parameters \"%1$s\" and \"%2$s\" cannot be provided at the same time
+1111 = Property \"%1$s\" expects value(s) of type %2$s
+1112 = Invalid format for property \"%1$s\"
+1113 = Invalid pattern %1$s
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index e57b31a..6ef5a23 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -19,19 +19,24 @@
package org.apache.asterix.external.input.record.reader.aws;
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXCLUDE;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_INCLUDE;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.function.BiPredicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
-import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IInputStreamFactory;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -49,12 +54,9 @@ import software.amazon.awssdk.services.s3.model.S3Object;
public class AwsS3InputStreamFactory implements IInputStreamFactory {
private static final long serialVersionUID = 1L;
- private Map<String, String> configuration;
- // Files to read from
- private final List<S3Object> filesOnly = new ArrayList<>();
+ private Map<String, String> configuration;
private final List<PartitionWorkLoadBasedOnSize> partitionWorkLoadsBasedOnSize = new ArrayList<>();
-
private transient AlgebricksAbsolutePartitionConstraint partitionConstraint;
@Override
@@ -84,6 +86,42 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
String container = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ List<S3Object> filesOnly = new ArrayList<>();
+
+ // Ensure the validity of include/exclude
+ ExternalDataUtils.AwsS3.validateIncludeExclude(configuration);
+
+ // Get and compile the patterns for include/exclude if provided
+ List<Matcher> includeMatchers = new ArrayList<>();
+ List<Matcher> excludeMatchers = new ArrayList<>();
+ String pattern = null;
+ try {
+ for (Map.Entry<String, String> entry : configuration.entrySet()) {
+ if (entry.getKey().startsWith(KEY_INCLUDE)) {
+ pattern = entry.getValue();
+ includeMatchers.add(Pattern.compile(ExternalDataUtils.wildcardToRegex(pattern)).matcher(""));
+ } else if (entry.getKey().startsWith(KEY_EXCLUDE)) {
+ pattern = entry.getValue();
+ excludeMatchers.add(Pattern.compile(ExternalDataUtils.wildcardToRegex(pattern)).matcher(""));
+ }
+ }
+ } catch (PatternSyntaxException ex) {
+ throw new CompilationException(ErrorCode.INVALID_REGEX_PATTERN, pattern);
+ }
+
+ List<Matcher> matchersList;
+ BiPredicate<List<Matcher>, String> p;
+ if (!includeMatchers.isEmpty()) {
+ matchersList = includeMatchers;
+ p = (matchers, key) -> ExternalDataUtils.matchPatterns(matchers, key);
+ } else if (!excludeMatchers.isEmpty()) {
+ matchersList = excludeMatchers;
+ p = (matchers, key) -> !ExternalDataUtils.matchPatterns(matchers, key);
+ } else {
+ matchersList = Collections.emptyList();
+ p = (matchers, key) -> true;
+ }
+
S3Client s3Client = ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
// Get all objects in a bucket and extract the paths to files
@@ -97,8 +135,6 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
boolean done = false;
String newMarker = null;
- String fileFormat = configuration.get(ExternalDataConstants.KEY_FORMAT);
-
try {
while (!done) {
// List the objects from the start, or from the last marker in case of truncated result
@@ -110,7 +146,7 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
}
// Collect the paths to files only
- collectFilesOnly(listObjectsResponse.contents(), fileFormat);
+ collectAndFilterFiles(listObjectsResponse.contents(), p, matchersList, filesOnly);
// Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
if (!listObjectsResponse.isTruncated()) {
@@ -140,33 +176,20 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
* a file if it does not end up with a "/" which is the separator in a folder structure.
*
* @param s3Objects List of returned objects
- * @param fileFormat The expected file format
- *
- * @throws AsterixException AsterixException
*/
- private void collectFilesOnly(List<S3Object> s3Objects, String fileFormat) throws AsterixException {
- String fileExtension = getFileExtension(fileFormat);
- if (fileExtension == null) {
- throw AsterixException.create(ErrorCode.PROVIDER_STREAM_RECORD_READER_UNKNOWN_FORMAT, fileFormat);
- }
-
- s3Objects.stream().filter(object -> isValidFile(object.key(), fileFormat)).forEach(filesOnly::add);
- }
-
- /**
- * Checks if the file name is of the provided format, or in the provided format in a compressed (.gz or .gzip) state
- *
- * @param fileName file name to be checked
- * @param format expected format
- * @return {@code true} if the file name is of the expected format, {@code false} otherwise
- */
- private boolean isValidFile(String fileName, String format) {
- String lowCaseName = fileName.toLowerCase();
- String lowCaseFormat = format.toLowerCase();
- String gzExt = lowCaseFormat + ".gz";
- String gzipExt = lowCaseFormat + ".gzip";
+ private void collectAndFilterFiles(List<S3Object> s3Objects, BiPredicate<List<Matcher>, String> predicate,
+ List<Matcher> matchers, List<S3Object> filesOnly) {
+ for (S3Object object : s3Objects) {
+ // skip folders
+ if (object.key().endsWith("/")) {
+ continue;
+ }
- return lowCaseName.endsWith(lowCaseFormat) || lowCaseName.endsWith(gzExt) || lowCaseName.endsWith(gzipExt);
+ // No filter, add file
+ if (predicate.test(matchers, object.key())) {
+ filesOnly.add(object);
+ }
+ }
}
/**
@@ -216,29 +239,9 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
return smallest;
}
- /**
- * Returns the file extension for the provided file format.
- *
- * @param format file format
- *
- * @return file extension for the provided file format, null otherwise.
- */
- private String getFileExtension(String format) {
- switch (format.toLowerCase()) {
- case ExternalDataConstants.FORMAT_JSON_LOWER_CASE:
- return ".json";
- case ExternalDataConstants.FORMAT_CSV:
- return ".csv";
- case ExternalDataConstants.FORMAT_TSV:
- return ".tsv";
- default:
- return null;
- }
- }
-
private static class PartitionWorkLoadBasedOnSize implements Serializable {
private static final long serialVersionUID = 1L;
- private List<String> filePaths = new ArrayList<>();
+ private final List<String> filePaths = new ArrayList<>();
private long totalSize = 0;
PartitionWorkLoadBasedOnSize() {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index e93c3b9..252ed5b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -71,6 +71,8 @@ public class ExternalDataConstants {
public static final String KEY_EXPRESSION = "expression";
public static final String KEY_LOCAL_SOCKET_PATH = "local-socket-path";
public static final String KEY_FORMAT = "format";
+ public static final String KEY_INCLUDE = "include";
+ public static final String KEY_EXCLUDE = "exclude";
public static final String KEY_QUOTE = "quote";
public static final String KEY_ESCAPE = "escape";
public static final String KEY_PARSER = "parser";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 1501287..8496761 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -26,10 +26,12 @@ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.regex.Matcher;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -43,8 +45,10 @@ import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.runtime.evaluators.common.NumberUtils;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.dataflow.common.data.parsers.BooleanParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.DoubleParserFactory;
@@ -453,16 +457,18 @@ public class ExternalDataUtils {
}
/**
- * Ensures that the external source container is present
+ * Validates adapter specific external dataset properties. Specific properties for different adapters should be
+ * validated here
*
- * @param configuration external source properties
+ * @param configuration properties
*/
- public static void validateExternalSourceContainer(Map<String, String> configuration) throws CompilationException {
+ public static void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc)
+ throws CompilationException {
String type = configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE);
switch (type) {
case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
- ExternalDataUtils.AwsS3.validateExternalSourceContainer(configuration);
+ ExternalDataUtils.AwsS3.validateProperties(configuration, srcLoc);
break;
default:
// Nothing needs to be done
@@ -470,11 +476,109 @@ public class ExternalDataUtils {
}
}
+ /**
+ * Regex matches all the provided patterns against the provided path
+ *
+ * @param path path to check against
+ *
+ * @return {@code true} if all patterns match, {@code false} otherwise
+ */
+ public static boolean matchPatterns(List<Matcher> matchers, String path) {
+ for (Matcher matcher : matchers) {
+ if (matcher.reset(path).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Converts the wildcard to proper regex
+ *
+ * @param wildcard wildcard pattern to convert
+ *
+ * @return regex expression
+ */
+ public static String wildcardToRegex(String wildcard) {
+ StringBuilder builder = new StringBuilder(wildcard.length());
+ builder.append('^');
+
+ // This keeps an eye on the presence inside or outside a sequence, everything inside a sequence is a literal
+ // e.g ("*" ===> ".*" while "[*]" ===> "[\*]"
+ boolean outsideBracketSequence = true;
+
+ for (int i = 0; i < wildcard.length(); i++) {
+ char c = wildcard.charAt(i);
+ switch (c) {
+ case '*':
+ builder.append(outsideBracketSequence ? "." : "\\").append(c);
+ break;
+ case '?':
+ builder.append(outsideBracketSequence ? "." : "\\?");
+ break;
+ case '[':
+ if (outsideBracketSequence) {
+ outsideBracketSequence = false;
+ builder.append(c);
+ if (i + 1 < wildcard.length()) {
+ if (wildcard.charAt(i + 1) == '!') {
+ i++;
+ builder.append('^');
+ }
+ }
+ } else {
+ // escape the open bracket "[" if we are already inside a bracket sequence
+ builder.append("\\").append(c);
+ }
+ break;
+ case ']':
+ if (outsideBracketSequence) {
+ // escape if we are outside bracket sequence
+ builder.append("\\").append(c);
+ } else {
+ // Inside bracket, close it and mark as outside bracket
+ outsideBracketSequence = true;
+ builder.append(c);
+ }
+ break;
+ // escape special regexp-characters
+ case '(':
+ case ')':
+ case '$':
+ case '^':
+ case '.':
+ case '{':
+ case '}':
+ case '|':
+ case '+':
+ case '=':
+ case '<':
+ case '>':
+ case '!':
+ case '\\':
+ builder.append("\\").append(c);
+ break;
+ default:
+ builder.append(c);
+ break;
+ }
+ }
+ builder.append('$');
+ return builder.toString();
+ }
+
public static class AwsS3 {
private AwsS3() {
throw new AssertionError("do not instantiate");
}
+ /**
+ * Builds the S3 client using the provided configuration
+ *
+ * @param configuration properties
+ * @return S3 client
+ * @throws CompilationException CompilationException
+ */
public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
@@ -518,16 +622,24 @@ public class ExternalDataUtils {
}
/**
- * Validates if the container being used is available or not.
+ * Validate external dataset properties
*
- * @param configuration external datasource configuration
+ * @param configuration properties
*
* @throws CompilationException Compilation exception
*/
- public static void validateExternalSourceContainer(Map<String, String> configuration)
+ public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc)
throws CompilationException {
- S3Client s3Client = null;
+ // check if the format property is present
+ if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+ throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
+ }
+
+ validateIncludeExclude(configuration);
+
+ // Check if the bucket is present
+ S3Client s3Client = null;
try {
String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME);
s3Client = buildAwsS3Client(configuration);
@@ -547,5 +659,51 @@ public class ExternalDataUtils {
}
}
}
+
+ /**
+ * TODO(Hussain)
+ * @param configuration
+ * @throws CompilationException
+ */
+ public static void validateIncludeExclude(Map<String, String> configuration) throws CompilationException {
+ // Ensure that include and exclude are not provided at the same time + ensure valid format or property
+ List<Map.Entry<String, String>> includes = new ArrayList<>();
+ List<Map.Entry<String, String>> excludes = new ArrayList<>();
+
+ // Accepted formats are include, include#1, include#2, ... etc, same for excludes
+ for (Map.Entry<String, String> entry : configuration.entrySet()) {
+ String key = entry.getKey();
+
+ if (key.equals(ExternalDataConstants.KEY_INCLUDE)) {
+ includes.add(entry);
+ } else if (key.equals(ExternalDataConstants.KEY_EXCLUDE)) {
+ excludes.add(entry);
+ } else if (key.startsWith(ExternalDataConstants.KEY_INCLUDE)
+ || key.startsWith(ExternalDataConstants.KEY_EXCLUDE)) {
+
+ // Split by the "#", length should be 2, left should be include/exclude, right should be integer
+ String[] splits = key.split("#");
+
+ if (key.startsWith(ExternalDataConstants.KEY_INCLUDE) && splits.length == 2
+ && splits[0].equals(ExternalDataConstants.KEY_INCLUDE)
+ && NumberUtils.isIntegerNumericString(splits[1])) {
+ includes.add(entry);
+ } else if (key.startsWith(ExternalDataConstants.KEY_EXCLUDE) && splits.length == 2
+ && splits[0].equals(ExternalDataConstants.KEY_EXCLUDE)
+ && NumberUtils.isIntegerNumericString(splits[1])) {
+ excludes.add(entry);
+ } else {
+ throw new CompilationException(ErrorCode.INVALID_PROPERTY_FORMAT, key);
+ }
+ }
+ }
+
+ // TODO: Should include/exclude be a common check or S3 specific?
+ // Ensure either include or exclude are provided, but not both of them
+ if (!includes.isEmpty() && !excludes.isEmpty()) {
+ throw new CompilationException(ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME,
+ ExternalDataConstants.KEY_INCLUDE, ExternalDataConstants.KEY_EXCLUDE);
+ }
+ }
}
}