You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ht...@apache.org on 2020/06/16 11:11:21 UTC
[asterixdb] 01/03: [ASTERIXDB-2741][EXT] Support include/exclude
filters for S3 external datasets
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0163af68d0cd1d79d1dc95655f274fced2da48ca
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Mon Jun 8 16:38:53 2020 +0300
[ASTERIXDB-2741][EXT] Support include/exclude filters for S3 external datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added support for include/exclude filters.
- Added test cases for include/exclude.
Change-Id: I21dbc5c08456f95a2cad1a8fd9147472fad15376
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6604
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../asterix/app/translator/QueryTranslator.java | 14 +-
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 6 +-
.../aws/AwsS3ExternalDatasetTest.java | 154 ++++++++++++++----
.../external_dataset.000.ddl.sqlpp | 2 +-
.../bad-name-1/test.000.ddl.sqlpp} | 8 +-
.../bad-name-1/test.099.ddl.sqlpp} | 19 +--
.../bad-name-2/test.000.ddl.sqlpp} | 8 +-
.../bad-name-2/test.099.ddl.sqlpp} | 19 +--
.../bad-name-3/test.000.ddl.sqlpp} | 8 +-
.../bad-name-3/test.099.ddl.sqlpp} | 19 +--
.../both/test.000.ddl.sqlpp} | 8 +-
.../both/test.099.ddl.sqlpp} | 19 +--
.../exclude-1/test.000.ddl.sqlpp} | 7 +-
.../exclude-1/test.001.query.sqlpp} | 18 +--
.../exclude-1/test.099.ddl.sqlpp} | 19 +--
.../exclude-2/test.000.ddl.sqlpp} | 9 +-
.../exclude-2/test.001.query.sqlpp} | 18 +--
.../exclude-2/test.099.ddl.sqlpp} | 19 +--
.../exclude-3/test.000.ddl.sqlpp} | 8 +-
.../exclude-3/test.001.query.sqlpp} | 18 +--
.../exclude-3/test.099.ddl.sqlpp} | 19 +--
.../exclude-4/test.000.ddl.sqlpp} | 8 +-
.../exclude-4/test.001.query.sqlpp} | 18 +--
.../exclude-4/test.099.ddl.sqlpp} | 19 +--
.../exclude-5/test.000.ddl.sqlpp} | 8 +-
.../exclude-5/test.001.query.sqlpp} | 18 +--
.../exclude-5/test.099.ddl.sqlpp} | 19 +--
.../exclude-6/test.000.ddl.sqlpp} | 7 +-
.../exclude-6/test.001.query.sqlpp} | 18 +--
.../exclude-6/test.099.ddl.sqlpp} | 19 +--
.../exclude-all/test.000.ddl.sqlpp} | 7 +-
.../exclude-all/test.001.query.sqlpp} | 18 +--
.../exclude-all/test.099.ddl.sqlpp} | 19 +--
.../include-1/test.000.ddl.sqlpp} | 7 +-
.../include-1/test.001.query.sqlpp} | 18 +--
.../include-1/test.099.ddl.sqlpp} | 19 +--
.../include-10/test.000.ddl.sqlpp} | 8 +-
.../include-10/test.001.query.sqlpp} | 18 +--
.../include-10/test.099.ddl.sqlpp} | 19 +--
.../include-2/test.000.ddl.sqlpp} | 7 +-
.../include-2/test.001.query.sqlpp} | 18 +--
.../include-2/test.002.query.sqlpp} | 18 +--
.../include-2/test.099.ddl.sqlpp} | 19 +--
.../include-3/test.000.ddl.sqlpp} | 10 +-
.../include-3/test.001.query.sqlpp} | 18 +--
.../include-3/test.099.ddl.sqlpp} | 19 +--
.../include-4/test.000.ddl.sqlpp} | 10 +-
.../include-4/test.001.query.sqlpp} | 18 +--
.../include-4/test.099.ddl.sqlpp} | 19 +--
.../include-5/test.000.ddl.sqlpp} | 7 +-
.../include-5/test.001.query.sqlpp} | 18 +--
.../include-5/test.099.ddl.sqlpp} | 19 +--
.../include-6/test.000.ddl.sqlpp} | 7 +-
.../include-6/test.001.query.sqlpp} | 18 +--
.../include-6/test.099.ddl.sqlpp} | 19 +--
.../include-7/test.000.ddl.sqlpp} | 8 +-
.../include-7/test.001.query.sqlpp} | 18 +--
.../include-7/test.099.ddl.sqlpp} | 19 +--
.../include-8/test.000.ddl.sqlpp} | 8 +-
.../include-8/test.001.query.sqlpp} | 18 +--
.../include-8/test.099.ddl.sqlpp} | 19 +--
.../include-9/test.000.ddl.sqlpp} | 8 +-
.../include-9/test.001.query.sqlpp} | 18 +--
.../include-9/test.099.ddl.sqlpp} | 19 +--
.../include-all/test.000.ddl.sqlpp} | 7 +-
.../include-all/test.001.query.sqlpp} | 18 +--
.../include-all/test.099.ddl.sqlpp} | 19 +--
.../s3/include-exclude/exclude-1/result.001.adm | 1 +
.../s3/include-exclude/exclude-2/result.001.adm | 1 +
.../s3/include-exclude/exclude-3/result.001.adm | 1 +
.../s3/include-exclude/exclude-4/result.001.adm | 1 +
.../s3/include-exclude/exclude-5/result.001.adm | 1 +
.../s3/include-exclude/exclude-6/result.001.adm | 1 +
.../s3/include-exclude/exclude-all/result.001.adm | 1 +
.../s3/include-exclude/include-1/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.001.adm | 1 +
.../s3/include-exclude/include-2/result.002.adm | 4 +
.../s3/include-exclude/include-3/result.001.adm | 1 +
.../s3/include-exclude/include-4/result.001.adm | 1 +
.../s3/include-exclude/include-5/result.001.adm | 1 +
.../s3/include-exclude/include-6/result.001.adm | 1 +
.../s3/include-exclude/include-7/result.001.adm | 1 +
.../s3/include-exclude/include-8/result.001.adm | 1 +
.../s3/include-exclude/include-9/result.001.adm | 1 +
.../runtimets/testsuite_external_dataset.xml | 118 ++++++++++++++
.../asterix/common/exceptions/ErrorCode.java | 4 +
.../src/main/resources/asx_errormsg/en.properties | 4 +
.../record/reader/aws/AwsS3InputStreamFactory.java | 113 ++++++-------
.../external/util/ExternalDataConstants.java | 2 +
.../asterix/external/util/ExternalDataUtils.java | 174 ++++++++++++++++++++-
90 files changed, 656 insertions(+), 888 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index b814b0a..1be3894 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -3240,16 +3240,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
protected void validateExternalDatasetProperties(ExternalDetailsDecl externalDetails,
Map<String, String> properties, SourceLocation srcLoc, MetadataTransactionContext mdTxnCtx)
throws AlgebricksException, HyracksDataException {
+ // Validate adapter specific properties
String adapter = externalDetails.getAdapter();
- // "format" parameter is needed for "S3" data source
- if (ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3.equals(adapter)
- && properties.get(ExternalDataConstants.KEY_FORMAT) == null) {
- throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
- }
-
Map<String, String> details = new HashMap<>(properties);
details.put(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE, adapter);
- validateExternalSourceContainer(details);
+ validateAdapterSpecificProperties(details, srcLoc);
}
/**
@@ -3257,7 +3252,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
*
* @param configuration external source properties
*/
- protected void validateExternalSourceContainer(Map<String, String> configuration) throws CompilationException {
- ExternalDataUtils.validateExternalSourceContainer(configuration);
+ protected void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc)
+ throws CompilationException {
+ ExternalDataUtils.validateAdapterSpecificProperties(configuration, srcLoc);
}
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
index bd433f1..48e966e 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
@@ -43,8 +43,9 @@ public class AwsS3ExternalDatasetOnePartitionTest extends AwsS3ExternalDatasetTe
SUITE_TESTS = "testsuite_external_dataset_one_partition.xml";
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
- PREPARE_S3_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
+ PREPARE_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareFixedDataBucket;
+ PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareMixedDataBucket;
return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@@ -53,4 +54,7 @@ public class AwsS3ExternalDatasetOnePartitionTest extends AwsS3ExternalDatasetTe
private static void prepareFixedDataBucket() {
}
+
+ private static void prepareMixedDataBucket() {
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 23bf3bd..8ec311d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -83,25 +83,28 @@ public class AwsS3ExternalDatasetTest {
static String SUITE_TESTS;
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
- static Runnable PREPARE_S3_BUCKET;
+ static Runnable PREPARE_BUCKET;
static Runnable PREPARE_FIXED_DATA_BUCKET;
+ static Runnable PREPARE_MIXED_DATA_BUCKET;
// Base directory paths for data files
private static final String JSON_DATA_PATH = joinPath("data", "json");
private static final String CSV_DATA_PATH = joinPath("data", "csv");
private static final String TSV_DATA_PATH = joinPath("data", "tsv");
+ private static final String MIXED_DATA_PATH = joinPath("data", "mixed");
// Service endpoint
- private static final int S3_MOCK_SERVER_PORT = 8001;
- private static final String S3_MOCK_SERVER_HOSTNAME = "http://localhost:" + S3_MOCK_SERVER_PORT;
+ private static final int MOCK_SERVER_PORT = 8001;
+ private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
// Region, bucket and definitions
- private static final String S3_MOCK_SERVER_REGION = "us-west-2";
- private static final String S3_MOCK_SERVER_BUCKET = "playground";
- private static final String S3_MOCK_SERVER_FIXED_DATA_BUCKET = "fixed-data-bucket"; // Do not use, has fixed data
- private static final String S3_MOCK_SERVER_BUCKET_JSON_DEFINITION = "json-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_BUCKET_CSV_DEFINITION = "csv-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_BUCKET_TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
+ private static final String MOCK_SERVER_REGION = "us-west-2";
+ private static final String MOCK_SERVER_BUCKET = "playground";
+ private static final String FIXED_DATA_BUCKET = "fixed-data"; // Do not use, has fixed data
+ private static final String INCLUDE_EXCLUDE_BUCKET = "include-exclude"; // include & exclude bucket
+ private static final String JSON_DEFINITION = "json-data/reviews/"; // data resides here
+ private static final String CSV_DEFINITION = "csv-data/reviews/"; // data resides here
+ private static final String TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
// This is used for a test to generate over 1000 number of files
private static final String OVER_1000_OBJECTS_PATH = "over-1000-objects";
@@ -115,7 +118,9 @@ public class AwsS3ExternalDatasetTest {
// IMPORTANT: The following values must be used in the AWS S3 test case
private static S3Mock s3MockServer;
private static S3Client client;
- private static PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET);
+ private static final PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(MOCK_SERVER_BUCKET);
+ private static final PutObjectRequest.Builder includeExcludeBuilder =
+ PutObjectRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET);
protected TestCaseContext tcCtx;
@@ -151,8 +156,9 @@ public class AwsS3ExternalDatasetTest {
SUITE_TESTS = "testsuite_external_dataset.xml";
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
- PREPARE_S3_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
+ PREPARE_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareFixedDataBucket;
+ PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareMixedDataBucket;
return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@@ -180,30 +186,31 @@ public class AwsS3ExternalDatasetTest {
private static void startAwsS3MockServer() {
// Starting S3 mock server to be used instead of real S3 server
LOGGER.info("Starting S3 mock server");
- s3MockServer = new S3Mock.Builder().withPort(S3_MOCK_SERVER_PORT).withInMemoryBackend().build();
+ s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
s3MockServer.start();
LOGGER.info("S3 mock server started successfully");
// Create a client and add some files to the S3 mock server
LOGGER.info("Creating S3 client to load initial files to S3 mock server");
S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(S3_MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
- builder.region(Region.of(S3_MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+ URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
.endpointOverride(endpoint);
client = builder.build();
LOGGER.info("Client created successfully");
// Create the bucket and upload some json files
- PREPARE_S3_BUCKET.run();
+ PREPARE_BUCKET.run();
PREPARE_FIXED_DATA_BUCKET.run();
+ PREPARE_MIXED_DATA_BUCKET.run();
}
/**
* Creates a bucket and fills it with some files for testing purpose.
*/
private static void prepareS3Bucket() {
- LOGGER.info("creating bucket " + S3_MOCK_SERVER_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(S3_MOCK_SERVER_BUCKET).build());
+ LOGGER.info("creating bucket " + MOCK_SERVER_BUCKET);
+ client.createBucket(CreateBucketRequest.builder().bucket(MOCK_SERVER_BUCKET).build());
LOGGER.info("bucket created successfully");
LOGGER.info("Adding JSON files to the bucket");
@@ -228,24 +235,24 @@ public class AwsS3ExternalDatasetTest {
* changed, the test case will fail and its result will need to be updated each time
*/
private static void prepareFixedDataBucket() {
- LOGGER.info("creating bucket " + S3_MOCK_SERVER_FIXED_DATA_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).build());
- LOGGER.info("bucket " + S3_MOCK_SERVER_FIXED_DATA_BUCKET + " created successfully");
+ LOGGER.info("creating bucket " + FIXED_DATA_BUCKET);
+ client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_BUCKET).build());
+ LOGGER.info("bucket " + FIXED_DATA_BUCKET + " created successfully");
- LOGGER.info("Loading fixed data to " + S3_MOCK_SERVER_FIXED_DATA_BUCKET);
+ LOGGER.info("Loading fixed data to " + FIXED_DATA_BUCKET);
// Files data
RequestBody requestBody = RequestBody.fromFile(Paths.get(JSON_DATA_PATH, "single-line", "20-records.json"));
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("1.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("2.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("lvl1/3.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("lvl1/4.json").build(), requestBody);
- client.putObject(builder.bucket(S3_MOCK_SERVER_FIXED_DATA_BUCKET).key("lvl1/lvl2/5.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("1.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("2.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/3.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/4.json").build(), requestBody);
+ client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/lvl2/5.json").build(), requestBody);
}
private static void loadJsonFiles() {
String dataBasePath = JSON_DATA_PATH;
- String definition = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION;
+ String definition = JSON_DEFINITION;
// Normal format
String definitionSegment = "json";
@@ -279,7 +286,7 @@ public class AwsS3ExternalDatasetTest {
private static void loadCsvFiles() {
String dataBasePath = CSV_DATA_PATH;
- String definition = S3_MOCK_SERVER_BUCKET_CSV_DEFINITION;
+ String definition = CSV_DEFINITION;
// Normal format
String definitionSegment = "csv";
@@ -301,7 +308,7 @@ public class AwsS3ExternalDatasetTest {
private static void loadTsvFiles() {
String dataBasePath = TSV_DATA_PATH;
- String definition = S3_MOCK_SERVER_BUCKET_TSV_DEFINITION;
+ String definition = TSV_DEFINITION;
// Normal format
String definitionSegment = "tsv";
@@ -399,6 +406,95 @@ public class AwsS3ExternalDatasetTest {
}
}
+ /**
+ * Loads a combination of different file formats in the same path
+ */
+ private static void prepareMixedDataBucket() {
+ LOGGER.info("creating bucket " + INCLUDE_EXCLUDE_BUCKET);
+ client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET).build());
+ LOGGER.info("bucket " + INCLUDE_EXCLUDE_BUCKET + " created successfully");
+
+ // JSON
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+
+ // CSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2018.csv").build(),
+ RequestBody.fromString("7,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2019.csv").build(),
+ RequestBody.fromString("8,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2020.csv").build(),
+ RequestBody.fromString("{9,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2018.csv").build(),
+ RequestBody.fromString("10,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2019.csv").build(),
+ RequestBody.fromString("11,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2020.csv").build(),
+ RequestBody.fromString("12,\"good\""));
+
+ // TSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2018.tsv").build(),
+ RequestBody.fromString("13\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2019.tsv").build(),
+ RequestBody.fromString("14\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2020.tsv").build(),
+ RequestBody.fromString("15\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2018.tsv").build(),
+ RequestBody.fromString("16\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2019.tsv").build(),
+ RequestBody.fromString("17\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2020.tsv").build(),
+ RequestBody.fromString("18\t\"good\""));
+
+ // JSON no extension
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+ }
+
static class AwsTestExecutor extends TestExecutor {
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
index bb95094..a3bfd53 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
@@ -31,7 +31,7 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
+("container"="fixed-data"),
("definition"=""),
("format"="json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.000.ddl.sqlpp
index bb95094..1b463bf 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.json"),
+("exclude1"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-1/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.000.ddl.sqlpp
index bb95094..fe3922d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.json"),
+("exclude#"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-2/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.000.ddl.sqlpp
index bb95094..efeffc4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.json"),
+("exclude#hello"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/bad-name-3/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.000.ddl.sqlpp
index bb95094..542046d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*.json"),
+("exclude"="*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/both/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.000.ddl.sqlpp
index bb95094..6e84301 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.?sv")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-1/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.000.ddl.sqlpp
similarity index 84%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.000.ddl.sqlpp
index bb95094..0ce88b7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.000.ddl.sqlpp
@@ -31,7 +31,10 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#1"="data/mixed/json/extension*"),
+("exclude#100"="data/mixed/json/EXTENSION*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-2/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.000.ddl.sqlpp
index bb95094..8c05fa6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#100"="data/mixed/json*201?*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-3/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.000.ddl.sqlpp
index bb95094..e1a50b8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#1"="data/mixed/json*bye*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-4/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.000.ddl.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.000.ddl.sqlpp
index bb95094..32e115a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="data/mixed/?sv*"),
+("exclude#1"="data/mixed/json/extension/hello-world-2018.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-5/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.000.ddl.sqlpp
index bb95094..78a8310 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*.[abct][abcs][abcv]")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-6/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.000.ddl.sqlpp
index bb95094..f09079a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("exclude"="*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/exclude-all/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.000.ddl.sqlpp
index bb95094..1b12acb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*2018*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-1/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.000.ddl.sqlpp
index bb95094..32c57b7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*[abc][.*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-10/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.000.ddl.sqlpp
index bb95094..e129135 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*201?*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.002.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.002.query.sqlpp
index bb95094..9de4a37 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.002.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select value test from test order by id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-2/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.000.ddl.sqlpp
index bb95094..4203999 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.000.ddl.sqlpp
@@ -23,6 +23,8 @@ use test;
drop type test if exists;
create type test as open {
+f1: int,
+f2: string
};
drop dataset test if exists;
@@ -31,7 +33,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*201?*.csv")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-3/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.000.ddl.sqlpp
similarity index 88%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.000.ddl.sqlpp
index bb95094..92fed55 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.000.ddl.sqlpp
@@ -23,6 +23,8 @@ use test;
drop type test if exists;
create type test as open {
+f1: int,
+f2: string
};
drop dataset test if exists;
@@ -31,7 +33,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="tsv"),
+("header"=false),
+("include"="*201?*.tsv")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-4/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.000.ddl.sqlpp
index bb95094..2d38a8c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*bye*.json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-5/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.000.ddl.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.000.ddl.sqlpp
index bb95094..5b4344a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="data/mixed/json/*EXTENSION*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-6/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.000.ddl.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.000.ddl.sqlpp
index bb95094..e4e2050 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="data/mixed/json/NO-EXTENSION*"),
+("include#0"="data/mixed/json/EXTENSION*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-7/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.000.ddl.sqlpp
index bb95094..7d093b2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*.[!xyt][!xyz][!xyz]")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-8/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.000.ddl.sqlpp
similarity index 89%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.000.ddl.sqlpp
index bb95094..9b0e7ff 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.000.ddl.sqlpp
@@ -31,7 +31,9 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="csv"),
+("header"=false),
+("include"="*.[a-c][a-z][a-z]")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-9/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.000.ddl.sqlpp
similarity index 91%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.000.ddl.sqlpp
index bb95094..5545e38 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.000.ddl.sqlpp
@@ -31,7 +31,8 @@ create external dataset test(test) using S3 (
("secretAccessKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
+("container"="include-exclude"),
+("definition"="data/mixed/"),
+("format"="json"),
+("include"="*")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.001.query.sqlpp
similarity index 67%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.001.query.sqlpp
index bb95094..3306d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.001.query.sqlpp
@@ -17,21 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+select count(*) as `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.099.ddl.sqlpp
similarity index 66%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.099.ddl.sqlpp
index bb95094..548e632 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/empty-string-definition/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/include-exclude/include-all/test.099.ddl.sqlpp
@@ -17,21 +17,4 @@
* under the License.
*/
-drop dataverse test if exists;
-create dataverse test;
-use test;
-
-drop type test if exists;
-create type test as open {
-};
-
-drop dataset test if exists;
-create external dataset test(test) using S3 (
-("accessKeyId"="dummyAccessKey"),
-("secretAccessKey"="dummySecretKey"),
-("region"="us-west-2"),
-("serviceEndpoint"="http://localhost:8001"),
-("container"="fixed-data-bucket"),
-("definition"=""),
-("format"="json")
-);
\ No newline at end of file
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm
new file mode 100644
index 0000000..207f123
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-1/result.001.adm
@@ -0,0 +1 @@
+{ "count": 12 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-2/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-3/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-4/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm
new file mode 100644
index 0000000..8ac9199
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-5/result.001.adm
@@ -0,0 +1 @@
+{ "count": 11 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm
new file mode 100644
index 0000000..207f123
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-6/result.001.adm
@@ -0,0 +1 @@
+{ "count": 12 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-all/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-all/result.001.adm
new file mode 100644
index 0000000..c1a0ea2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/exclude-all/result.001.adm
@@ -0,0 +1 @@
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm
new file mode 100644
index 0000000..df462fe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-1/result.001.adm
@@ -0,0 +1 @@
+{ "count": 2 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm
new file mode 100644
index 0000000..9b5f94a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-2/result.002.adm
@@ -0,0 +1,4 @@
+{ "id": 1 }
+{ "id": 2 }
+{ "id": 4 }
+{ "id": 5 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-3/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm
new file mode 100644
index 0000000..09eace4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-4/result.001.adm
@@ -0,0 +1 @@
+{ "count": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm
new file mode 100644
index 0000000..8637a38
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-5/result.001.adm
@@ -0,0 +1 @@
+{ "count": 3 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-6/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-7/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-8/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm
new file mode 100644
index 0000000..95204aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/include-exclude/include-9/result.001.adm
@@ -0,0 +1 @@
+{ "count": 6 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 7c1aa48..7c4cf37 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -100,4 +100,122 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="include-exclude">
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/bad-name-1">
+ <output-dir compare="Text">aws/s3/include-exclude/bad-name-1</output-dir>
+ <expected-error>Invalid format for property "exclude1"</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/bad-name-2">
+ <output-dir compare="Text">aws/s3/include-exclude/bad-name-2</output-dir>
+ <expected-error>Invalid format for property "exclude#"</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/bad-name-3">
+ <output-dir compare="Text">aws/s3/include-exclude/bad-name-3</output-dir>
+ <expected-error>Invalid format for property "exclude#hello"</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/both">
+ <output-dir compare="Text">aws/s3/include-exclude/both</output-dir>
+ <expected-error>The parameters "include" and "exclude" cannot be provided at the same time</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-all">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-all</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-1">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-2">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-3">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-4">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-5">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/exclude-6">
+ <output-dir compare="Text">aws/s3/include-exclude/exclude-6</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-all">
+ <output-dir compare="Text">aws/s3/include-exclude/include-all</output-dir>
+ <expected-error>Malformed input stream</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-1">
+ <output-dir compare="Text">aws/s3/include-exclude/include-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-2">
+ <output-dir compare="Text">aws/s3/include-exclude/include-2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-3">
+ <output-dir compare="Text">aws/s3/include-exclude/include-3</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-4">
+ <output-dir compare="Text">aws/s3/include-exclude/include-4</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-5">
+ <output-dir compare="Text">aws/s3/include-exclude/include-5</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-6">
+ <output-dir compare="Text">aws/s3/include-exclude/include-6</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-7">
+ <output-dir compare="Text">aws/s3/include-exclude/include-7</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-8">
+ <output-dir compare="Text">aws/s3/include-exclude/include-8</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-9">
+ <output-dir compare="Text">aws/s3/include-exclude/include-9</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/include-exclude/include-10">
+ <output-dir compare="Text">aws/s3/include-exclude/include-10</output-dir>
+ <expected-error>Invalid pattern *[abc][.*</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
</test-suite>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 6496c94..46a6228 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -198,6 +198,10 @@ public class ErrorCode {
public static final int UNEXPECTED_HINT = 1107;
public static final int EXTERNAL_SOURCE_ERROR = 1108;
public static final int EXTERNAL_SOURCE_CONTAINER_NOT_FOUND = 1109;
+ public static final int PARAMETERS_NOT_ALLOWED_AT_SAME_TIME = 1110;
+ public static final int PROPERTY_INVALID_VALUE_TYPE = 1111;
+ public static final int INVALID_PROPERTY_FORMAT = 1112;
+ public static final int INVALID_REGEX_PATTERN = 1113;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 7e75a51..b070552 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -193,6 +193,10 @@
1107 = Unexpected hint: %1$s. %2$s expected at this location
1108 = External source error. %1$s
1109 = External source container %1$s not found
+1110 = The parameters \"%1$s\" and \"%2$s\" cannot be provided at the same time
+1111 = Property \"%1$s\" expects value(s) of type %2$s
+1112 = Invalid format for property \"%1$s\"
+1113 = Invalid pattern %1$s
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index e57b31a..6ef5a23 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -19,19 +19,24 @@
package org.apache.asterix.external.input.record.reader.aws;
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXCLUDE;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_INCLUDE;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.function.BiPredicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
-import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IInputStreamFactory;
-import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -49,12 +54,9 @@ import software.amazon.awssdk.services.s3.model.S3Object;
public class AwsS3InputStreamFactory implements IInputStreamFactory {
private static final long serialVersionUID = 1L;
- private Map<String, String> configuration;
- // Files to read from
- private final List<S3Object> filesOnly = new ArrayList<>();
+ private Map<String, String> configuration;
private final List<PartitionWorkLoadBasedOnSize> partitionWorkLoadsBasedOnSize = new ArrayList<>();
-
private transient AlgebricksAbsolutePartitionConstraint partitionConstraint;
@Override
@@ -84,6 +86,42 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
String container = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+ List<S3Object> filesOnly = new ArrayList<>();
+
+ // Ensure the validity of include/exclude
+ ExternalDataUtils.AwsS3.validateIncludeExclude(configuration);
+
+ // Get and compile the patterns for include/exclude if provided
+ List<Matcher> includeMatchers = new ArrayList<>();
+ List<Matcher> excludeMatchers = new ArrayList<>();
+ String pattern = null;
+ try {
+ for (Map.Entry<String, String> entry : configuration.entrySet()) {
+ if (entry.getKey().startsWith(KEY_INCLUDE)) {
+ pattern = entry.getValue();
+ includeMatchers.add(Pattern.compile(ExternalDataUtils.wildcardToRegex(pattern)).matcher(""));
+ } else if (entry.getKey().startsWith(KEY_EXCLUDE)) {
+ pattern = entry.getValue();
+ excludeMatchers.add(Pattern.compile(ExternalDataUtils.wildcardToRegex(pattern)).matcher(""));
+ }
+ }
+ } catch (PatternSyntaxException ex) {
+ throw new CompilationException(ErrorCode.INVALID_REGEX_PATTERN, pattern);
+ }
+
+ List<Matcher> matchersList;
+ BiPredicate<List<Matcher>, String> p;
+ if (!includeMatchers.isEmpty()) {
+ matchersList = includeMatchers;
+ p = (matchers, key) -> ExternalDataUtils.matchPatterns(matchers, key);
+ } else if (!excludeMatchers.isEmpty()) {
+ matchersList = excludeMatchers;
+ p = (matchers, key) -> !ExternalDataUtils.matchPatterns(matchers, key);
+ } else {
+ matchersList = Collections.emptyList();
+ p = (matchers, key) -> true;
+ }
+
S3Client s3Client = ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
// Get all objects in a bucket and extract the paths to files
@@ -97,8 +135,6 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
boolean done = false;
String newMarker = null;
- String fileFormat = configuration.get(ExternalDataConstants.KEY_FORMAT);
-
try {
while (!done) {
// List the objects from the start, or from the last marker in case of truncated result
@@ -110,7 +146,7 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
}
// Collect the paths to files only
- collectFilesOnly(listObjectsResponse.contents(), fileFormat);
+ collectAndFilterFiles(listObjectsResponse.contents(), p, matchersList, filesOnly);
// Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
if (!listObjectsResponse.isTruncated()) {
@@ -140,33 +176,20 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
* a file if it does not end up with a "/" which is the separator in a folder structure.
*
* @param s3Objects List of returned objects
- * @param fileFormat The expected file format
- *
- * @throws AsterixException AsterixException
*/
- private void collectFilesOnly(List<S3Object> s3Objects, String fileFormat) throws AsterixException {
- String fileExtension = getFileExtension(fileFormat);
- if (fileExtension == null) {
- throw AsterixException.create(ErrorCode.PROVIDER_STREAM_RECORD_READER_UNKNOWN_FORMAT, fileFormat);
- }
-
- s3Objects.stream().filter(object -> isValidFile(object.key(), fileFormat)).forEach(filesOnly::add);
- }
-
- /**
- * Checks if the file name is of the provided format, or in the provided format in a compressed (.gz or .gzip) state
- *
- * @param fileName file name to be checked
- * @param format expected format
- * @return {@code true} if the file name is of the expected format, {@code false} otherwise
- */
- private boolean isValidFile(String fileName, String format) {
- String lowCaseName = fileName.toLowerCase();
- String lowCaseFormat = format.toLowerCase();
- String gzExt = lowCaseFormat + ".gz";
- String gzipExt = lowCaseFormat + ".gzip";
+ private void collectAndFilterFiles(List<S3Object> s3Objects, BiPredicate<List<Matcher>, String> predicate,
+ List<Matcher> matchers, List<S3Object> filesOnly) {
+ for (S3Object object : s3Objects) {
+ // skip folders
+ if (object.key().endsWith("/")) {
+ continue;
+ }
- return lowCaseName.endsWith(lowCaseFormat) || lowCaseName.endsWith(gzExt) || lowCaseName.endsWith(gzipExt);
+ // No filter, add file
+ if (predicate.test(matchers, object.key())) {
+ filesOnly.add(object);
+ }
+ }
}
/**
@@ -216,29 +239,9 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
return smallest;
}
- /**
- * Returns the file extension for the provided file format.
- *
- * @param format file format
- *
- * @return file extension for the provided file format, null otherwise.
- */
- private String getFileExtension(String format) {
- switch (format.toLowerCase()) {
- case ExternalDataConstants.FORMAT_JSON_LOWER_CASE:
- return ".json";
- case ExternalDataConstants.FORMAT_CSV:
- return ".csv";
- case ExternalDataConstants.FORMAT_TSV:
- return ".tsv";
- default:
- return null;
- }
- }
-
private static class PartitionWorkLoadBasedOnSize implements Serializable {
private static final long serialVersionUID = 1L;
- private List<String> filePaths = new ArrayList<>();
+ private final List<String> filePaths = new ArrayList<>();
private long totalSize = 0;
PartitionWorkLoadBasedOnSize() {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index e93c3b9..252ed5b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -71,6 +71,8 @@ public class ExternalDataConstants {
public static final String KEY_EXPRESSION = "expression";
public static final String KEY_LOCAL_SOCKET_PATH = "local-socket-path";
public static final String KEY_FORMAT = "format";
+ public static final String KEY_INCLUDE = "include";
+ public static final String KEY_EXCLUDE = "exclude";
public static final String KEY_QUOTE = "quote";
public static final String KEY_ESCAPE = "escape";
public static final String KEY_PARSER = "parser";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 1501287..8496761 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -26,10 +26,12 @@ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.regex.Matcher;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -43,8 +45,10 @@ import org.apache.asterix.external.api.IRecordReaderFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.runtime.evaluators.common.NumberUtils;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.dataflow.common.data.parsers.BooleanParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.DoubleParserFactory;
@@ -453,16 +457,18 @@ public class ExternalDataUtils {
}
/**
- * Ensures that the external source container is present
+ * Validates adapter specific external dataset properties. Specific properties for different adapters should be
+ * validated here
*
- * @param configuration external source properties
+ * @param configuration properties
*/
- public static void validateExternalSourceContainer(Map<String, String> configuration) throws CompilationException {
+ public static void validateAdapterSpecificProperties(Map<String, String> configuration, SourceLocation srcLoc)
+ throws CompilationException {
String type = configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE);
switch (type) {
case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
- ExternalDataUtils.AwsS3.validateExternalSourceContainer(configuration);
+ ExternalDataUtils.AwsS3.validateProperties(configuration, srcLoc);
break;
default:
// Nothing needs to be done
@@ -470,11 +476,109 @@ public class ExternalDataUtils {
}
}
+ /**
+ * Regex matches all the provided patterns against the provided path
+ *
+ * @param path path to check against
+ *
+ * @return {@code true} if all patterns match, {@code false} otherwise
+ */
+ public static boolean matchPatterns(List<Matcher> matchers, String path) {
+ for (Matcher matcher : matchers) {
+ if (matcher.reset(path).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Converts the wildcard to proper regex
+ *
+ * @param wildcard wildcard pattern to convert
+ *
+ * @return regex expression
+ */
+ public static String wildcardToRegex(String wildcard) {
+ StringBuilder builder = new StringBuilder(wildcard.length());
+ builder.append('^');
+
+ // This keeps an eye on the presence inside or outside a sequence, everything inside a sequence is a literal
+ // e.g ("*" ===> ".*" while "[*]" ===> "[\*]"
+ boolean outsideBracketSequence = true;
+
+ for (int i = 0; i < wildcard.length(); i++) {
+ char c = wildcard.charAt(i);
+ switch (c) {
+ case '*':
+ builder.append(outsideBracketSequence ? "." : "\\").append(c);
+ break;
+ case '?':
+ builder.append(outsideBracketSequence ? "." : "\\?");
+ break;
+ case '[':
+ if (outsideBracketSequence) {
+ outsideBracketSequence = false;
+ builder.append(c);
+ if (i + 1 < wildcard.length()) {
+ if (wildcard.charAt(i + 1) == '!') {
+ i++;
+ builder.append('^');
+ }
+ }
+ } else {
+ // escape the open bracket "[" if we are already inside a bracket sequence
+ builder.append("\\").append(c);
+ }
+ break;
+ case ']':
+ if (outsideBracketSequence) {
+ // escape if we are outside bracket sequence
+ builder.append("\\").append(c);
+ } else {
+ // Inside bracket, close it and mark as outside bracket
+ outsideBracketSequence = true;
+ builder.append(c);
+ }
+ break;
+ // escape special regexp-characters
+ case '(':
+ case ')':
+ case '$':
+ case '^':
+ case '.':
+ case '{':
+ case '}':
+ case '|':
+ case '+':
+ case '=':
+ case '<':
+ case '>':
+ case '!':
+ case '\\':
+ builder.append("\\").append(c);
+ break;
+ default:
+ builder.append(c);
+ break;
+ }
+ }
+ builder.append('$');
+ return builder.toString();
+ }
+
public static class AwsS3 {
private AwsS3() {
throw new AssertionError("do not instantiate");
}
+ /**
+ * Builds the S3 client using the provided configuration
+ *
+ * @param configuration properties
+ * @return S3 client
+ * @throws CompilationException CompilationException
+ */
public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
@@ -518,16 +622,24 @@ public class ExternalDataUtils {
}
/**
- * Validates if the container being used is available or not.
+ * Validate external dataset properties
*
- * @param configuration external datasource configuration
+ * @param configuration properties
*
* @throws CompilationException Compilation exception
*/
- public static void validateExternalSourceContainer(Map<String, String> configuration)
+ public static void validateProperties(Map<String, String> configuration, SourceLocation srcLoc)
throws CompilationException {
- S3Client s3Client = null;
+ // check if the format property is present
+ if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+ throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
+ }
+
+ validateIncludeExclude(configuration);
+
+ // Check if the bucket is present
+ S3Client s3Client = null;
try {
String container = configuration.get(ExternalDataConstants.AwsS3.CONTAINER_NAME_FIELD_NAME);
s3Client = buildAwsS3Client(configuration);
@@ -547,5 +659,51 @@ public class ExternalDataUtils {
}
}
}
+
+ /**
+ * TODO(Hussain)
+ * @param configuration
+ * @throws CompilationException
+ */
+ public static void validateIncludeExclude(Map<String, String> configuration) throws CompilationException {
+ // Ensure that include and exclude are not provided at the same time + ensure valid format or property
+ List<Map.Entry<String, String>> includes = new ArrayList<>();
+ List<Map.Entry<String, String>> excludes = new ArrayList<>();
+
+ // Accepted formats are include, include#1, include#2, ... etc, same for excludes
+ for (Map.Entry<String, String> entry : configuration.entrySet()) {
+ String key = entry.getKey();
+
+ if (key.equals(ExternalDataConstants.KEY_INCLUDE)) {
+ includes.add(entry);
+ } else if (key.equals(ExternalDataConstants.KEY_EXCLUDE)) {
+ excludes.add(entry);
+ } else if (key.startsWith(ExternalDataConstants.KEY_INCLUDE)
+ || key.startsWith(ExternalDataConstants.KEY_EXCLUDE)) {
+
+ // Split by the "#", length should be 2, left should be include/exclude, right should be integer
+ String[] splits = key.split("#");
+
+ if (key.startsWith(ExternalDataConstants.KEY_INCLUDE) && splits.length == 2
+ && splits[0].equals(ExternalDataConstants.KEY_INCLUDE)
+ && NumberUtils.isIntegerNumericString(splits[1])) {
+ includes.add(entry);
+ } else if (key.startsWith(ExternalDataConstants.KEY_EXCLUDE) && splits.length == 2
+ && splits[0].equals(ExternalDataConstants.KEY_EXCLUDE)
+ && NumberUtils.isIntegerNumericString(splits[1])) {
+ excludes.add(entry);
+ } else {
+ throw new CompilationException(ErrorCode.INVALID_PROPERTY_FORMAT, key);
+ }
+ }
+ }
+
+ // TODO: Should include/exclude be a common check or S3 specific?
+ // Ensure either include or exclude are provided, but not both of them
+ if (!includes.isEmpty() && !excludes.isEmpty()) {
+ throw new CompilationException(ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME,
+ ExternalDataConstants.KEY_INCLUDE, ExternalDataConstants.KEY_EXCLUDE);
+ }
+ }
}
}