You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/05/14 20:25:58 UTC
[asterixdb] 05/26: [ASTERIXDB-2718][EXT] Add more test cases to
cover complex JSON formats
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 45b4ee37f8045eea6499c37ffdb2dcc951367203
Author: Hussain Towaileb <Hu...@Gmail.com>
AuthorDate: Thu Apr 23 12:26:12 2020 +0300
[ASTERIXDB-2718][EXT] Add more test cases to cover complex JSON formats
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added test cases for JSON single line, multiple lines, arrays, nested
objects, ... etc.
Change-Id: I3cdcd2a3f417b9a045067c2fb50249c3bdf02211
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5923
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../json/multi-lines-with-arrays/5-records.json | 19 +++
.../multi-lines-with-nested-objects/5-records.json | 32 +++++
.../data/json/multi-lines/20-records.json | 34 +++++
.../data/json/single-line/20-records.json | 20 +++
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 17 +--
.../aws/AwsS3ExternalDatasetTest.java | 154 +++++++++++----------
.../{001 => csv/000}/query-dataset.000.ddl.sqlpp | 0
.../{002 => csv/000}/query-dataset.002.query.sqlpp | 0
.../{002 => csv/000}/query-dataset.003.ddl.sqlpp | 0
.../{ => json}/000/external_dataset.000.ddl.sqlpp | 38 ++++-
.../000/external_dataset.001.query.sqlpp | 4 +-
.../000/external_dataset.002.query.sqlpp} | 4 +-
.../000/external_dataset.003.query.sqlpp} | 4 +-
.../000/external_dataset.004.query.sqlpp} | 4 +-
.../000/external_dataset.005.query.sqlpp} | 4 +-
.../000/external_dataset.006.query.sqlpp} | 4 +-
.../000/external_dataset.007.ddl.sqlpp} | 0
.../{002 => tsv/000}/query-dataset.000.ddl.sqlpp | 0
.../{001 => tsv/000}/query-dataset.002.query.sqlpp | 0
.../{001 => tsv/000}/query-dataset.003.ddl.sqlpp | 0
.../aws/s3/000/external_dataset.001.adm | 14 --
.../s3/{001 => csv/000}/external_dataset.001.adm | 0
.../aws/s3/json/000/external_dataset.002.adm | 1 +
.../aws/s3/json/000/external_dataset.003.adm | 1 +
.../aws/s3/json/000/external_dataset.004.adm | 1 +
.../aws/s3/json/000/external_dataset.005.adm | 10 ++
.../aws/s3/json/000/external_dataset.006.adm | 1 +
.../aws/s3/json/000/external_dataset.007.adm | 10 ++
.../s3/{002 => tsv/000}/external_dataset.001.adm | 0
.../runtimets/testsuite_external_dataset.xml | 12 +-
30 files changed, 264 insertions(+), 124 deletions(-)
diff --git a/asterixdb/asterix-app/data/json/multi-lines-with-arrays/5-records.json b/asterixdb/asterix-app/data/json/multi-lines-with-arrays/5-records.json
new file mode 100644
index 0000000..fd8526b
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/multi-lines-with-arrays/5-records.json
@@ -0,0 +1,19 @@
+{"id": 1, "year": null, "quarter": null, "review": "good", "array": [1, 2, 3]}
+
+
+{"id": 2, "year": null, "quarter": null, "review": "good", "array": [1, [1, 2], [1]]}
+{"id": 3, "year": 2018, "quarter": null, "review": "good", "array": [1,
+2, 3]}
+
+{"id": 4,
+ "year": 2018,
+ "quarter": null,
+ "review": "bad",
+ "array": [
+ 1,
+ 2,
+ 3
+ ]
+}
+{"id": 5,
+ "year": 2018, "quarter": 1, "review": "good", "array": [1, 2, 3]}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/multi-lines-with-nested-objects/5-records.json b/asterixdb/asterix-app/data/json/multi-lines-with-nested-objects/5-records.json
new file mode 100644
index 0000000..b89e81e
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/multi-lines-with-nested-objects/5-records.json
@@ -0,0 +1,32 @@
+{"id": 1, "year": null, "quarter": null, "review": "good", "array": [1, 2, 3], "nested": { "id" : 1}}
+
+
+{"id": 2, "year": null, "quarter": null, "review": "good", "array": [1, [1, 2], [1]], "nested": { "id" : 1}, "nested2": [ {"id":1 }]}
+{"id":3,"year":2018,"quarter":null,"review":"good","array":[1,2,3,{"nested":{"array":[{"nested":{"array":[1,2]}}]}}]}
+
+{
+ "id": 4,
+ "year": 2018,
+ "quarter": null,
+ "review": "bad",
+ "array": [
+ 1,
+ 2,
+ 3,
+ {
+ "nested1": {
+ "id": 1,
+ "nested2": {
+ "id": 2,
+ "nested3": [
+ {
+ "nested4": null
+ }
+ ]
+ }
+ }
+ }
+ ]
+}
+{"id": 5,
+ "year": 2018, "quarter": 1, "review": "good", "array": [1, 2, 3, {"nested": { "array": [1, 2]}}]}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/multi-lines/20-records.json b/asterixdb/asterix-app/data/json/multi-lines/20-records.json
new file mode 100644
index 0000000..0591790
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/multi-lines/20-records.json
@@ -0,0 +1,34 @@
+{"id": 1, "year": null, "quarter": null, "review": "good"}
+
+
+{"id": 2, "year": null, "quarter": null, "review": "good"}
+{"id": 3, "year": 2018, "quarter": null, "review": "good"}
+
+{"id": 4,
+ "year": 2018,
+ "quarter": null,
+ "review": "bad"
+}
+{"id": 5,
+ "year": 2018, "quarter": 1, "review": "good"}
+{"id": 6, "year": 2018, "quarter": 1, "review": "bad"
+}
+{"id": 7, "year": 2018, "quarter": 2, "review": "good"}
+{"id": 8, "year": 2018, "quarter": 2, "review": "bad"}
+{"id": 9, "year": 2019, "quarter": null,
+
+ "review": "good"}
+{"id": 10, "year": 2019,
+
+ "quarter": null,
+ "review": "bad"}
+{"id": 11, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 12, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 13, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 14, "year": 2019, "quarter": 2, "review": "bad"}
+{"id": 15, "year": 2019, "quarter": null, "review": "good"}
+{"id": 16, "year": 2019, "quarter": null, "review": "bad"}
+{"id": 17, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 18, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 19, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 20, "year": 2019, "quarter": 2, "review": "bad"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/single-line/20-records.json b/asterixdb/asterix-app/data/json/single-line/20-records.json
new file mode 100644
index 0000000..01176da
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/single-line/20-records.json
@@ -0,0 +1,20 @@
+{"id": 1, "year": null, "quarter": null, "review": "good"}
+{"id": 2, "year": null, "quarter": null, "review": "good"}
+{"id": 3, "year": 2018, "quarter": null, "review": "good"}
+{"id": 4, "year": 2018, "quarter": null, "review": "bad"}
+{"id": 5, "year": 2018, "quarter": 1, "review": "good"}
+{"id": 6, "year": 2018, "quarter": 1, "review": "bad"}
+{"id": 7, "year": 2018, "quarter": 2, "review": "good"}
+{"id": 8, "year": 2018, "quarter": 2, "review": "bad"}
+{"id": 9, "year": 2019, "quarter": null, "review": "good"}
+{"id": 10, "year": 2019, "quarter": null, "review": "bad"}
+{"id": 11, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 12, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 13, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 14, "year": 2019, "quarter": 2, "review": "bad"}
+{"id": 15, "year": 2019, "quarter": null, "review": "good"}
+{"id": 16, "year": 2019, "quarter": null, "review": "bad"}
+{"id": 17, "year": 2019, "quarter": 1, "review": "good"}
+{"id": 18, "year": 2019, "quarter": 1, "review": "bad"}
+{"id": 19, "year": 2019, "quarter": 2, "review": "good"}
+{"id": 20, "year": 2019, "quarter": 2, "review": "bad"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
index 6ac1259..88cd6b5 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
@@ -22,8 +22,6 @@ import java.util.Collection;
import org.apache.asterix.test.runtime.LangExecutionUtil;
import org.apache.asterix.testframework.context.TestCaseContext;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import org.junit.FixMethodOrder;
import org.junit.runner.RunWith;
import org.junit.runners.MethodSorters;
@@ -36,18 +34,17 @@ import org.junit.runners.Parameterized;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class AwsS3ExternalDatasetOnePartitionTest extends AwsS3ExternalDatasetTest {
- private static final Logger LOGGER = LogManager.getLogger();
- private static final String SUITE_PATH = "testsuite_external_dataset_one_partition.xml";
+ public AwsS3ExternalDatasetOnePartitionTest(TestCaseContext tcCtx) {
+ super(tcCtx);
+ }
- @Parameterized.Parameters(name = "SqlppExecutionTest {index}: {0}")
+ @Parameterized.Parameters(name = "AwsS3ExternalDatasetOnePartitionTest {index}: {0}")
public static Collection<Object[]> tests() throws Exception {
+ SUITE_TESTS = "testsuite_external_dataset_one_partition.xml";
+ ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
PREPARE_S3_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
- return LangExecutionUtil.tests("only_external_dataset.xml", SUITE_PATH);
- }
-
- public AwsS3ExternalDatasetOnePartitionTest(TestCaseContext tcCtx) {
- super(tcCtx);
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
private static void prepareS3Bucket() {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index b3fff7a..6ff59ee 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.HashMap;
@@ -73,28 +74,43 @@ public class AwsS3ExternalDatasetTest {
private static final Logger LOGGER = LogManager.getLogger();
- protected static String TEST_CONFIG_FILE_NAME;
+ // subclasses of this class MUST instantiate these variables before using them to avoid unexpected behavior
+ static String SUITE_TESTS;
+ static String ONLY_TESTS;
+ static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_S3_BUCKET;
- // S3 mock server
- private static S3Mock s3MockServer;
+ // Base directory paths for data files
+ private static final String JSON_DATA_PATH = joinPath("data", "json");
+ private static final String CSV_DATA_PATH = joinPath("data", "csv");
+ private static final String TSV_DATA_PATH = joinPath("data", "tsv");
// IMPORTANT: The following values must be used in the AWS S3 test case
+ private static S3Mock s3MockServer;
private static S3Client client;
+
+ // Service endpoint
+ private static final int S3_MOCK_SERVER_PORT = 8001;
+ private static final String S3_MOCK_SERVER_HOSTNAME = "http://localhost:" + S3_MOCK_SERVER_PORT;
+
+ // Region, bucket and definitions
+ private static final String S3_MOCK_SERVER_REGION = "us-west-2";
private static final String S3_MOCK_SERVER_BUCKET = "playground";
- private static final String S3_MOCK_SERVER_BUCKET_DEFINITION = "json-data/reviews/"; // data resides here
+ private static final String S3_MOCK_SERVER_BUCKET_JSON_DEFINITION = "json-data/reviews/"; // data resides here
private static final String S3_MOCK_SERVER_BUCKET_CSV_DEFINITION = "csv-data/reviews/"; // data resides here
private static final String S3_MOCK_SERVER_BUCKET_TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
- private static final String S3_MOCK_SERVER_REGION = "us-west-2";
- private static final int S3_MOCK_SERVER_PORT = 8001;
- private static final String S3_MOCK_SERVER_HOSTNAME = "http://localhost:" + S3_MOCK_SERVER_PORT;
- private static final String CSV_DATA_PATH = joinPath("data", "csv");
- private static final String TSV_DATA_PATH = joinPath("data", "tsv");
+
private static final Set<String> fileNames = new HashSet<>();
private static final CreateBucketRequest.Builder CREATE_BUCKET_BUILDER = CreateBucketRequest.builder();
private static final DeleteBucketRequest.Builder DELETE_BUCKET_BUILDER = DeleteBucketRequest.builder();
private static final PutObjectRequest.Builder PUT_OBJECT_BUILDER = PutObjectRequest.builder();
+ protected TestCaseContext tcCtx;
+
+ public AwsS3ExternalDatasetTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
@BeforeClass
public static void setUp() throws Exception {
final TestExecutor testExecutor = new AwsTestExecutor();
@@ -118,17 +134,13 @@ public class AwsS3ExternalDatasetTest {
LOGGER.info("S3 mock down and client shut down successfully");
}
- @Parameters(name = "SqlppExecutionTest {index}: {0}")
+ @Parameters(name = "AwsS3ExternalDatasetTest {index}: {0}")
public static Collection<Object[]> tests() throws Exception {
+ SUITE_TESTS = "testsuite_external_dataset.xml";
+ ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_S3_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
- return LangExecutionUtil.tests("only_external_dataset.xml", "testsuite_external_dataset.xml");
- }
-
- protected TestCaseContext tcCtx;
-
- public AwsS3ExternalDatasetTest(TestCaseContext tcCtx) {
- this.tcCtx = tcCtx;
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@Test
@@ -180,64 +192,58 @@ public class AwsS3ExternalDatasetTest {
client.createBucket(CreateBucketRequest.builder().bucket(S3_MOCK_SERVER_BUCKET).build());
LOGGER.info("bucket created successfully");
+ // Load JSON files
+ loadJsonFiles();
+ loadCsvFiles();
+ loadTsvFiles();
+
+ LOGGER.info("Files added successfully");
+ }
+
+ private static void loadJsonFiles() {
LOGGER.info("Adding JSON files to the bucket");
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "0.json").build(),
- RequestBody.fromString("{\"id\": 1, \"year\": null, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "1.json").build(),
- RequestBody.fromString("{\"id\": 2, \"year\": null, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/1.json").build(),
- RequestBody.fromString("{\"id\": 3, \"year\": 2018, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/2.json").build(),
- RequestBody.fromString("{\"id\": 4, \"year\": 2018, \"quarter\": null, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q1/1.json").build(),
- RequestBody.fromString("{\"id\": 5, \"year\": 2018, \"quarter\": 1, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q1/2.json").build(),
- RequestBody.fromString("{\"id\": 6, \"year\": 2018, \"quarter\": 1, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q2/1.json").build(),
- RequestBody.fromString("{\"id\": 7, \"year\": 2018, \"quarter\": 2, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2018/q2/2.json").build(),
- RequestBody.fromString("{\"id\": 8, \"year\": 2018, \"quarter\": 2, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/1.json").build(),
- RequestBody.fromString("{\"id\": 9, \"year\": 2019, \"quarter\": null, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/2.json").build(),
- RequestBody.fromString("{\"id\": 10, \"year\": 2019, \"quarter\": null, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q1/1.json").build(),
- RequestBody.fromString("{\"id\": 11, \"year\": 2019, \"quarter\": 1, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q1/2.json").build(),
- RequestBody.fromString("{\"id\": 12, \"year\": 2019, \"quarter\": 1, \"review\": \"bad\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q2/1.json").build(),
- RequestBody.fromString("{\"id\": 13, \"year\": 2019, \"quarter\": 2, \"review\": \"good\"}"));
- client.putObject(
- PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
- .key(S3_MOCK_SERVER_BUCKET_DEFINITION + "2019/q2/2.json").build(),
- RequestBody.fromString("{\"id\": 14, \"year\": 2019, \"quarter\": 2, \"review\": \"bad\"}"));
+ // Set the bucket
+ PutObjectRequest.Builder builder1 = PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET);
+
+ // load multi-level single line JSON files
+ String singleLineBasePath = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "single-line/";
+ Path filePath1 = Paths.get(JSON_DATA_PATH, "single-line", "20-records.json");
+ RequestBody reqBody1 = RequestBody.fromFile(filePath1);
+ client.putObject(builder1.key(singleLineBasePath + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1a/" + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1b/" + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1a/level2a/" + "20-records.json").build(), reqBody1);
+ client.putObject(builder1.key(singleLineBasePath + "level1a/level2b/" + "20-records.json").build(), reqBody1);
+
+ // Load multi-level multi-lines JSON files
+ String multiLinesBasePath = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "multi-lines/";
+ Path filePath2 = Paths.get(JSON_DATA_PATH, "multi-lines", "20-records.json");
+ RequestBody reqBody2 = RequestBody.fromFile(filePath2);
+ client.putObject(builder1.key(multiLinesBasePath + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1a/" + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1b/" + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1a/level2a/" + "20-records.json").build(), reqBody2);
+ client.putObject(builder1.key(multiLinesBasePath + "level1a/level2b/" + "20-records.json").build(), reqBody2);
+
+ // Load multi-level multi-lines with array JSON files
+ String multiLinesWithArraysBasePath = S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "multi-lines-with-arrays/";
+ Path filePath3 = Paths.get(JSON_DATA_PATH, "multi-lines-with-arrays", "5-records.json");
+ RequestBody reqBody3 = RequestBody.fromFile(filePath3);
+ client.putObject(builder1.key(multiLinesWithArraysBasePath + "5-records.json").build(), reqBody3);
+ client.putObject(builder1.key(multiLinesWithArraysBasePath + "level1a/" + "5-records.json").build(), reqBody3);
+
+ // Load multi-level multi-lines with nested objects JSON files
+ String multiLinesWithNestedObjectsBasePath =
+ S3_MOCK_SERVER_BUCKET_JSON_DEFINITION + "multi-lines-with-nested-objects/";
+ Path filePath4 = Paths.get(JSON_DATA_PATH, "multi-lines-with-nested-objects", "5-records.json");
+ RequestBody reqBody4 = RequestBody.fromFile(filePath4);
+ client.putObject(builder1.key(multiLinesWithNestedObjectsBasePath + "5-records.json").build(), reqBody4);
+ client.putObject(builder1.key(multiLinesWithNestedObjectsBasePath + "level1a/" + "5-records.json").build(),
+ reqBody4);
+ }
+
+ private static void loadCsvFiles() {
LOGGER.info("Adding CSV files to the bucket");
client.putObject(
PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
@@ -247,7 +253,9 @@ public class AwsS3ExternalDatasetTest {
PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
.key(S3_MOCK_SERVER_BUCKET_CSV_DEFINITION + "2018/01.csv").build(),
RequestBody.fromFile(Paths.get(CSV_DATA_PATH, "02.csv")));
+ }
+ private static void loadTsvFiles() {
LOGGER.info("Adding TSV files to the bucket");
client.putObject(
PutObjectRequest.builder().bucket(S3_MOCK_SERVER_BUCKET)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.002.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.003.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/csv/000/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.000.ddl.sqlpp
similarity index 50%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.000.ddl.sqlpp
index 9c6a994..8d084a1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.000.ddl.sqlpp
@@ -25,14 +25,46 @@ drop type test if exists;
create type test as open {
};
-drop dataset test if exists;
-create external dataset test(test) using S3 (
+drop dataset test1 if exists;
+create external dataset test1(test) using S3 (
("accessKey"="dummyAccessKey"),
("secretKey"="dummySecretKey"),
("region"="us-west-2"),
("serviceEndpoint"="http://localhost:8001"),
("container"="playground"),
-("definition"="json-data/reviews"),
+("definition"="json-data/reviews/single-line"),
("format"="json")
);
+drop dataset test2 if exists;
+create external dataset test2(test) using S3 (
+("accessKey"="dummyAccessKey"),
+("secretKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines"),
+("format"="json")
+);
+
+drop dataset test3 if exists;
+create external dataset test3(test) using S3 (
+("accessKey"="dummyAccessKey"),
+("secretKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-arrays"),
+("format"="json")
+);
+
+drop dataset test4 if exists;
+create external dataset test4(test) using S3 (
+("accessKey"="dummyAccessKey"),
+("secretKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/multi-lines-with-nested-objects"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.001.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.001.query.sqlpp
index 2dd9cc5..b72e741 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.001.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.002.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.002.query.sqlpp
index 2dd9cc5..c3e883e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.002.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.003.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.003.query.sqlpp
index 2dd9cc5..e3d64ff 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.003.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.004.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.004.query.sqlpp
index 2dd9cc5..1fda76a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.004.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select value test3 from test3 order by id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.005.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.005.query.sqlpp
index 2dd9cc5..2b9b5cb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.005.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select count(*) `count` from test4;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.006.query.sqlpp
similarity index 94%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.006.query.sqlpp
index 2dd9cc5..2cd184d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.001.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.006.query.sqlpp
@@ -19,7 +19,5 @@
use test;
-from test
-select value test
-order by id asc;
+select value test4 from test4 order by id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.007.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/000/external_dataset.002.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/json/000/external_dataset.007.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/002/query-dataset.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.002.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.002.query.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/001/query-dataset.003.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/tsv/000/query-dataset.003.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/000/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/000/external_dataset.001.adm
deleted file mode 100644
index a7ce908..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/000/external_dataset.001.adm
+++ /dev/null
@@ -1,14 +0,0 @@
-{ "id": 1, "year": null, "quarter": null, "review": "good" }
-{ "id": 2, "year": null, "quarter": null, "review": "good" }
-{ "id": 3, "year": 2018, "quarter": null, "review": "good" }
-{ "id": 4, "year": 2018, "quarter": null, "review": "bad" }
-{ "id": 5, "year": 2018, "quarter": 1, "review": "good" }
-{ "id": 6, "year": 2018, "quarter": 1, "review": "bad" }
-{ "id": 7, "year": 2018, "quarter": 2, "review": "good" }
-{ "id": 8, "year": 2018, "quarter": 2, "review": "bad" }
-{ "id": 9, "year": 2019, "quarter": null, "review": "good" }
-{ "id": 10, "year": 2019, "quarter": null, "review": "bad" }
-{ "id": 11, "year": 2019, "quarter": 1, "review": "good" }
-{ "id": 12, "year": 2019, "quarter": 1, "review": "bad" }
-{ "id": 13, "year": 2019, "quarter": 2, "review": "good" }
-{ "id": 14, "year": 2019, "quarter": 2, "review": "bad" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/001/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/000/external_dataset.001.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/001/external_dataset.001.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/csv/000/external_dataset.001.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.002.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.002.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.003.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.003.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.004.adm
new file mode 100644
index 0000000..d10a16c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.004.adm
@@ -0,0 +1 @@
+{ "count": 10 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.005.adm
new file mode 100644
index 0000000..a1881c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.005.adm
@@ -0,0 +1,10 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.006.adm
new file mode 100644
index 0000000..d10a16c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.006.adm
@@ -0,0 +1 @@
+{ "count": 10 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.007.adm
new file mode 100644
index 0000000..d24a08b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/json/000/external_dataset.007.adm
@@ -0,0 +1,10 @@
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 1, "year": null, "quarter": null, "review": "good", "array": [ 1, 2, 3 ], "nested": { "id": 1 } }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 2, "year": null, "quarter": null, "review": "good", "array": [ 1, [ 1, 2 ], [ 1 ] ], "nested": { "id": 1 }, "nested2": [ { "id": 1 } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 3, "year": 2018, "quarter": null, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ { "nested": { "array": [ 1, 2 ] } } ] } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 4, "year": 2018, "quarter": null, "review": "bad", "array": [ 1, 2, 3, { "nested1": { "id": 1, "nested2": { "id": 2, "nested3": [ { "nested4": null } ] } } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
+{ "id": 5, "year": 2018, "quarter": 1, "review": "good", "array": [ 1, 2, 3, { "nested": { "array": [ 1, 2 ] } } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/002/external_dataset.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/000/external_dataset.001.adm
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/002/external_dataset.001.adm
rename to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/aws/s3/tsv/000/external_dataset.001.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 2456f13..a6774fa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -20,18 +20,18 @@
<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
<test-group name="external-dataset">
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/000">
- <output-dir compare="Text">aws/s3/000</output-dir>
+ <compilation-unit name="aws/s3/json/000">
+ <output-dir compare="Text">aws/s3/json/000</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/001">
- <output-dir compare="Text">aws/s3/001</output-dir>
+ <compilation-unit name="aws/s3/csv/000">
+ <output-dir compare="Text">aws/s3/csv/000</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">
- <compilation-unit name="aws/s3/002">
- <output-dir compare="Text">aws/s3/002</output-dir>
+ <compilation-unit name="aws/s3/tsv/000">
+ <output-dir compare="Text">aws/s3/tsv/000</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset">