You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/05/12 17:00:52 UTC
Change in asterixdb[cheshire-cat]: Reuse/share code for external dataset tests (p1)
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444 )
Change subject: Reuse/share code for external dataset tests (p1)
......................................................................
Reuse/share code for external dataset tests (p1)
Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java
2 files changed, 439 insertions(+), 334 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/44/11444/1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 387f7f6..50329bd 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -18,15 +18,13 @@
*/
package org.apache.asterix.test.external_dataset.aws;
+import static org.apache.asterix.test.external_dataset.aws.ExternalDatasetTestUtils.setDataPaths;
import static org.apache.hyracks.util.file.FileUtil.joinPath;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.BitSet;
import java.util.Collection;
@@ -35,7 +33,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.zip.GZIPOutputStream;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.test.common.TestExecutor;
@@ -50,7 +47,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@@ -79,6 +75,12 @@
private static final Logger LOGGER = LogManager.getLogger();
+ // Base directory paths for data files
+ public static final String JSON_DATA_PATH = joinPath("data", "json");
+ public static final String CSV_DATA_PATH = joinPath("data", "csv");
+ public static final String TSV_DATA_PATH = joinPath("data", "tsv");
+ public static final String MIXED_DATA_PATH = joinPath("data", "mixed");
+
// subclasses of this class MUST instantiate these variables before using them to avoid unexpected behavior
static String SUITE_TESTS;
static String ONLY_TESTS;
@@ -87,40 +89,17 @@
static Runnable PREPARE_FIXED_DATA_BUCKET;
static Runnable PREPARE_MIXED_DATA_BUCKET;
- // Base directory paths for data files
- private static final String JSON_DATA_PATH = joinPath("data", "json");
- private static final String CSV_DATA_PATH = joinPath("data", "csv");
- private static final String TSV_DATA_PATH = joinPath("data", "tsv");
- private static final String MIXED_DATA_PATH = joinPath("data", "mixed");
-
- // Service endpoint
- private static final int MOCK_SERVER_PORT = 8001;
- private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
-
- // Region, bucket and definitions
- private static final String MOCK_SERVER_REGION = "us-west-2";
- private static final String MOCK_SERVER_BUCKET = "playground";
- private static final String FIXED_DATA_BUCKET = "fixed-data"; // Do not use, has fixed data
- private static final String INCLUDE_EXCLUDE_BUCKET = "include-exclude"; // include & exclude bucket
- private static final String JSON_DEFINITION = "json-data/reviews/"; // data resides here
- private static final String CSV_DEFINITION = "csv-data/reviews/"; // data resides here
- private static final String TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
-
- // This is used for a test to generate over 1000 number of files
- private static final String OVER_1000_OBJECTS_PATH = "over-1000-objects";
- private static final int OVER_1000_OBJECTS_COUNT = 2999;
-
private static final Set<String> fileNames = new HashSet<>();
private static final CreateBucketRequest.Builder CREATE_BUCKET_BUILDER = CreateBucketRequest.builder();
private static final DeleteBucketRequest.Builder DELETE_BUCKET_BUILDER = DeleteBucketRequest.builder();
private static final PutObjectRequest.Builder PUT_OBJECT_BUILDER = PutObjectRequest.builder();
// IMPORTANT: The following values must be used in the AWS S3 test case
+ private static final String MOCK_SERVER_REGION = "us-west-2";
+ private static final int MOCK_SERVER_PORT = 8001;
+ private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
private static S3Mock s3MockServer;
private static S3Client client;
- private static final PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(MOCK_SERVER_BUCKET);
- private static final PutObjectRequest.Builder includeExcludeBuilder =
- PutObjectRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET);
protected TestCaseContext tcCtx;
@@ -156,9 +135,9 @@
SUITE_TESTS = "testsuite_external_dataset_s3.xml";
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
- PREPARE_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
- PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareFixedDataBucket;
- PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareMixedDataBucket;
+ PREPARE_BUCKET = () -> ExternalDatasetTestUtils.prepareS3Bucket(client);
+ PREPARE_FIXED_DATA_BUCKET = () -> ExternalDatasetTestUtils.prepareFixedDataBucket(client);
+ PREPARE_MIXED_DATA_BUCKET = () -> ExternalDatasetTestUtils.prepareMixedDataBucket(client);
return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@@ -200,311 +179,12 @@
LOGGER.info("Client created successfully");
// Create the bucket and upload some json files
+ setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH, MIXED_DATA_PATH);
PREPARE_BUCKET.run();
PREPARE_FIXED_DATA_BUCKET.run();
PREPARE_MIXED_DATA_BUCKET.run();
}
- /**
- * Creates a bucket and fills it with some files for testing purpose.
- */
- private static void prepareS3Bucket() {
- LOGGER.info("creating bucket " + MOCK_SERVER_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(MOCK_SERVER_BUCKET).build());
- LOGGER.info("bucket created successfully");
-
- LOGGER.info("Adding JSON files to the bucket");
- loadJsonFiles();
- LOGGER.info("JSON Files added successfully");
-
- LOGGER.info("Adding CSV files to the bucket");
- loadCsvFiles();
- LOGGER.info("CSV Files added successfully");
-
- LOGGER.info("Adding TSV files to the bucket");
- loadTsvFiles();
- LOGGER.info("TSV Files added successfully");
-
- LOGGER.info("Loading " + OVER_1000_OBJECTS_COUNT + " into " + OVER_1000_OBJECTS_PATH);
- loadLargeNumberOfFiles();
- LOGGER.info("Added " + OVER_1000_OBJECTS_COUNT + " files into " + OVER_1000_OBJECTS_PATH + " successfully");
- }
-
- /**
- * This bucket is being filled by fixed data, a test is counting all records in this bucket. If this bucket is
- * changed, the test case will fail and its result will need to be updated each time
- */
- private static void prepareFixedDataBucket() {
- LOGGER.info("creating bucket " + FIXED_DATA_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_BUCKET).build());
- LOGGER.info("bucket " + FIXED_DATA_BUCKET + " created successfully");
-
- LOGGER.info("Loading fixed data to " + FIXED_DATA_BUCKET);
-
- // Files data
- RequestBody requestBody = RequestBody.fromFile(Paths.get(JSON_DATA_PATH, "single-line", "20-records.json"));
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("1.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("2.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/3.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/4.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/lvl2/5.json").build(), requestBody);
- }
-
- private static void loadJsonFiles() {
- String dataBasePath = JSON_DATA_PATH;
- String definition = JSON_DEFINITION;
-
- // Normal format
- String definitionSegment = "json";
- loadData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
-
- definitionSegment = "json-array-of-objects";
- loadData(dataBasePath, "single-line", "array_of_objects.json", "json-data/", definitionSegment, false, false);
-
- // gz compressed format
- definitionSegment = "gz";
- loadGzData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
-
- // Mixed normal and gz compressed format
- definitionSegment = "mixed";
- loadData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
- loadGzData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
- }
-
- private static void loadCsvFiles() {
- String dataBasePath = CSV_DATA_PATH;
- String definition = CSV_DEFINITION;
-
- // Normal format
- String definitionSegment = "csv";
- loadData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
-
- // gz compressed format
- definitionSegment = "gz";
- loadGzData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
-
- // Mixed normal and gz compressed format
- definitionSegment = "mixed";
- loadData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
- }
-
- private static void loadTsvFiles() {
- String dataBasePath = TSV_DATA_PATH;
- String definition = TSV_DEFINITION;
-
- // Normal format
- String definitionSegment = "tsv";
- loadData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
-
- // gz compressed format
- definitionSegment = "gz";
- loadGzData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
-
- // Mixed normal and gz compressed format
- definitionSegment = "mixed";
- loadData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
- }
-
- private static void loadData(String fileBasePath, String filePathSegment, String filename, String definition,
- String definitionSegment, boolean removeExtension) {
- loadData(fileBasePath, filePathSegment, filename, definition, definitionSegment, removeExtension, true);
- }
-
- private static void loadData(String fileBasePath, String filePathSegment, String filename, String definition,
- String definitionSegment, boolean removeExtension, boolean copyToSubLevels) {
- // Files data
- Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
- RequestBody requestBody = RequestBody.fromFile(filePath);
-
- // Keep or remove the file extension
- Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
- String finalFileName;
- if (removeExtension) {
- finalFileName = FilenameUtils.removeExtension(filename);
- } else {
- finalFileName = filename;
- }
-
- // Files base definition
- filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
- definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
- String basePath = definition + filePathSegment + definitionSegment;
-
- // Load the data
- client.putObject(builder.key(basePath + finalFileName).build(), requestBody);
- if (copyToSubLevels) {
- client.putObject(builder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
- }
- }
-
- private static void loadGzData(String fileBasePath, String filePathSegment, String filename, String definition,
- String definitionSegment, boolean removeExtension) {
- try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
-
- // Files data
- Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
-
- // Get the compressed data
- gzipOutputStream.write(Files.readAllBytes(filePath));
- gzipOutputStream.close(); // Need to close or data will be invalid
- byte[] gzipBytes = byteArrayOutputStream.toByteArray();
- RequestBody requestBody = RequestBody.fromBytes(gzipBytes);
-
- // Keep or remove the file extension
- Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
- String finalFileName;
- if (removeExtension) {
- finalFileName = FilenameUtils.removeExtension(filename);
- } else {
- finalFileName = filename;
- }
- finalFileName += ".gz";
-
- // Files base definition
- filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
- definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
- String basePath = definition + filePathSegment + definitionSegment;
-
- // Load the data
- client.putObject(builder.key(basePath + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
- } catch (Exception ex) {
- LOGGER.error(ex.getMessage());
- }
- }
-
- /**
- * Generates over 1000 objects and upload them to S3 mock server, 1 record per object
- */
- private static void loadLargeNumberOfFiles() {
- for (int i = 0; i < OVER_1000_OBJECTS_COUNT; i++) {
- RequestBody body = RequestBody.fromString("{\"id\":" + i + "}");
- client.putObject(builder.key(OVER_1000_OBJECTS_PATH + "/" + i + ".json").build(), body);
- }
- }
-
- /**
- * Loads a combination of different file formats in the same path
- */
- private static void prepareMixedDataBucket() {
- LOGGER.info("creating bucket " + INCLUDE_EXCLUDE_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET).build());
- LOGGER.info("bucket " + INCLUDE_EXCLUDE_BUCKET + " created successfully");
-
- // JSON
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2018.json").build(),
- RequestBody.fromString("{\"id\":" + 1 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2019.json").build(),
- RequestBody.fromString("{\"id\":" + 2 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2020.json").build(),
- RequestBody.fromString("{\"id\":" + 3 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2018.json").build(),
- RequestBody.fromString("{\"id\":" + 4 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2019.json").build(),
- RequestBody.fromString("{\"id\":" + 5 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2020.json").build(),
- RequestBody.fromString("{\"id\":" + 6 + "}"));
-
- // CSV
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2018.csv").build(),
- RequestBody.fromString("7,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2019.csv").build(),
- RequestBody.fromString("8,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2020.csv").build(),
- RequestBody.fromString("{9,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2018.csv").build(),
- RequestBody.fromString("10,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2019.csv").build(),
- RequestBody.fromString("11,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2020.csv").build(),
- RequestBody.fromString("12,\"good\""));
-
- // TSV
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2018.tsv").build(),
- RequestBody.fromString("13\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2019.tsv").build(),
- RequestBody.fromString("14\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2020.tsv").build(),
- RequestBody.fromString("15\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2018.tsv").build(),
- RequestBody.fromString("16\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2019.tsv").build(),
- RequestBody.fromString("17\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2020.tsv").build(),
- RequestBody.fromString("18\t\"good\""));
-
- // JSON no extension
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2018").build(),
- RequestBody.fromString("{\"id\":" + 1 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2019").build(),
- RequestBody.fromString("{\"id\":" + 2 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2020").build(),
- RequestBody.fromString("{\"id\":" + 3 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2018").build(),
- RequestBody.fromString("{\"id\":" + 4 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2019").build(),
- RequestBody.fromString("{\"id\":" + 5 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2020").build(),
- RequestBody.fromString("{\"id\":" + 6 + "}"));
- }
-
static class AwsTestExecutor extends TestExecutor {
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java
new file mode 100644
index 0000000..08e1fe7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.external_dataset.aws;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+@RunWith(Parameterized.class)
+public class ExternalDatasetTestUtils {
+
+ protected static final Logger LOGGER = LogManager.getLogger();
+
+ // Base directory paths for data files
+ private static String JSON_DATA_PATH;
+ private static String CSV_DATA_PATH;
+ private static String TSV_DATA_PATH;
+ private static String MIXED_DATA_PATH;
+
+ // IMPORTANT: The following values must be used in the AWS S3 test case
+ // Region, container and definitions
+ public static final String PLAYGROUND_CONTAINER = "playground";
+ public static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data
+ public static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
+ public static final String JSON_DEFINITION = "json-data/reviews/";
+ public static final String CSV_DEFINITION = "csv-data/reviews/";
+ public static final String TSV_DEFINITION = "tsv-data/reviews/";
+
+ // This is used for a test to generate over 1000 number of files
+ public static final String OVER_1000_OBJECTS_PATH = "over-1000-objects";
+ public static final int OVER_1000_OBJECTS_COUNT = 2999;
+
+ public static final PutObjectRequest.Builder playgroundBuilder =
+ PutObjectRequest.builder().bucket(PLAYGROUND_CONTAINER);
+ public static final PutObjectRequest.Builder fixedDataBuilder =
+ PutObjectRequest.builder().bucket(FIXED_DATA_CONTAINER);
+ public static final PutObjectRequest.Builder includeExcludeBuilder =
+ PutObjectRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER);
+
+ protected TestCaseContext tcCtx;
+
+ public ExternalDatasetTestUtils(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ public static void setDataPaths(String jsonDataPath, String csvDataPath, String tsvDataPath, String mixedDataPath) {
+ JSON_DATA_PATH = jsonDataPath;
+ CSV_DATA_PATH = csvDataPath;
+ TSV_DATA_PATH = tsvDataPath;
+ MIXED_DATA_PATH = mixedDataPath;
+ }
+
+ /**
+ * Creates a bucket and fills it with some files for testing purpose.
+ */
+ public static void prepareS3Bucket(S3Client client) {
+ LOGGER.info("creating bucket " + PLAYGROUND_CONTAINER);
+ client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
+ LOGGER.info("bucket created successfully");
+
+ LOGGER.info("Adding JSON files to the bucket");
+ loadJsonFiles(client);
+ LOGGER.info("JSON Files added successfully");
+
+ LOGGER.info("Adding CSV files to the bucket");
+ loadCsvFiles(client);
+ LOGGER.info("CSV Files added successfully");
+
+ LOGGER.info("Adding TSV files to the bucket");
+ loadTsvFiles(client);
+ LOGGER.info("TSV Files added successfully");
+
+ LOGGER.info("Adding a big JSON file");
+ loadBigJson(client);
+ LOGGER.info("JSON file added successfully");
+
+ LOGGER.info("Loading " + OVER_1000_OBJECTS_COUNT + " into " + OVER_1000_OBJECTS_PATH);
+ loadLargeNumberOfFiles(client);
+ LOGGER.info("Added " + OVER_1000_OBJECTS_COUNT + " files into " + OVER_1000_OBJECTS_PATH + " successfully");
+
+ LOGGER.info("Files added successfully");
+ }
+
+ /**
+ * This bucket is being filled by fixed data, a test is counting all records in this bucket. If this bucket is
+ * changed, the test case will fail and its result will need to be updated each time
+ */
+ public static void prepareFixedDataBucket(S3Client client) {
+ LOGGER.info("creating bucket " + FIXED_DATA_CONTAINER);
+ client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_CONTAINER).build());
+ LOGGER.info("bucket " + FIXED_DATA_CONTAINER + " created successfully");
+
+ LOGGER.info("Loading fixed data to " + FIXED_DATA_CONTAINER);
+
+ // Files data
+ RequestBody requestBody = RequestBody.fromFile(Paths.get(JSON_DATA_PATH, "single-line", "20-records.json"));
+ client.putObject(fixedDataBuilder.key("1.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("2.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("lvl1/3.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("lvl1/4.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("lvl1/lvl2/5.json").build(), requestBody);
+ }
+
+ public static void loadJsonFiles(S3Client client) {
+ String dataBasePath = JSON_DATA_PATH;
+ String definition = JSON_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "json";
+ loadData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+
+ definitionSegment = "json-array-of-objects";
+ loadData(client, dataBasePath, "single-line", "array_of_objects.json", "json-data/", definitionSegment, false,
+ false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadGzData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+ loadGzData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadGzData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+ }
+
+ private static void loadCsvFiles(S3Client client) {
+ String dataBasePath = CSV_DATA_PATH;
+ String definition = CSV_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "csv";
+ loadData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+ }
+
+ private static void loadTsvFiles(S3Client client) {
+ String dataBasePath = TSV_DATA_PATH;
+ String definition = TSV_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "tsv";
+ loadData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+ }
+
+ private static void loadData(S3Client client, String fileBasePath, String filePathSegment, String filename,
+ String definition, String definitionSegment, boolean removeExtension) {
+ loadData(client, fileBasePath, filePathSegment, filename, definition, definitionSegment, removeExtension, true);
+ }
+
+ private static void loadData(S3Client client, String fileBasePath, String filePathSegment, String filename,
+ String definition, String definitionSegment, boolean removeExtension, boolean copyToSubLevels) {
+ // Files data
+ Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
+ RequestBody requestBody = RequestBody.fromFile(filePath);
+
+ // Keep or remove the file extension
+ Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
+ String finalFileName;
+ if (removeExtension) {
+ finalFileName = FilenameUtils.removeExtension(filename);
+ } else {
+ finalFileName = filename;
+ }
+
+ // Files base definition
+ filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
+ definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
+ String basePath = definition + filePathSegment + definitionSegment;
+
+ // Load the data
+ client.putObject(playgroundBuilder.key(basePath + finalFileName).build(), requestBody);
+ if (copyToSubLevels) {
+ client.putObject(playgroundBuilder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
+ }
+ }
+
+ private static void loadGzData(S3Client client, String fileBasePath, String filePathSegment, String filename,
+ String definition, String definitionSegment, boolean removeExtension) {
+ try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
+
+ // Files data
+ Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
+
+ // Get the compressed data
+ gzipOutputStream.write(Files.readAllBytes(filePath));
+ gzipOutputStream.close(); // Need to close or data will be invalid
+ byte[] gzipBytes = byteArrayOutputStream.toByteArray();
+ RequestBody requestBody = RequestBody.fromBytes(gzipBytes);
+
+ // Keep or remove the file extension
+ Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
+ String finalFileName;
+ if (removeExtension) {
+ finalFileName = FilenameUtils.removeExtension(filename);
+ } else {
+ finalFileName = filename;
+ }
+ finalFileName += ".gz";
+
+ // Files base definition
+ filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
+ definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
+ String basePath = definition + filePathSegment + definitionSegment;
+
+ // Load the data
+ client.putObject(playgroundBuilder.key(basePath + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
+ } catch (Exception ex) {
+ LOGGER.error(ex.getMessage());
+ }
+ }
+
+ private static void loadBigJson(S3Client client) {
+ String fileName = "big_record.json";
+ int bufferSize = 4 * 1024 * 1024;
+ int maxSize = bufferSize * 9;
+ Path filePath = Paths.get("target", "rttest", "tmp", fileName);
+ try {
+ if (Files.notExists(filePath)) {
+ Files.createDirectories(filePath.getParent());
+ Files.createFile(filePath);
+ }
+ } catch (IOException ex) {
+ throw new IllegalStateException("File " + fileName + " not found");
+ }
+
+ try (FileWriter writer = new FileWriter(filePath.toFile(), false);
+ BufferedWriter bw = new BufferedWriter(writer, bufferSize)) {
+ bw.append("{ \"large_field\": \"");
+ for (int i = 0; i < maxSize; i++) {
+ bw.append('A');
+ }
+ bw.append("\" }");
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ String key = "big-json/" + fileName;
+ client.putObject(playgroundBuilder.key(key).build(), RequestBody.fromFile(filePath));
+ }
+
+ /**
+ * Generates over 1000 objects and upload them to S3 mock server, 1 record per object
+ */
+ private static void loadLargeNumberOfFiles(S3Client client) {
+ for (int i = 0; i < OVER_1000_OBJECTS_COUNT; i++) {
+ RequestBody body = RequestBody.fromString("{\"id\":" + i + "}");
+ client.putObject(playgroundBuilder.key(OVER_1000_OBJECTS_PATH + "/" + i + ".json").build(), body);
+ }
+ }
+
+ /**
+ * Loads a combination of different file formats in the same path
+ */
+ public static void prepareMixedDataBucket(S3Client client) {
+ LOGGER.info("creating bucket " + INCLUDE_EXCLUDE_CONTAINER);
+ client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER).build());
+ LOGGER.info("bucket " + INCLUDE_EXCLUDE_CONTAINER + " created successfully");
+
+ // JSON
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+
+ // CSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2018.csv").build(),
+ RequestBody.fromString("7,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2019.csv").build(),
+ RequestBody.fromString("8,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2020.csv").build(),
+ RequestBody.fromString("9,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2018.csv").build(),
+ RequestBody.fromString("10,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2019.csv").build(),
+ RequestBody.fromString("11,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2020.csv").build(),
+ RequestBody.fromString("12,\"good\""));
+
+ // TSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2018.tsv").build(),
+ RequestBody.fromString("13\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2019.tsv").build(),
+ RequestBody.fromString("14\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2020.tsv").build(),
+ RequestBody.fromString("15\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2018.tsv").build(),
+ RequestBody.fromString("16\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2019.tsv").build(),
+ RequestBody.fromString("17\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2020.tsv").build(),
+ RequestBody.fromString("18\t\"good\""));
+
+ // JSON no extension
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
Gerrit-Change-Number: 11444
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange
Change in asterixdb[cheshire-cat]: Reuse/share code for external dataset tests (p1)
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has abandoned this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444 )
Change subject: Reuse/share code for external dataset tests (p1)
......................................................................
Abandoned
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
Gerrit-Change-Number: 11444
Gerrit-PatchSet: 3
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-MessageType: abandon
Change in asterixdb[cheshire-cat]: Reuse/share code for external dataset tests (p1)
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444 )
Change subject: Reuse/share code for external dataset tests (p1)
......................................................................
Reuse/share code for external dataset tests (p1)
Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java
2 files changed, 439 insertions(+), 334 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/44/11444/1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 387f7f6..50329bd 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -18,15 +18,13 @@
*/
package org.apache.asterix.test.external_dataset.aws;
+import static org.apache.asterix.test.external_dataset.aws.ExternalDatasetTestUtils.setDataPaths;
import static org.apache.hyracks.util.file.FileUtil.joinPath;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.BitSet;
import java.util.Collection;
@@ -35,7 +33,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.zip.GZIPOutputStream;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.test.common.TestExecutor;
@@ -50,7 +47,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@@ -79,6 +75,12 @@
private static final Logger LOGGER = LogManager.getLogger();
+ // Base directory paths for data files
+ public static final String JSON_DATA_PATH = joinPath("data", "json");
+ public static final String CSV_DATA_PATH = joinPath("data", "csv");
+ public static final String TSV_DATA_PATH = joinPath("data", "tsv");
+ public static final String MIXED_DATA_PATH = joinPath("data", "mixed");
+
// subclasses of this class MUST instantiate these variables before using them to avoid unexpected behavior
static String SUITE_TESTS;
static String ONLY_TESTS;
@@ -87,40 +89,17 @@
static Runnable PREPARE_FIXED_DATA_BUCKET;
static Runnable PREPARE_MIXED_DATA_BUCKET;
- // Base directory paths for data files
- private static final String JSON_DATA_PATH = joinPath("data", "json");
- private static final String CSV_DATA_PATH = joinPath("data", "csv");
- private static final String TSV_DATA_PATH = joinPath("data", "tsv");
- private static final String MIXED_DATA_PATH = joinPath("data", "mixed");
-
- // Service endpoint
- private static final int MOCK_SERVER_PORT = 8001;
- private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
-
- // Region, bucket and definitions
- private static final String MOCK_SERVER_REGION = "us-west-2";
- private static final String MOCK_SERVER_BUCKET = "playground";
- private static final String FIXED_DATA_BUCKET = "fixed-data"; // Do not use, has fixed data
- private static final String INCLUDE_EXCLUDE_BUCKET = "include-exclude"; // include & exclude bucket
- private static final String JSON_DEFINITION = "json-data/reviews/"; // data resides here
- private static final String CSV_DEFINITION = "csv-data/reviews/"; // data resides here
- private static final String TSV_DEFINITION = "tsv-data/reviews/"; // data resides here
-
- // This is used for a test to generate over 1000 number of files
- private static final String OVER_1000_OBJECTS_PATH = "over-1000-objects";
- private static final int OVER_1000_OBJECTS_COUNT = 2999;
-
private static final Set<String> fileNames = new HashSet<>();
private static final CreateBucketRequest.Builder CREATE_BUCKET_BUILDER = CreateBucketRequest.builder();
private static final DeleteBucketRequest.Builder DELETE_BUCKET_BUILDER = DeleteBucketRequest.builder();
private static final PutObjectRequest.Builder PUT_OBJECT_BUILDER = PutObjectRequest.builder();
// IMPORTANT: The following values must be used in the AWS S3 test case
+ private static final String MOCK_SERVER_REGION = "us-west-2";
+ private static final int MOCK_SERVER_PORT = 8001;
+ private static final String MOCK_SERVER_HOSTNAME = "http://localhost:" + MOCK_SERVER_PORT;
private static S3Mock s3MockServer;
private static S3Client client;
- private static final PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(MOCK_SERVER_BUCKET);
- private static final PutObjectRequest.Builder includeExcludeBuilder =
- PutObjectRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET);
protected TestCaseContext tcCtx;
@@ -156,9 +135,9 @@
SUITE_TESTS = "testsuite_external_dataset_s3.xml";
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
- PREPARE_BUCKET = AwsS3ExternalDatasetTest::prepareS3Bucket;
- PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareFixedDataBucket;
- PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetTest::prepareMixedDataBucket;
+ PREPARE_BUCKET = () -> ExternalDatasetTestUtils.prepareS3Bucket(client);
+ PREPARE_FIXED_DATA_BUCKET = () -> ExternalDatasetTestUtils.prepareFixedDataBucket(client);
+ PREPARE_MIXED_DATA_BUCKET = () -> ExternalDatasetTestUtils.prepareMixedDataBucket(client);
return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
}
@@ -200,311 +179,12 @@
LOGGER.info("Client created successfully");
// Create the bucket and upload some json files
+ setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH, MIXED_DATA_PATH);
PREPARE_BUCKET.run();
PREPARE_FIXED_DATA_BUCKET.run();
PREPARE_MIXED_DATA_BUCKET.run();
}
- /**
- * Creates a bucket and fills it with some files for testing purpose.
- */
- private static void prepareS3Bucket() {
- LOGGER.info("creating bucket " + MOCK_SERVER_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(MOCK_SERVER_BUCKET).build());
- LOGGER.info("bucket created successfully");
-
- LOGGER.info("Adding JSON files to the bucket");
- loadJsonFiles();
- LOGGER.info("JSON Files added successfully");
-
- LOGGER.info("Adding CSV files to the bucket");
- loadCsvFiles();
- LOGGER.info("CSV Files added successfully");
-
- LOGGER.info("Adding TSV files to the bucket");
- loadTsvFiles();
- LOGGER.info("TSV Files added successfully");
-
- LOGGER.info("Loading " + OVER_1000_OBJECTS_COUNT + " into " + OVER_1000_OBJECTS_PATH);
- loadLargeNumberOfFiles();
- LOGGER.info("Added " + OVER_1000_OBJECTS_COUNT + " files into " + OVER_1000_OBJECTS_PATH + " successfully");
- }
-
- /**
- * This bucket is being filled by fixed data, a test is counting all records in this bucket. If this bucket is
- * changed, the test case will fail and its result will need to be updated each time
- */
- private static void prepareFixedDataBucket() {
- LOGGER.info("creating bucket " + FIXED_DATA_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_BUCKET).build());
- LOGGER.info("bucket " + FIXED_DATA_BUCKET + " created successfully");
-
- LOGGER.info("Loading fixed data to " + FIXED_DATA_BUCKET);
-
- // Files data
- RequestBody requestBody = RequestBody.fromFile(Paths.get(JSON_DATA_PATH, "single-line", "20-records.json"));
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("1.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("2.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/3.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/4.json").build(), requestBody);
- client.putObject(builder.bucket(FIXED_DATA_BUCKET).key("lvl1/lvl2/5.json").build(), requestBody);
- }
-
- private static void loadJsonFiles() {
- String dataBasePath = JSON_DATA_PATH;
- String definition = JSON_DEFINITION;
-
- // Normal format
- String definitionSegment = "json";
- loadData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
-
- definitionSegment = "json-array-of-objects";
- loadData(dataBasePath, "single-line", "array_of_objects.json", "json-data/", definitionSegment, false, false);
-
- // gz compressed format
- definitionSegment = "gz";
- loadGzData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
-
- // Mixed normal and gz compressed format
- definitionSegment = "mixed";
- loadData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
- loadGzData(dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
- loadGzData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
- false);
- }
-
- private static void loadCsvFiles() {
- String dataBasePath = CSV_DATA_PATH;
- String definition = CSV_DEFINITION;
-
- // Normal format
- String definitionSegment = "csv";
- loadData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
-
- // gz compressed format
- definitionSegment = "gz";
- loadGzData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
-
- // Mixed normal and gz compressed format
- definitionSegment = "mixed";
- loadData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "01.csv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.csv", definition, definitionSegment, false);
- }
-
- private static void loadTsvFiles() {
- String dataBasePath = TSV_DATA_PATH;
- String definition = TSV_DEFINITION;
-
- // Normal format
- String definitionSegment = "tsv";
- loadData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
-
- // gz compressed format
- definitionSegment = "gz";
- loadGzData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
-
- // Mixed normal and gz compressed format
- definitionSegment = "mixed";
- loadData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "01.tsv", definition, definitionSegment, false);
- loadGzData(dataBasePath, "", "02.tsv", definition, definitionSegment, false);
- }
-
- private static void loadData(String fileBasePath, String filePathSegment, String filename, String definition,
- String definitionSegment, boolean removeExtension) {
- loadData(fileBasePath, filePathSegment, filename, definition, definitionSegment, removeExtension, true);
- }
-
- private static void loadData(String fileBasePath, String filePathSegment, String filename, String definition,
- String definitionSegment, boolean removeExtension, boolean copyToSubLevels) {
- // Files data
- Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
- RequestBody requestBody = RequestBody.fromFile(filePath);
-
- // Keep or remove the file extension
- Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
- String finalFileName;
- if (removeExtension) {
- finalFileName = FilenameUtils.removeExtension(filename);
- } else {
- finalFileName = filename;
- }
-
- // Files base definition
- filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
- definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
- String basePath = definition + filePathSegment + definitionSegment;
-
- // Load the data
- client.putObject(builder.key(basePath + finalFileName).build(), requestBody);
- if (copyToSubLevels) {
- client.putObject(builder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
- }
- }
-
- private static void loadGzData(String fileBasePath, String filePathSegment, String filename, String definition,
- String definitionSegment, boolean removeExtension) {
- try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
-
- // Files data
- Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
-
- // Get the compressed data
- gzipOutputStream.write(Files.readAllBytes(filePath));
- gzipOutputStream.close(); // Need to close or data will be invalid
- byte[] gzipBytes = byteArrayOutputStream.toByteArray();
- RequestBody requestBody = RequestBody.fromBytes(gzipBytes);
-
- // Keep or remove the file extension
- Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
- String finalFileName;
- if (removeExtension) {
- finalFileName = FilenameUtils.removeExtension(filename);
- } else {
- finalFileName = filename;
- }
- finalFileName += ".gz";
-
- // Files base definition
- filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
- definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
- String basePath = definition + filePathSegment + definitionSegment;
-
- // Load the data
- client.putObject(builder.key(basePath + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
- client.putObject(builder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
- } catch (Exception ex) {
- LOGGER.error(ex.getMessage());
- }
- }
-
- /**
- * Generates over 1000 objects and upload them to S3 mock server, 1 record per object
- */
- private static void loadLargeNumberOfFiles() {
- for (int i = 0; i < OVER_1000_OBJECTS_COUNT; i++) {
- RequestBody body = RequestBody.fromString("{\"id\":" + i + "}");
- client.putObject(builder.key(OVER_1000_OBJECTS_PATH + "/" + i + ".json").build(), body);
- }
- }
-
- /**
- * Loads a combination of different file formats in the same path
- */
- private static void prepareMixedDataBucket() {
- LOGGER.info("creating bucket " + INCLUDE_EXCLUDE_BUCKET);
- client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_BUCKET).build());
- LOGGER.info("bucket " + INCLUDE_EXCLUDE_BUCKET + " created successfully");
-
- // JSON
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2018.json").build(),
- RequestBody.fromString("{\"id\":" + 1 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2019.json").build(),
- RequestBody.fromString("{\"id\":" + 2 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2020.json").build(),
- RequestBody.fromString("{\"id\":" + 3 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2018.json").build(),
- RequestBody.fromString("{\"id\":" + 4 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2019.json").build(),
- RequestBody.fromString("{\"id\":" + 5 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2020.json").build(),
- RequestBody.fromString("{\"id\":" + 6 + "}"));
-
- // CSV
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2018.csv").build(),
- RequestBody.fromString("7,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2019.csv").build(),
- RequestBody.fromString("8,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2020.csv").build(),
- RequestBody.fromString("{9,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2018.csv").build(),
- RequestBody.fromString("10,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2019.csv").build(),
- RequestBody.fromString("11,\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2020.csv").build(),
- RequestBody.fromString("12,\"good\""));
-
- // TSV
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2018.tsv").build(),
- RequestBody.fromString("13\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2019.tsv").build(),
- RequestBody.fromString("14\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2020.tsv").build(),
- RequestBody.fromString("15\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2018.tsv").build(),
- RequestBody.fromString("16\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2019.tsv").build(),
- RequestBody.fromString("17\t\"good\""));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2020.tsv").build(),
- RequestBody.fromString("18\t\"good\""));
-
- // JSON no extension
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2018").build(),
- RequestBody.fromString("{\"id\":" + 1 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2019").build(),
- RequestBody.fromString("{\"id\":" + 2 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2020").build(),
- RequestBody.fromString("{\"id\":" + 3 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2018").build(),
- RequestBody.fromString("{\"id\":" + 4 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2019").build(),
- RequestBody.fromString("{\"id\":" + 5 + "}"));
- client.putObject(
- includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2020").build(),
- RequestBody.fromString("{\"id\":" + 6 + "}"));
- }
-
static class AwsTestExecutor extends TestExecutor {
public void executeTestFile(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java
new file mode 100644
index 0000000..08e1fe7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.external_dataset.aws;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+@RunWith(Parameterized.class)
+public class ExternalDatasetTestUtils {
+
+ protected static final Logger LOGGER = LogManager.getLogger();
+
+ // Base directory paths for data files
+ private static String JSON_DATA_PATH;
+ private static String CSV_DATA_PATH;
+ private static String TSV_DATA_PATH;
+ private static String MIXED_DATA_PATH;
+
+ // IMPORTANT: The following values must be used in the AWS S3 test case
+ // Region, container and definitions
+ public static final String PLAYGROUND_CONTAINER = "playground";
+ public static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data
+ public static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
+ public static final String JSON_DEFINITION = "json-data/reviews/";
+ public static final String CSV_DEFINITION = "csv-data/reviews/";
+ public static final String TSV_DEFINITION = "tsv-data/reviews/";
+
+ // This is used for a test to generate over 1000 number of files
+ public static final String OVER_1000_OBJECTS_PATH = "over-1000-objects";
+ public static final int OVER_1000_OBJECTS_COUNT = 2999;
+
+ public static final PutObjectRequest.Builder playgroundBuilder =
+ PutObjectRequest.builder().bucket(PLAYGROUND_CONTAINER);
+ public static final PutObjectRequest.Builder fixedDataBuilder =
+ PutObjectRequest.builder().bucket(FIXED_DATA_CONTAINER);
+ public static final PutObjectRequest.Builder includeExcludeBuilder =
+ PutObjectRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER);
+
+ protected TestCaseContext tcCtx;
+
+ public ExternalDatasetTestUtils(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ public static void setDataPaths(String jsonDataPath, String csvDataPath, String tsvDataPath, String mixedDataPath) {
+ JSON_DATA_PATH = jsonDataPath;
+ CSV_DATA_PATH = csvDataPath;
+ TSV_DATA_PATH = tsvDataPath;
+ MIXED_DATA_PATH = mixedDataPath;
+ }
+
+ /**
+ * Creates a bucket and fills it with some files for testing purpose.
+ */
+ public static void prepareS3Bucket(S3Client client) {
+ LOGGER.info("creating bucket " + PLAYGROUND_CONTAINER);
+ client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
+ LOGGER.info("bucket created successfully");
+
+ LOGGER.info("Adding JSON files to the bucket");
+ loadJsonFiles(client);
+ LOGGER.info("JSON Files added successfully");
+
+ LOGGER.info("Adding CSV files to the bucket");
+ loadCsvFiles(client);
+ LOGGER.info("CSV Files added successfully");
+
+ LOGGER.info("Adding TSV files to the bucket");
+ loadTsvFiles(client);
+ LOGGER.info("TSV Files added successfully");
+
+ LOGGER.info("Adding a big JSON file");
+ loadBigJson(client);
+ LOGGER.info("JSON file added successfully");
+
+ LOGGER.info("Loading " + OVER_1000_OBJECTS_COUNT + " into " + OVER_1000_OBJECTS_PATH);
+ loadLargeNumberOfFiles(client);
+ LOGGER.info("Added " + OVER_1000_OBJECTS_COUNT + " files into " + OVER_1000_OBJECTS_PATH + " successfully");
+
+ LOGGER.info("Files added successfully");
+ }
+
+ /**
+ * This bucket is being filled by fixed data, a test is counting all records in this bucket. If this bucket is
+ * changed, the test case will fail and its result will need to be updated each time
+ */
+ public static void prepareFixedDataBucket(S3Client client) {
+ LOGGER.info("creating bucket " + FIXED_DATA_CONTAINER);
+ client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_CONTAINER).build());
+ LOGGER.info("bucket " + FIXED_DATA_CONTAINER + " created successfully");
+
+ LOGGER.info("Loading fixed data to " + FIXED_DATA_CONTAINER);
+
+ // Files data
+ RequestBody requestBody = RequestBody.fromFile(Paths.get(JSON_DATA_PATH, "single-line", "20-records.json"));
+ client.putObject(fixedDataBuilder.key("1.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("2.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("lvl1/3.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("lvl1/4.json").build(), requestBody);
+ client.putObject(fixedDataBuilder.key("lvl1/lvl2/5.json").build(), requestBody);
+ }
+
+ public static void loadJsonFiles(S3Client client) {
+ String dataBasePath = JSON_DATA_PATH;
+ String definition = JSON_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "json";
+ loadData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+
+ definitionSegment = "json-array-of-objects";
+ loadData(client, dataBasePath, "single-line", "array_of_objects.json", "json-data/", definitionSegment, false,
+ false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadGzData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+ loadGzData(client, dataBasePath, "single-line", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines", "20-records.json", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment,
+ false);
+ loadGzData(client, dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition,
+ definitionSegment, false);
+ }
+
+ private static void loadCsvFiles(S3Client client) {
+ String dataBasePath = CSV_DATA_PATH;
+ String definition = CSV_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "csv";
+ loadData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "01.csv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.csv", definition, definitionSegment, false);
+ }
+
+ private static void loadTsvFiles(S3Client client) {
+ String dataBasePath = TSV_DATA_PATH;
+ String definition = TSV_DEFINITION;
+
+ // Normal format
+ String definitionSegment = "tsv";
+ loadData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+
+ // gz compressed format
+ definitionSegment = "gz";
+ loadGzData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+
+ // Mixed normal and gz compressed format
+ definitionSegment = "mixed";
+ loadData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "01.tsv", definition, definitionSegment, false);
+ loadGzData(client, dataBasePath, "", "02.tsv", definition, definitionSegment, false);
+ }
+
+ private static void loadData(S3Client client, String fileBasePath, String filePathSegment, String filename,
+ String definition, String definitionSegment, boolean removeExtension) {
+ loadData(client, fileBasePath, filePathSegment, filename, definition, definitionSegment, removeExtension, true);
+ }
+
+ private static void loadData(S3Client client, String fileBasePath, String filePathSegment, String filename,
+ String definition, String definitionSegment, boolean removeExtension, boolean copyToSubLevels) {
+ // Files data
+ Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
+ RequestBody requestBody = RequestBody.fromFile(filePath);
+
+ // Keep or remove the file extension
+ Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
+ String finalFileName;
+ if (removeExtension) {
+ finalFileName = FilenameUtils.removeExtension(filename);
+ } else {
+ finalFileName = filename;
+ }
+
+ // Files base definition
+ filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
+ definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
+ String basePath = definition + filePathSegment + definitionSegment;
+
+ // Load the data
+ client.putObject(playgroundBuilder.key(basePath + finalFileName).build(), requestBody);
+ if (copyToSubLevels) {
+ client.putObject(playgroundBuilder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
+ }
+ }
+
+ private static void loadGzData(S3Client client, String fileBasePath, String filePathSegment, String filename,
+ String definition, String definitionSegment, boolean removeExtension) {
+ try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
+
+ // Files data
+ Path filePath = Paths.get(fileBasePath, filePathSegment, filename);
+
+ // Get the compressed data
+ gzipOutputStream.write(Files.readAllBytes(filePath));
+ gzipOutputStream.close(); // Need to close or data will be invalid
+ byte[] gzipBytes = byteArrayOutputStream.toByteArray();
+ RequestBody requestBody = RequestBody.fromBytes(gzipBytes);
+
+ // Keep or remove the file extension
+ Assert.assertFalse("Files with no extension are not supported yet for external datasets", removeExtension);
+ String finalFileName;
+ if (removeExtension) {
+ finalFileName = FilenameUtils.removeExtension(filename);
+ } else {
+ finalFileName = filename;
+ }
+ finalFileName += ".gz";
+
+ // Files base definition
+ filePathSegment = filePathSegment.isEmpty() ? "" : filePathSegment + "/";
+ definitionSegment = definitionSegment.isEmpty() ? "" : definitionSegment + "/";
+ String basePath = definition + filePathSegment + definitionSegment;
+
+ // Load the data
+ client.putObject(playgroundBuilder.key(basePath + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1b/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2a/" + finalFileName).build(), requestBody);
+ client.putObject(playgroundBuilder.key(basePath + "level1a/level2b/" + finalFileName).build(), requestBody);
+ } catch (Exception ex) {
+ LOGGER.error(ex.getMessage());
+ }
+ }
+
+ private static void loadBigJson(S3Client client) {
+ String fileName = "big_record.json";
+ int bufferSize = 4 * 1024 * 1024;
+ int maxSize = bufferSize * 9;
+ Path filePath = Paths.get("target", "rttest", "tmp", fileName);
+ try {
+ if (Files.notExists(filePath)) {
+ Files.createDirectories(filePath.getParent());
+ Files.createFile(filePath);
+ }
+ } catch (IOException ex) {
+ throw new IllegalStateException("File " + fileName + " not found");
+ }
+
+ try (FileWriter writer = new FileWriter(filePath.toFile(), false);
+ BufferedWriter bw = new BufferedWriter(writer, bufferSize)) {
+ bw.append("{ \"large_field\": \"");
+ for (int i = 0; i < maxSize; i++) {
+ bw.append('A');
+ }
+ bw.append("\" }");
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ String key = "big-json/" + fileName;
+ client.putObject(playgroundBuilder.key(key).build(), RequestBody.fromFile(filePath));
+ }
+
+ /**
+ * Generates over 1000 objects and upload them to S3 mock server, 1 record per object
+ */
+ private static void loadLargeNumberOfFiles(S3Client client) {
+ for (int i = 0; i < OVER_1000_OBJECTS_COUNT; i++) {
+ RequestBody body = RequestBody.fromString("{\"id\":" + i + "}");
+ client.putObject(playgroundBuilder.key(OVER_1000_OBJECTS_PATH + "/" + i + ".json").build(), body);
+ }
+ }
+
+ /**
+ * Loads a combination of different file formats in the same path
+ */
+ public static void prepareMixedDataBucket(S3Client client) {
+ LOGGER.info("creating bucket " + INCLUDE_EXCLUDE_CONTAINER);
+ client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER).build());
+ LOGGER.info("bucket " + INCLUDE_EXCLUDE_CONTAINER + " created successfully");
+
+ // JSON
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/extension/" + "hello-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2018.json").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2019.json").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/EXTENSION/" + "goodbye-world-2020.json").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+
+ // CSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2018.csv").build(),
+ RequestBody.fromString("7,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2019.csv").build(),
+ RequestBody.fromString("8,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/extension/" + "hello-world-2020.csv").build(),
+ RequestBody.fromString("9,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2018.csv").build(),
+ RequestBody.fromString("10,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2019.csv").build(),
+ RequestBody.fromString("11,\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/csv/EXTENSION/" + "goodbye-world-2020.csv").build(),
+ RequestBody.fromString("12,\"good\""));
+
+ // TSV
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2018.tsv").build(),
+ RequestBody.fromString("13\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2019.tsv").build(),
+ RequestBody.fromString("14\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/extension/" + "hello-world-2020.tsv").build(),
+ RequestBody.fromString("15\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2018.tsv").build(),
+ RequestBody.fromString("16\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2019.tsv").build(),
+ RequestBody.fromString("17\t\"good\""));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/tsv/EXTENSION/" + "goodbye-world-2020.tsv").build(),
+ RequestBody.fromString("18\t\"good\""));
+
+ // JSON no extension
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 1 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 2 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/no-extension/" + "hello-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 3 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2018").build(),
+ RequestBody.fromString("{\"id\":" + 4 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2019").build(),
+ RequestBody.fromString("{\"id\":" + 5 + "}"));
+ client.putObject(
+ includeExcludeBuilder.key(MIXED_DATA_PATH + "/json/NO-EXTENSION/" + "goodbye-world-2020").build(),
+ RequestBody.fromString("{\"id\":" + 6 + "}"));
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
Gerrit-Change-Number: 11444
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange
Change in asterixdb[cheshire-cat]: Reuse/share code for external dataset tests (p1)
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Hussain Towaileb <hu...@gmail.com>:
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444
to look at the new patch set (#2).
Change subject: Reuse/share code for external dataset tests (p1)
......................................................................
Reuse/share code for external dataset tests (p1)
Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/ExternalDatasetTestUtils.java
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/bad-name-1/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/bad-name-2/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/bad-name-3/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/both/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/exclude-1/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/exclude-2/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/exclude-3/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/exclude-4/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/exclude-5/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/exclude-6/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/exclude-all/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-1/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-10/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-11/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-12/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-2/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-3/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-4/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-5/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-6/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-7/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-8/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-9/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/include-exclude/include-all/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/no-files-returned/exclude-all-files/test.000.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/no-files-returned/include-no-files/test.000.ddl.sqlpp
28 files changed, 431 insertions(+), 344 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/44/11444/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
Gerrit-Change-Number: 11444
Gerrit-PatchSet: 2
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-MessageType: newpatchset
Change in asterixdb[cheshire-cat]: Reuse/share code for external dataset tests (p1)
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Jenkins <je...@fulliautomatix.ics.uci.edu>:
Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444 )
Change subject: Reuse/share code for external dataset tests (p1)
......................................................................
Patch Set 3: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/11995/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11444
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I88c5c9b037063035712ce640aeef5fe4caad9e87
Gerrit-Change-Number: 11444
Gerrit-PatchSet: 3
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Comment-Date: Thu, 20 May 2021 18:13:28 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment