You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by pr...@apache.org on 2023/03/30 19:16:00 UTC
[druid] branch master updated: Using MinIO to run S3DeepStorage ITs (#13997)
This is an automated email from the ASF dual-hosted git repository.
progers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new eb31207402 Using MinIO to run S3DeepStorage ITs (#13997)
eb31207402 is described below
commit eb3120740227f10f6d198ee614e32ba7aacd810d
Author: abhagraw <99...@users.noreply.github.com>
AuthorDate: Fri Mar 31 00:45:53 2023 +0530
Using MinIO to run S3DeepStorage ITs (#13997)
* Using MinIO to S3DeepStorage ITs
* Adding S3DeepStorageTest to github actions revised ITs
---
.github/workflows/reusable-revised-its.yml | 20 +++++++++++++++
.github/workflows/revised-its.yml | 15 +++++++++++
.../cases/cluster/Common/dependencies.yaml | 30 +++++++++++++++++++++-
.../org/apache/druid/testsEx/utils/S3TestUtil.java | 11 +++++++-
.../cases/templates/S3DeepStorage.py | 25 ++++++++++++++++++
it.sh | 2 +-
6 files changed, 100 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/reusable-revised-its.yml b/.github/workflows/reusable-revised-its.yml
index 5a220d5755..4f75c9f5ba 100644
--- a/.github/workflows/reusable-revised-its.yml
+++ b/.github/workflows/reusable-revised-its.yml
@@ -42,9 +42,29 @@ on:
required: false
type: string
default: com.mysql.jdbc.Driver
+ DRUID_CLOUD_BUCKET:
+ required: false
+ type: string
+ DRUID_CLOUD_PATH:
+ required: false
+ type: string
+ AWS_REGION:
+ required: false
+ type: string
+ AWS_ACCESS_KEY_ID:
+ required: false
+ type: string
+ AWS_SECRET_ACCESS_KEY:
+ required: false
+ type: string
env:
MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} # Used by tests to connect to metadata store directly.
+ DRUID_CLOUD_BUCKET: ${{ inputs.DRUID_CLOUD_BUCKET }}
+ DRUID_CLOUD_PATH: ${{ inputs.DRUID_CLOUD_PATH }}
+ AWS_REGION: ${{ inputs.AWS_REGION }}
+ AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
jobs:
diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml
index 83cad23f42..576fbd3fb5 100644
--- a/.github/workflows/revised-its.yml
+++ b/.github/workflows/revised-its.yml
@@ -36,3 +36,18 @@ jobs:
script: ./it.sh github ${{ matrix.it }}
it: ${{ matrix.it }}
mysql_driver: com.mysql.jdbc.Driver
+
+ s3-deep-storage-minio:
+ uses: ./.github/workflows/reusable-revised-its.yml
+ with:
+ build_jdk: 8
+ runtime_jdk: 11
+ use_indexer: middleManager
+ script: ./it.sh github S3DeepStorage
+ it: S3DeepStorage
+ mysql_driver: com.mysql.jdbc.Driver
+ DRUID_CLOUD_BUCKET: druid-qa
+ DRUID_CLOUD_PATH: aws-${{ github.run_id }}-${{ github.run_attempt }}
+ AWS_REGION: us-east-1
+ AWS_ACCESS_KEY_ID: admin
+ AWS_SECRET_ACCESS_KEY: miniopassword
diff --git a/integration-tests-ex/cases/cluster/Common/dependencies.yaml b/integration-tests-ex/cases/cluster/Common/dependencies.yaml
index 4eeb85a49c..1f43dc5a75 100644
--- a/integration-tests-ex/cases/cluster/Common/dependencies.yaml
+++ b/integration-tests-ex/cases/cluster/Common/dependencies.yaml
@@ -86,7 +86,35 @@ services:
MYSQL_USER: druid
MYSQL_PASSWORD: diurd
-## TODO: Not yet retested
+ minio:
+ container_name: minio
+ command: server /data --console-address ":9001"
+ networks:
+ druid-it-net:
+ ipv4_address: 172.172.172.5
+ image: minio/minio:latest
+ ports:
+ - '9000:9000'
+ - '9001:9001'
+ volumes:
+ - ${SHARED_DIR}/minio:/data
+ environment:
+ - MINIO_ROOT_USER=${AWS_ACCESS_KEY_ID}
+ - MINIO_ROOT_PASSWORD=${AWS_SECRET_ACCESS_KEY}
+
+ create_minio_buckets:
+ image: minio/mc
+ networks:
+ druid-it-net:
+ ipv4_address: 172.172.172.6
+ entrypoint: >
+ /bin/sh -c "
+ /usr/bin/mc alias set s3 http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY};
+ /usr/bin/mc mb s3/${DRUID_CLOUD_BUCKET};
+ /usr/bin/mc anonymous set public s3/${DRUID_CLOUD_BUCKET};
+ "
+
+ ## TODO: Not yet retested
### Optional supporting infra
openldap:
diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java
index 73c851e428..26244f78e2 100644
--- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java
+++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/utils/S3TestUtil.java
@@ -19,9 +19,12 @@
package org.apache.druid.testsEx.utils;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
@@ -81,10 +84,16 @@ public class S3TestUtil
private AmazonS3 s3Client()
{
AWSCredentials credentials = new BasicAWSCredentials(S3_ACCESS_KEY, S3_SECRET_KEY);
+ ClientConfiguration clientConfig = new ClientConfiguration();
+ clientConfig.setProtocol(Protocol.HTTP);
return AmazonS3ClientBuilder
.standard()
.withCredentials(new AWSStaticCredentialsProvider(credentials))
- .withRegion(S3_REGION)
+ // Setting endpoint to MinIO S3 API endpoint (e.g., "http://localhost:9000")
+ // configured in integration-tests-ex/cases/cluster/Common/dependencies.yaml
+ .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:9000", S3_REGION))
+ .withPathStyleAccessEnabled(true)
+ .withClientConfiguration(clientConfig)
.build();
}
diff --git a/integration-tests-ex/cases/templates/S3DeepStorage.py b/integration-tests-ex/cases/templates/S3DeepStorage.py
index 9fb85ca6a7..772ce333d1 100644
--- a/integration-tests-ex/cases/templates/S3DeepStorage.py
+++ b/integration-tests-ex/cases/templates/S3DeepStorage.py
@@ -14,6 +14,7 @@
# limitations under the License.
from template import BaseTemplate, generate
+from template import ZOO_KEEPER, METADATA, OVERLORD
class Template(BaseTemplate):
@@ -39,8 +40,32 @@ class Template(BaseTemplate):
self.add_property(service, 'druid.storage.baseKey', '${DRUID_CLOUD_PATH}')
self.add_env(service, 'AWS_REGION', '${AWS_REGION}')
+ # Adding the following to make druid work with MinIO
+ # See https://blog.min.io/how-to-druid-superset-minio/ for more details
+ self.add_property(service, 'druid.s3.protocol', 'http')
+ self.add_property(service, 'druid.s3.enablePathStyleAccess', 'true')
+ self.add_property(service, 'druid.s3.endpoint.url', 'http://172.172.172.5:9000/')
+
+ def define_overlord(self):
+ service = self.define_druid_service(OVERLORD, OVERLORD)
+ self.add_depends(service, [ZOO_KEEPER, METADATA, "create_minio_buckets"])
+ return service
+
# This test uses different data than the default.
def define_data_dir(self, service):
self.add_volume(service, '../data', '/resources')
+ def create_minio_container(self):
+ return self.define_external_service("minio")
+
+ def create_minio_bucket(self):
+ service = self.define_external_service("create_minio_buckets")
+ self.add_depends(service, ["minio"])
+ return service
+
+ def define_custom_services(self):
+ self.create_minio_container()
+ self.create_minio_bucket()
+
+
generate(__file__, Template())
diff --git a/it.sh b/it.sh
index b03b7acb57..2c6aa86062 100755
--- a/it.sh
+++ b/it.sh
@@ -148,7 +148,7 @@ function prepare_docker {
}
function require_env_var {
- if [ -n "$1" ]; then
+ if [ -z "$1" ]; then
echo "$1 must be set for test category $CATEGORY" 1>&2
exit 1
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org