[dolphinscheduler] branch dev updated: [Feature-7993][E2E] Replace HDFS image with MinIO S3 image (#8066)

[ke...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kezhenxu94 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


The following commit(s) were added to refs/heads/dev by this push:
     new b705a08  [Feature-7993][E2E] Replace HDFS image with MinIO S3 image (#8066)
b705a08 is described below

commit b705a08794f34acdd0be32e6e535a1e6973267ad
Author: xiangzihao <46...@qq.com>
AuthorDate: Mon Jan 17 19:32:47 2022 +0800

    [Feature-7993][E2E] Replace HDFS image with MinIO S3 image (#8066)
---
 .../resources/docker/file-manage/common.properties | 10 ++---
 .../docker/file-manage/docker-compose.yaml         | 45 ++++++++++++++++++----
 2 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
index a561374..57f591a 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
@@ -19,7 +19,7 @@
 data.basedir.path=/tmp/dolphinscheduler
 
 # resource storage type: HDFS, S3, NONE
-resource.storage.type=HDFS
+resource.storage.type=S3
 
 # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
 resource.upload.path=/dolphinscheduler
@@ -46,16 +46,16 @@ kerberos.expire.time=2
 hdfs.root.user=hdfs
 
 # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
-fs.defaultFS=hdfs://hdfs:8020
+fs.defaultFS=s3a://dolphinscheduler
 
 # if resource.storage.type=S3, s3 endpoint
-fs.s3a.endpoint=http://192.168.xx.xx:9010
+fs.s3a.endpoint=http://10.1.0.1:9000
 
 # if resource.storage.type=S3, s3 access key
-fs.s3a.access.key=A3DXS30FO22544RE
+fs.s3a.access.key=accessKey123
 
 # if resource.storage.type=S3, s3 secret key
-fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
+fs.s3a.secret.key=secretKey123
 
 # resourcemanager port, the default value is 8088 if not specified
 resource.manager.httpaddress.port=8088
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
index c9e8db7..4b5e254 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
@@ -35,22 +35,51 @@ services:
     volumes:
       - ./common.properties:/opt/dolphinscheduler/conf/common.properties
     depends_on:
-      hdfs:
+      s3:
         condition: service_healthy
-  hdfs:
-    image: mdouchement/hdfs:latest
-    hostname: hdfs
+      mc:
+        condition: service_healthy
+  s3:
+    image: minio/minio:latest
+    hostname: s3
     tty: true
     stdin_open: true
-    expose:
-      - 8020
+    command: server /data --console-address ":9001"
+    ports:
+      - 9000:9000
     networks:
       - e2e
+    environment:
+      MINIO_ROOT_USER: accessKey123
+      MINIO_ROOT_PASSWORD: secretKey123
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:50070" ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 5s
       timeout: 120s
       retries: 120
-
+  mc:
+    image: minio/mc:latest
+    entrypoint: ""
+    hostname: mc
+    tty: true
+    stdin_open: true
+    networks:
+      - e2e
+    command: bash -c '
+      mc alias set s3 http://s3:9000 accessKey123 secretKey123
+      && mc mb s3/dolphinscheduler && tail -f /dev/null'
+    healthcheck:
+      test: [ "CMD", "echo", "1" ]
+      interval: 5s
+      timeout: 120s
+      retries: 120
+    depends_on:
+      s3:
+        condition: service_healthy
 networks:
   e2e:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.1.0.0/24
+          gateway: 10.1.0.1