You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2021/06/02 13:59:03 UTC
[incubator-hop] branch master updated: HOP-2914: update integration
testing to be more modular (#845)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new 2c8cfbe HOP-2914: update integration testing to be more modular (#845)
2c8cfbe is described below
commit 2c8cfbe9ea94b2e2fc0f7fac32682cbecefa7fc0
Author: Hans Van Akelyen <ha...@gmail.com>
AuthorDate: Wed Jun 2 15:58:55 2021 +0200
HOP-2914: update integration testing to be more modular (#845)
* HOP-2914: split compose files and make tests more modular
* HOP-2914: fix containers on jenkins
* HOP-2914: add build args to custom images
* HOP-2914: add args
* HOP-2914: fix variables
* HOP-2914: update docs and enable full build again
---
Jenkinsfile.daily | 6 +-
docker/integration-tests/docker-compose-ci.yaml | 89 -----------------
.../integration-tests/integration-tests-base.yaml | 2 +-
...-base.yaml => integration-tests-cassandra.yaml} | 34 ++++---
...ests-base.yaml => integration-tests-mongo.yaml} | 33 +++---
...ests-base.yaml => integration-tests-neo4j.yaml} | 34 ++++---
.../modules/ROOT/pages/integration-testing.adoc | 21 ++--
.../scripts/main-0001-xml-join-test.hwf | 111 ---------------------
integration-tests/scripts/run-tests-docker.sh | 95 ++++++++++++++++++
integration-tests/scripts/run-tests.sh | 11 +-
.../spreadsheet/files/exelwriter-testfile.xlsx | Bin 3624 -> 3619 bytes
11 files changed, 172 insertions(+), 264 deletions(-)
diff --git a/Jenkinsfile.daily b/Jenkinsfile.daily
index f6d9292..f5e6692 100644
--- a/Jenkinsfile.daily
+++ b/Jenkinsfile.daily
@@ -97,15 +97,11 @@ pipeline {
}
stage('Create Docker image & Run Tests'){
steps {
- sh "docker-compose -f 'docker/integration-tests/docker-compose-ci.yaml' build --build-arg JENKINS_USER=${jenkins_user} --build-arg JENKINS_UID=${jenkins_uid} --build-arg JENKINS_GROUP=${jenkins_group} --build-arg JENKINS_GID=${jenkins_gid}"
- sh "docker-compose -f 'docker/integration-tests/docker-compose-ci.yaml' up -d"
- sh "docker wait integration-tests_integration_test_1"
- sh "docker logs integration-tests_integration_test_1"
+ sh "integration-tests/scripts/run-tests-docker.sh JENKINS_USER=${jenkins_user} JENKINS_UID=${jenkins_uid} JENKINS_GROUP=${jenkins_group} JENKINS_GID=${jenkins_gid}"
}
post {
always {
junit(testResults: 'integration-tests/**/surefire-reports/*.xml', allowEmptyResults: true)
- sh "docker-compose -f 'docker/integration-tests/docker-compose-ci.yaml' down --rmi all"
}
}
}
diff --git a/docker/integration-tests/docker-compose-ci.yaml b/docker/integration-tests/docker-compose-ci.yaml
deleted file mode 100644
index 89807fe..0000000
--- a/docker/integration-tests/docker-compose-ci.yaml
+++ /dev/null
@@ -1,89 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-version: '3'
-services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
- depends_on:
- postgres:
- condition: service_healthy
- neo4j:
- condition: service_healthy
- mongo:
- condition: service_healthy
- cassandra:
- condition: service_healthy
- links:
- - postgres
- - neo4j
- - mongo
- - cassandra
- environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
- postgres:
- image: postgres:latest
- env_file:
- - database.env
- ports:
- - "5432"
- healthcheck:
- test: [ "CMD-SHELL", "pg_isready -U postgres" ]
- interval: 10s
- timeout: 5s
- retries: 5
- neo4j:
- image: neo4j:latest
- ports:
- - "7474"
- - "7687"
- environment:
- - NEO4J_AUTH=neo4j/abcd
- healthcheck:
- test: echo 'ok'
- interval: 10s
- timeout: 10s
- retries: 5
- mongo:
- image: mongo:latest
- ports:
- - "27017"
- healthcheck:
- test: echo 'db.runCommand("ping").ok' | mongo 127.0.0.1:27017/test --quiet
- interval: 10s
- timeout: 10s
- retries: 5
- cassandra:
- image: cassandra:latest
- ports:
- - "9042"
- - "9160"
- healthcheck:
- test: [ "CMD-SHELL", "[ $$(nodetool statusgossip) = running ]" ]
- interval: 10s
- timeout: 5s
- retries: 50
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml b/docker/integration-tests/integration-tests-base.yaml
index ed9f0cd..31360bf 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-base.yaml
@@ -30,4 +30,4 @@ services:
- ../../integration-tests/:/files
environment:
- FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml b/docker/integration-tests/integration-tests-cassandra.yaml
similarity index 64%
copy from docker/integration-tests/integration-tests-base.yaml
copy to docker/integration-tests/integration-tests-cassandra.yaml
index ed9f0cd..6f2b5c1 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-cassandra.yaml
@@ -17,17 +17,23 @@
version: '3'
services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
- environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ integration_test_cassandra:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ depends_on:
+ cassandra:
+ condition: service_healthy
+ links:
+ - cassandra
+
+ cassandra:
+ image: cassandra:latest
+ ports:
+ - "9042"
+ - "9160"
+ healthcheck:
+ test: [ "CMD-SHELL", "[ $$(nodetool statusgossip) = running ]" ]
+ interval: 10s
+ timeout: 5s
+ retries: 50
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml b/docker/integration-tests/integration-tests-mongo.yaml
similarity index 66%
copy from docker/integration-tests/integration-tests-base.yaml
copy to docker/integration-tests/integration-tests-mongo.yaml
index ed9f0cd..16ee0da 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-mongo.yaml
@@ -17,17 +17,22 @@
version: '3'
services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
- environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ integration_test_mongo:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ depends_on:
+ mongo:
+ condition: service_healthy
+ links:
+ - mongo
+
+ mongo:
+ image: mongo:latest
+ ports:
+ - "27017"
+ healthcheck:
+ test: echo 'db.runCommand("ping").ok' | mongo 127.0.0.1:27017/test --quiet
+ interval: 10s
+ timeout: 10s
+ retries: 5
\ No newline at end of file
diff --git a/docker/integration-tests/integration-tests-base.yaml b/docker/integration-tests/integration-tests-neo4j.yaml
similarity index 67%
copy from docker/integration-tests/integration-tests-base.yaml
copy to docker/integration-tests/integration-tests-neo4j.yaml
index ed9f0cd..6926281 100644
--- a/docker/integration-tests/integration-tests-base.yaml
+++ b/docker/integration-tests/integration-tests-neo4j.yaml
@@ -17,17 +17,25 @@
version: '3'
services:
- integration_test:
- build:
- context: ../../.
- dockerfile: docker/integration-tests/Dockerfile.unit-tests
- args:
- - JENKINS_USER=jenkins
- - JENKINS_UID=1000
- - JENKINS_GROUP=jenkins
- - JENKINS_GID=1000
- volumes:
- - ../../integration-tests/:/files
+ integration_test_neo4j:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ depends_on:
+ neo4j:
+ condition: service_healthy
+ links:
+ - neo4j
+
+ neo4j:
+ image: neo4j:latest
+ ports:
+ - "7474"
+ - "7687"
environment:
- - FLASK_ENV=docker
- command: [ "bash", "-c", "/files/scripts/run-tests.sh" ]
\ No newline at end of file
+ - NEO4J_AUTH=neo4j/abcd
+ healthcheck:
+ test: echo 'ok'
+ interval: 10s
+ timeout: 10s
+ retries: 5
\ No newline at end of file
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/integration-testing.adoc b/docs/hop-dev-manual/modules/ROOT/pages/integration-testing.adoc
index 22ad585..0b75463 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/integration-testing.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/integration-testing.adoc
@@ -85,19 +85,12 @@ mvn clean install
. Run the Docker Compose script
+
----
-docker-compose -f docker/integration-tests/docker-compose-ci.yaml up -d
-----
-+
-. Check the running container
-+
-----
-docker logs integration-tests_integration_test_1
-----
-+
-. Cleanup after run
-+
-----
-docker-compose -f docker/integration-tests/docker-compose-ci.yaml down --rmi all
+cd integration-tests/scripts
+./run-tests-docker.sh
+
+# Optionally you can run a single project by adding the PROJECT_NAME variable to the script
+
+./run-tests-docker.sh PROJECT_NAME=database
----
+
. Surefire Reports
@@ -108,7 +101,7 @@ We generate xml reports that can be used by Jenkins to generate reports. these r
A script is also provided to run the test without using Docker. This script is also used inside the automated test suite but can be used in a stand alone mode.
-This script can be found at following location `incubator-hop/integration-tests/scripts/run-tests.sh`. Be sure to export the needed system variables for the script to find the correct paths to run the tests.
+This script can be found at following location `incubator-hop/integration-tests/scripts/run-tests.sh`. Be sure to export the needed system variables for the script to find the correct paths to run the tests. You can aslo run a single project by adding the project name to the command `./run-tests.sh projectname`
== Special checks
diff --git a/integration-tests/scripts/main-0001-xml-join-test.hwf b/integration-tests/scripts/main-0001-xml-join-test.hwf
deleted file mode 100644
index 94139ef..0000000
--- a/integration-tests/scripts/main-0001-xml-join-test.hwf
+++ /dev/null
@@ -1,111 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<workflow>
- <name>main-0001-xml-join-test</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
- <extended_description/>
- <workflow_version/>
- <created_user>-</created_user>
- <created_date>2020/12/09 13:24:02.438</created_date>
- <modified_user>-</modified_user>
- <modified_date>2020/12/09 13:24:02.438</modified_date>
- <parameters>
- </parameters>
- <actions>
- <action>
- <name>START</name>
- <description/>
- <type>SPECIAL</type>
- <attributes/>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <intervalSeconds>0</intervalSeconds>
- <intervalMinutes>60</intervalMinutes>
- <hour>12</hour>
- <minutes>0</minutes>
- <weekDay>1</weekDay>
- <DayOfMonth>1</DayOfMonth>
- <parallel>N</parallel>
- <nr>0</nr>
- <xloc>64</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>0001-xml-join-test.hpl</name>
- <description/>
- <type>PIPELINE</type>
- <attributes/>
- <filename>${PROJECT_HOME}/0001-xml-join-test.hpl</filename>
- <params_from_previous>N</params_from_previous>
- <exec_per_row>N</exec_per_row>
- <clear_rows>N</clear_rows>
- <clear_files>N</clear_files>
- <set_logfile>N</set_logfile>
- <logfile/>
- <logext/>
- <add_date>N</add_date>
- <add_time>N</add_time>
- <loglevel>Basic</loglevel>
- <set_append_logfile>N</set_append_logfile>
- <wait_until_finished>Y</wait_until_finished>
- <follow_abort_remote>N</follow_abort_remote>
- <create_parent_folder>N</create_parent_folder>
- <run_configuration>local</run_configuration>
- <parameters>
- <pass_all_parameters>Y</pass_all_parameters>
- </parameters>
- <parallel>N</parallel>
- <nr>0</nr>
- <xloc>224</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>verify XML</name>
- <description/>
- <type>SIMPLE_EVAL</type>
- <attributes/>
- <valuetype>field</valuetype>
- <fieldname>xmlOutput</fieldname>
- <variablename/>
- <fieldtype>string</fieldtype>
- <mask/>
- <comparevalue><?xml version="1.0" encoding="ISO-8859-1" standalone="no"?><OrderList businessUnitId="EU10" plantId="EU11" source="XY" timestamp="19700101"><OrderHeaders><OrderHeader customerNumber="1000" orderDate="19700101" orderNumber="4711" orderType="AN" originator="G"><OrderLines><OrderLine orderLineNumber="1" principal="XY" productNumber="500000" productText="red" quantity="10" uom="KG"><OrderSubLines><OrderSubLine batch="B61" expiryDate="19700101" orde [...]
- <minvalue/>
- <maxvalue/>
- <successcondition>equal</successcondition>
- <successnumbercondition>equal</successnumbercondition>
- <successbooleancondition>false</successbooleancondition>
- <successwhenvarset>N</successwhenvarset>
- <parallel>N</parallel>
- <nr>0</nr>
- <xloc>400</xloc>
- <yloc>64</yloc>
- <attributes_hac/>
- </action>
- </actions>
- <hops>
- <hop>
- <from>START</from>
- <to>0001-xml-join-test.hpl</to>
- <from_nr>0</from_nr>
- <to_nr>0</to_nr>
- <enabled>Y</enabled>
- <evaluation>Y</evaluation>
- <unconditional>Y</unconditional>
- </hop>
- <hop>
- <from>0001-xml-join-test.hpl</from>
- <to>verify XML</to>
- <from_nr>0</from_nr>
- <to_nr>0</to_nr>
- <enabled>Y</enabled>
- <evaluation>Y</evaluation>
- <unconditional>N</unconditional>
- </hop>
- </hops>
- <notepads>
- </notepads>
- <attributes/>
-</workflow>
diff --git a/integration-tests/scripts/run-tests-docker.sh b/integration-tests/scripts/run-tests-docker.sh
new file mode 100755
index 0000000..0353790
--- /dev/null
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+DOCKER_FILES_DIR="$(cd ${CURRENT_DIR}/../../docker/integration-tests/ && pwd)"
+
+for ARGUMENT in "$@"
+do
+
+ KEY=$(echo $ARGUMENT | cut -f1 -d=)
+ VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+ case "$KEY" in
+ PROJECT_NAME) PROJECT_NAME=${VALUE} ;;
+ JENKINS_USER) JENKINS_USER=${VALUE} ;;
+ JENKINS_UID) JENKINS_UID=${VALUE} ;;
+ JENKINS_GROUP) JENKINS_GROUP=${VALUE} ;;
+ JENKINS_GID) JENKINS_GID=${VALUE} ;;
+ *)
+ esac
+
+
+done
+
+
+if [ -z "${PROJECT_NAME}" ]; then
+ PROJECT_NAME="*"
+fi
+
+if [ -z "${JENKINS_USER}" ]; then
+ JENKINS_USER="jenkins"
+fi
+
+if [ -z "${JENKINS_UID}" ]; then
+ JENKINS_UID="1000"
+fi
+
+if [ -z "${JENKINS_GROUP}" ]; then
+ JENKINS_GROUP="jenkins"
+fi
+
+if [ -z "${JENKINS_GID}" ]; then
+ JENKINS_GID="1000"
+fi
+
+#Cleanup surefire reports
+rm -rf "${CURRENT_DIR}"/../surefire-reports
+mkdir -p "${CURRENT_DIR}"/../surefire-reports/
+
+#Loop over project folders
+for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/ ; do
+
+ if [[ "$d" != *"scripts/" ]] && [[ "$d" != *"surefire-reports/" ]] && [[ "$d" != *"hopweb/" ]]; then
+
+ PROJECT_NAME=$(basename $d)
+
+ echo "Project name: ${PROJECT_NAME}"
+ echo "project path: $d"
+ echo "docker compose path: ${DOCKER_FILES_DIR}"
+
+ #Check if specific compose exists
+
+ if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml" ]; then
+ echo "Project compose exists."
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml build --build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID} --build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg JENKINS_GID=${JENKINS_GID}
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --abort-on-container-exit
+ docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml down --rmi all
+ else
+ echo "Project compose does not exists."
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml build --build-arg JENKINS_USER=${JENKINS_USER} --build-arg JENKINS_UID=${JENKINS_UID} --build-arg JENKINS_GROUP=${JENKINS_GROUP} --build-arg JENKINS_GID=${JENKINS_GID}
+ PROJECT_NAME=${PROJECT_NAME} docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
+ docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml down
+ fi
+ fi
+done
+
+#Cleanup all images
+docker-compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml down --rmi all --remove-orphans
\ No newline at end of file
diff --git a/integration-tests/scripts/run-tests.sh b/integration-tests/scripts/run-tests.sh
index 4a9b60c..f0b0ef2 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -19,6 +19,9 @@
#
CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJECT_NAME="$1"
+
+echo "The Project Name = ${PROJECT_NAME}"
if [ -z "${HOP_LOCATION}" ]; then
HOP_LOCATION=/opt/hop
@@ -49,6 +52,10 @@ if [ -z "${POSTGRES_PASSWORD}" ]; then
POSTGRES_PASSWORD=hop_password
fi
+if [ -z "${PROJECT_NAME}" ]; then
+ PROJECT_NAME="*"
+fi
+
#set global variables
SPACER="==========================================="
@@ -60,8 +67,6 @@ mkdir -p "${TMP_FOLDER}"
#cleanup Temp
export TMP_TESTCASES="${TMP_FOLDER}"/testcases.xml
rm -f "${TMP_TESTCASES}"
-rm -rf "${CURRENT_DIR}"/../surefire-reports
-mkdir -p "${CURRENT_DIR}"/../surefire-reports/
# Set up auditing
# Start with a new blank slate every time
@@ -75,7 +80,7 @@ mkdir -p "${HOP_AUDIT_FOLDER}"
TMP_CONFIG_FOLDER="${HOP_CONFIG_FOLDER}"
#Loop over project folders
-for d in "${CURRENT_DIR}"/../*/ ; do
+for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/ ; do
#cleanup project testcases
rm -f "${TMP_TESTCASES}"
diff --git a/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx b/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx
index eeaee0c..5a70089 100644
Binary files a/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx and b/integration-tests/spreadsheet/files/exelwriter-testfile.xlsx differ