You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2020/12/27 15:00:15 UTC
[incubator-hop] branch master updated: HOP-2318: add database
support to integration testing
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new 23e2696 HOP-2318: add database support to integration testing
new 896712c Merge pull request #499 from hansva/HOP-2318
23e2696 is described below
commit 23e2696627b02897f345d1c00583b0953f06e55b
Author: Hans Van Akelyen <ha...@gmail.com>
AuthorDate: Sun Dec 27 15:57:32 2020 +0100
HOP-2318: add database support to integration testing
- Move to docker-compose and change structure of project
---
Jenkinsfile.daily | 16 +++---
.../integration-tests/Dockerfile.unit-tests | 0
docker/integration-tests/database.env | 20 +++++++
docker/integration-tests/docker-compose-ci.yaml | 40 +++++++++++++
.../database/main-0001-database-connection.hwf | 66 ++++++++++++++++++++++
.../database/metadata/rdbms/unit-test-db.json | 26 +++++++++
.../metadata/workflow-run-configuration/local.json | 9 +++
integration-tests/database/project-config.json | 9 +++
integration-tests/scripts/run-tests.sh | 33 ++++++++++-
9 files changed, 209 insertions(+), 10 deletions(-)
diff --git a/Jenkinsfile.daily b/Jenkinsfile.daily
index ccb7ab3..2fd573a 100644
--- a/Jenkinsfile.daily
+++ b/Jenkinsfile.daily
@@ -102,24 +102,22 @@ pipeline {
stage('Create Docker image & Run Tests'){
steps {
script {
- dockerImage = docker.build(imagename, "--build-arg JENKINS_USER=${jenkins_user} --build-arg JENKINS_UID=${jenkins_uid} --build-arg JENKINS_GROUP=${jenkins_group} --build-arg JENKINS_GID=${jenkins_gid} -f Dockerfile.unit-tests .")
- dockerImage.inside("-v $WORKSPACE/integration-tests:/files"){
- echo 'Run tests'
- sh 'cd /files/scripts && ./run-tests.sh'
+ docker-compose -f docker/integration-tests/docker-compose-ci.yaml up -d
+ docker wait integration-tests_integration_test_1
+ docker logs integration-tests_integration_test_1
}
}
}
post {
always {
junit(testResults: 'integration-tests/**/surefire-reports/*.xml', allowEmptyResults: true)
+ script{
+ docker-compose -f docker/integration-tests/docker-compose-ci.yaml down --rmi all
+ }
}
}
}
- stage('Remove docker image'){
- steps {
- sh "docker rmi $imagename:latest"
- }
- }
+
}
post {
always {
diff --git a/Dockerfile.unit-tests b/docker/integration-tests/Dockerfile.unit-tests
similarity index 100%
rename from Dockerfile.unit-tests
rename to docker/integration-tests/Dockerfile.unit-tests
diff --git a/docker/integration-tests/database.env b/docker/integration-tests/database.env
new file mode 100644
index 0000000..fd25663
--- /dev/null
+++ b/docker/integration-tests/database.env
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+POSTGRES_USER=hop_user
+POSTGRES_PASSWORD=hop_password
+POSTGRES_DB=hop_database
\ No newline at end of file
diff --git a/docker/integration-tests/docker-compose-ci.yaml b/docker/integration-tests/docker-compose-ci.yaml
new file mode 100644
index 0000000..426d5a8
--- /dev/null
+++ b/docker/integration-tests/docker-compose-ci.yaml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3'
+services:
+ integration_test:
+ build:
+ context: ../../.
+ dockerfile: docker/integration-tests/Dockerfile.unit-tests
+ volumes:
+ - ../../integration-tests/:/files
+ depends_on:
+ - postgres
+ links:
+ - postgres
+ environment:
+ - FLASK_ENV=docker
+ #command: ["bash", "-c", "ls -al /files/scripts/"]
+ command: ["bash", "-c", "/files/scripts/run-tests.sh"]
+ #command: ["bash", "-c", "PGPASSWORD=hop_password psql --host=postgres --username=hop_user --dbname=hop_database"]
+ postgres:
+ image: postgres:latest
+ env_file:
+ - database.env
+ ports:
+ - "5432"
\ No newline at end of file
diff --git a/integration-tests/database/main-0001-database-connection.hwf b/integration-tests/database/main-0001-database-connection.hwf
new file mode 100644
index 0000000..1816055
--- /dev/null
+++ b/integration-tests/database/main-0001-database-connection.hwf
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<workflow>
+ <name>main-0001-database-connection</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2020/12/27 15:11:35.027</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2020/12/27 15:11:35.027</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>SQL</name>
+ <description/>
+ <type>SQL</type>
+ <attributes/>
+ <sql>select 1</sql>
+ <useVariableSubstitution>F</useVariableSubstitution>
+ <sqlfromfile>F</sqlfromfile>
+ <sqlfilename/>
+ <sendOneStatement>F</sendOneStatement>
+ <connection>unit-test-db</connection>
+ <parallel>N</parallel>
+ <nr>0</nr>
+ <xloc>195</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>SQL</to>
+ <from_nr>0</from_nr>
+ <to_nr>0</to_nr>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/database/metadata/rdbms/unit-test-db.json b/integration-tests/database/metadata/rdbms/unit-test-db.json
new file mode 100644
index 0000000..708d0a0
--- /dev/null
+++ b/integration-tests/database/metadata/rdbms/unit-test-db.json
@@ -0,0 +1,26 @@
+{
+ "rdbms": {
+ "POSTGRESQL": {
+ "databaseName": "${POSTGRES_DATABASE}",
+ "pluginId": "POSTGRESQL",
+ "accessType": 0,
+ "hostname": "${POSTGRES_HOST}",
+ "password": "${POSTGRES_PASSWORD}",
+ "port": "${POSTGRES_PORT}",
+ "pluginName": "PostgreSQL",
+ "attributes": {
+ "SUPPORTS_TIMESTAMP_DATA_TYPE": "Y",
+ "QUOTE_ALL_FIELDS": "N",
+ "SUPPORTS_BOOLEAN_DATA_TYPE": "Y",
+ "FORCE_IDENTIFIERS_TO_LOWERCASE": "N",
+ "PRESERVE_RESERVED_WORD_CASE": "Y",
+ "SQL_CONNECT": "",
+ "FORCE_IDENTIFIERS_TO_UPPERCASE": "N",
+ "PREFERRED_SCHEMA_NAME": ""
+ },
+ "manualUrl": "",
+ "username": "${POSTGRES_USER}"
+ }
+ },
+ "name": "unit-test-db"
+}
\ No newline at end of file
diff --git a/integration-tests/database/metadata/workflow-run-configuration/local.json b/integration-tests/database/metadata/workflow-run-configuration/local.json
new file mode 100644
index 0000000..e37a930
--- /dev/null
+++ b/integration-tests/database/metadata/workflow-run-configuration/local.json
@@ -0,0 +1,9 @@
+{
+ "engineRunConfiguration": {
+ "Local": {
+ "safe_mode": false
+ }
+ },
+ "name": "local",
+ "description": "Runs your workflows locally with the standard local Hop workflow engine"
+}
\ No newline at end of file
diff --git a/integration-tests/database/project-config.json b/integration-tests/database/project-config.json
new file mode 100644
index 0000000..f8d4595
--- /dev/null
+++ b/integration-tests/database/project-config.json
@@ -0,0 +1,9 @@
+{
+ "metadataBaseFolder" : "${PROJECT_HOME}/metadata",
+ "unitTestsBasePath" : "${PROJECT_HOME}",
+ "dataSetsCsvFolder" : "${PROJECT_HOME}/datasets",
+ "enforcingExecutionInHome" : true,
+ "config" : {
+ "variables" : [ ]
+ }
+}
\ No newline at end of file
diff --git a/integration-tests/scripts/run-tests.sh b/integration-tests/scripts/run-tests.sh
index 572cdb6..089dda5 100755
--- a/integration-tests/scripts/run-tests.sh
+++ b/integration-tests/scripts/run-tests.sh
@@ -24,6 +24,29 @@ if [ -z "${HOP_LOCATION}" ]; then
HOP_LOCATION=/opt/hop
fi
+
+# Get database parameters
+if [ -z "${POSTGRES_HOST}" ]; then
+ POSTGRES_HOST=postgres
+fi
+
+if [ -z "${POSTGRES_DATABASE}" ]; then
+ POSTGRES_DATABASE=hop_database
+fi
+
+if [ -z "${POSTGRES_PORT}" ]; then
+ POSTGRES_PORT=5432
+fi
+
+if [ -z "${POSTGRES_USER}" ]; then
+ POSTGRES_USER=hop_user
+fi
+
+if [ -z "${POSTGRES_PASSWORD}" ]; then
+ POSTGRES_PASSWORD=hop_password
+fi
+
+
#set global variables
SPACER="==========================================="
@@ -99,7 +122,15 @@ for d in "${CURRENT_DIR}"/../*/ ; do
start_time_test=$SECONDS
#Run Test
- $HOP_LOCATION/hop-run.sh -j ${PROJECT_NAME} -r "local" -f $hop_file > >(tee /tmp/test_output) 2> >(tee /tmp/test_output_err >&1)
+ $HOP_LOCATION/hop-run.sh \
+ -j ${PROJECT_NAME} \
+ -r "local" \
+ -p POSTGRES_HOST=${POSTGRES_HOST} \
+ -p POSTGRES_DATABASE=${POSTGRES_DATABASE} \
+ -p POSTGRES_PORT=${POSTGRES_PORT} \
+ -p POSTGRES_USER=${POSTGRES_USER} \
+ -p POSTGRES_PASSWORD=${POSTGRES_PASSWORD} \
+ -f $hop_file > >(tee /tmp/test_output) 2> >(tee /tmp/test_output_err >&1)
#Capture exit code
exit_code=${PIPESTATUS[0]}