You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2021/01/10 16:28:38 UTC
[iotdb] branch master updated: e2e: add a simple E2E case and set
up the workflow (#2166)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6add2e0 e2e: add a simple E2E case and set up the workflow (#2166)
6add2e0 is described below
commit 6add2e0a4d20972740498789e4b507687d60bbbc
Author: Zhenxu Ke <ke...@apache.org>
AuthorDate: Mon Jan 11 00:28:20 2021 +0800
e2e: add a simple E2E case and set up the workflow (#2166)
* test: rewrite Dockerfile to build from sources on the fly
* test: add a simple test case and set up the workflow
---
.dockerignore | 21 ++++++++++++++
.github/workflows/e2e.yml | 52 +++++++++++++++++++++++++++++++++
docker/src/main/Dockerfile | 46 ++++++++++++-----------------
test/e2e/base/docker-compose.yaml | 51 ++++++++++++++++++++++++++++++++
test/e2e/cases/README.md | 53 ++++++++++++++++++++++++++++++++++
test/e2e/cases/cli/README.md | 24 +++++++++++++++
test/e2e/cases/cli/cleanup.sh | 27 +++++++++++++++++
test/e2e/cases/cli/docker-compose.yaml | 41 ++++++++++++++++++++++++++
test/e2e/cases/cli/res/init.sql | 26 +++++++++++++++++
test/e2e/cases/cli/run.sh | 48 ++++++++++++++++++++++++++++++
10 files changed, 361 insertions(+), 28 deletions(-)
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..995e35a
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+*
+!distribution
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
new file mode 100644
index 0000000..59f332d
--- /dev/null
+++ b/.github/workflows/e2e.yml
@@ -0,0 +1,52 @@
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+name: E2E Tests
+
+on:
+ push:
+ branches:
+ - master
+ - 'rel/*'
+ - test/e2e
+ paths-ignore:
+ - 'docs/**'
+ pull_request:
+ branches:
+ - master
+ - 'rel/*'
+ - cluster_new
+ paths-ignore:
+ - 'docs/**'
+
+jobs:
+ E2E:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: true
+ matrix:
+ case:
+ - cli
+ steps:
+ - uses: actions/checkout@v2
+
+ - uses: actions/cache@v2
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2
+
+ - name: Build Distribution Zip
+ run: ./mvnw.sh -B -DskipTests clean package
+
+ - name: Build Docker Image
+ run: |
+ docker build . -f docker/src/main/Dockerfile -t "iotdb:$GITHUB_SHA"
+ docker images
+
+ - name: Run Test Case ${{ matrix.case }}
+ run: bash test/e2e/cases/${{ matrix.case }}/run.sh
+
+ - name: Clean Up
+ if: ${{ always() }}
+ run: bash test/e2e/cases/${{ matrix.case }}/cleanup.sh
diff --git a/docker/src/main/Dockerfile b/docker/src/main/Dockerfile
index 44d5361..d8ced28 100644
--- a/docker/src/main/Dockerfile
+++ b/docker/src/main/Dockerfile
@@ -17,37 +17,27 @@
# under the License.
#
-# this docker file shows how to build an IoTDB image from IoTDB source codes.
-FROM ubuntu:18.04
+# docker build context is the root path of the repository
+
+FROM openjdk:11-jre-slim
+
+ADD distribution/target/apache-iotdb-*-bin.zip /
+
RUN apt update \
- && apt install wget unzip lsof maven thrift-compiler=0.9.1-2.1 -y \
- && wget https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz -O jdk11.tar.gz \
- && tar -xzf jdk11.tar.gz \
- && rm -rf jdk11.tar.gz \
- && export JAVA_HOME=/jdk-11.0.2/ \
- && export PATH="$JAVA_HOME/bin:$PATH" \
- && wget https://github.com/apache/iotdb/archive/master.zip \
- && unzip master.zip \
- && rm master.zip \
- && cd iotdb-master \
- && mvn package -pl server,client -am -Papache-release -DskipTests -Dthrift.download-url="http://www.apache.org/licenses/LICENSE-2.0.txt" -Dthrift.exec.absolute.path="/usr/bin/thrift" \
- && cd target/ \
- && unzip apache-iotdb-0.12.0-SNAPSHOT-bin.zip \
- && mkdir /iotdb \
- && mv apache-iotdb-0.12.0-SNAPSHOT/* /iotdb/ \
- && cd ../../ \
- && mvn clean \
- && ls -lh ~/.m2 \
- && rm -rf ~/.m2 \
- && rm -rf /iotdb-master \
- && sed -i '119d' /iotdb/conf/logback.xml \
- && apt remove wget maven unzip thrift-compiler -y \
+ && apt install lsof procps unzip -y \
+ && unzip /apache-iotdb-*-bin.zip -d / \
+ && rm /apache-iotdb-*-bin.zip \
+ && mv /apache-iotdb-* /iotdb \
+ && apt remove unzip -y \
&& apt autoremove -y \
&& apt purge --auto-remove -y \
- && apt clean -y
-ENV JAVA_HOME "/jdk-11.0.2"
-ENV PATH "$JAVA_HOME/bin:$PATH"
+ && apt clean -y
+
EXPOSE 6667
+EXPOSE 31999
+EXPOSE 5555
+EXPOSE 8181
VOLUME /iotdb/data
VOLUME /iotdb/logs
-#ENTRYPOINT ["/iotdb/bin/start-server.sh"]
+ENV PATH="/iotdb/sbin/:/iotdb/tools/:${PATH}"
+ENTRYPOINT ["/iotdb/sbin/start-server.sh"]
diff --git a/test/e2e/base/docker-compose.yaml b/test/e2e/base/docker-compose.yaml
new file mode 100644
index 0000000..d4970d4
--- /dev/null
+++ b/test/e2e/base/docker-compose.yaml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+version: '3.8'
+
+services:
+ server-prototype:
+ build:
+ context: ../../..
+ dockerfile: docker/src/main/Dockerfile
+ ports:
+ - 6667:6667
+ networks:
+ iotdb:
+ healthcheck:
+ test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6667" ]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+
+ initializer:
+ build:
+ context: ../../..
+ dockerfile: docker/src/main/Dockerfile
+ networks:
+ iotdb:
+ entrypoint:
+ - bash
+ - -c
+ - |
+ cat /res/init.sql | grep -v '^--' | xargs -I {} /iotdb/sbin/start-cli.sh -h server -e {}
+ echo "Ready to Run IoTDB E2E Tests"
+
+networks:
+ iotdb:
diff --git a/test/e2e/cases/README.md b/test/e2e/cases/README.md
new file mode 100644
index 0000000..c4a15e3
--- /dev/null
+++ b/test/e2e/cases/README.md
@@ -0,0 +1,53 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# IoTDB E2E tests cases
+
+Test cases are organized into sub-directories, each of which contains the following files:
+
+* `run.sh`: the entry of the test case.
+* `cleanup.sh`: a cleanup script to clean up resources that are created during the test.
+* `res`: resources files that will be mounted into the container(s) and be used there.
+* `docker-compose.yaml`: orchestrates the services used in the test process.
+* `README.md` (Optional): docs or notes when running this case manually.
+
+any other additional files are completely acceptable here, for example, when building
+a case to test the JDBC SDK, the files structure may be something like:
+
+```text
+.
+├── README.md
+├── cleanup.sh
+├── docker-compose.yaml
+├── app <------- Java application that uses JDBC SDK to communicate with IoTDB
+│ ├── pom.xml
+│ ├── src
+│ │ ├── main
+│ │ │ └── java
+│ │ └── test
+│ │ └── java
+│ └── src
+│ ├── main
+│ └── test
+├── res
+│ └── init.sql
+└── run.sh
+```
diff --git a/test/e2e/cases/cli/README.md b/test/e2e/cases/cli/README.md
new file mode 100644
index 0000000..343abeb
--- /dev/null
+++ b/test/e2e/cases/cli/README.md
@@ -0,0 +1,24 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# Standalone Server Test
+
+The simplest test case that starts up an IoTDB server and verifies that the CLI works.
diff --git a/test/e2e/cases/cli/cleanup.sh b/test/e2e/cases/cli/cleanup.sh
new file mode 100644
index 0000000..a96f4d0
--- /dev/null
+++ b/test/e2e/cases/cli/cleanup.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -x
+
+cd "$(dirname "$0")" || exit 1
+
+docker-compose down
+
+cd - || exit 1
diff --git a/test/e2e/cases/cli/docker-compose.yaml b/test/e2e/cases/cli/docker-compose.yaml
new file mode 100644
index 0000000..2e9da04
--- /dev/null
+++ b/test/e2e/cases/cli/docker-compose.yaml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+version: '3.8'
+
+services:
+ server:
+ extends:
+ file: ../../base/docker-compose.yaml
+ service: server-prototype
+ volumes:
+ - ./res:/resources
+
+ initializer:
+ extends:
+ service: initializer
+ file: ../../base/docker-compose.yaml
+ volumes:
+ - ./res:/res:ro
+ depends_on:
+ server:
+ condition: service_healthy
+
+networks:
+ iotdb:
diff --git a/test/e2e/cases/cli/res/init.sql b/test/e2e/cases/cli/res/init.sql
new file mode 100644
index 0000000..cb9f6aa
--- /dev/null
+++ b/test/e2e/cases/cli/res/init.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+SET STORAGE GROUP TO root.ln;
+SHOW STORAGE GROUP;
+
+CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=PLAIN;
+
+INSERT INTO root.ln.wf01.wt01(timestamp,temperature) values(100, 16);
+INSERT INTO root.ln.wf01.wt01(timestamp,temperature) values(200, 26);
diff --git a/test/e2e/cases/cli/run.sh b/test/e2e/cases/cli/run.sh
new file mode 100644
index 0000000..a62bba7
--- /dev/null
+++ b/test/e2e/cases/cli/run.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+cd "$(dirname "$0")"
+
+docker-compose up -d
+
+max_attempts=10
+attempts=1
+
+while ! docker-compose logs | grep -c 'Ready to Run IoTDB E2E Tests' > /dev/null 2>&1; do
+ if [[ $attempts -gt $max_attempts ]]; then
+ echo "Preparation is not ready after $max_attempts attempts, will exit now"
+ exit 1
+ fi
+ echo "Preparation is not ready yet, retrying ($attempts/$max_attempts)"
+ sleep 3
+ attempts=$((attempts+1))
+done
+
+results=$(docker-compose exec -T server /iotdb/sbin/start-cli.sh -e 'SELECT temperature FROM root.ln.wf01.wt01')
+
+if [[ $results != *"Total line number = 2"* ]]; then
+ echo "Total line number should be 2"
+ echo "$results"
+ exit 1
+fi
+
+cd -