You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ly...@apache.org on 2022/05/31 18:01:17 UTC
[tinkerpop] branch 3.5-dev updated: Gremlin-Go: Updated Docker Testing Set Up (#1665)
This is an automated email from the ASF dual-hosted git repository.
lyndonb pushed a commit to branch 3.5-dev
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
The following commit(s) were added to refs/heads/3.5-dev by this push:
new 6b0169e37b Gremlin-Go: Updated Docker Testing Set Up (#1665)
6b0169e37b is described below
commit 6b0169e37ba0161f5d03df24fd81d8b4a214ec88
Author: Yang Xia <55...@users.noreply.github.com>
AuthorDate: Tue May 31 11:01:10 2022 -0700
Gremlin-Go: Updated Docker Testing Set Up (#1665)
* modified go docker set up to dynamically obtain server version & added a server connection check before integration tests
* added a max retry field to wait for script to impose limit if needed.
---
.github/workflows/build-test.yml | 5 +-
.gitignore | 1 +
gremlin-go/Dockerfile | 18 ++-----
gremlin-go/README.md | 23 ++++++++-
gremlin-go/docker-compose.yml | 14 ++---
gremlin-go/docker/docker-entrypoint.sh | 2 +
.../docker/gremlin-server-integration-secure.yaml | 15 +++---
gremlin-go/docker/gremlin-server-integration.yaml | 17 +++---
gremlin-go/run.sh | 50 ++++++++++++++++++
gremlin-go/wait-for-server.sh | 60 ++++++++++++++++++++++
10 files changed, 166 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 5befe4519f..afec8fe1df 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -224,8 +224,9 @@ jobs:
- name: Execute Go tests
working-directory: ./gremlin-go
run: |
- docker-compose up --exit-code-from gremlin-go-integration-tests
- docker-compose down
+ chmod +x ./run.sh
+ ./run.sh
+ shell: bash
- name: Upload to Codecov
uses: codecov/codecov-action@v2
diff --git a/.gitignore b/.gitignore
index fdcb934b92..468638e15a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,3 +42,4 @@ BenchmarkDotNet.Artifacts/
docs/gremlint/
gremlint/
coverage.out
+.env
diff --git a/gremlin-go/Dockerfile b/gremlin-go/Dockerfile
index d010366dc8..1549b7c216 100644
--- a/gremlin-go/Dockerfile
+++ b/gremlin-go/Dockerfile
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-ARG GREMLIN_SERVER_VERSION
-FROM tinkerpop/gremlin-server:${GREMLIN_SERVER_VERSION:-3.5.4-SNAPSHOT}
+ARG GREMLIN_SERVER
+FROM tinkerpop/gremlin-server:$GREMLIN_SERVER
USER root
RUN mkdir -p /opt
@@ -31,18 +31,8 @@ RUN chmod 755 /opt/docker-entrypoint.sh
RUN apk update && apk add dos2unix
RUN dos2unix /opt/docker-entrypoint.sh && apk del dos2unix
-ARG NEO4J_VERSION
-# Installs Neo4j libraries to this image so that we can test variants with transactions,
-# but only only port 45940 is configured with the neo4j graph as the neo4j-empty.properties
-# is statically pointing at a temp directory and that space can only be accessed by one
-# graph at a time.
-RUN /opt/gremlin-server/bin/gremlin-server.sh install org.apache.tinkerpop neo4j-gremlin ${NEO4J_VERSION:-3.5.4}
-
-# Gremlin server and neo4j versions are set to 3.5.4 by default, to change their versions, add args under the
-# docker-compose build step, e.g.:
-# args:
-# GREMLIN_SERVER_VERSION: 3.5.x-SNAPSHOT
-# NEO4J_VERSION: 3.5.x
+ARG GREMLIN_SERVER
+ENV GREMLIN_SERVER_VER=$GREMLIN_SERVER
EXPOSE 45940 45941
diff --git a/gremlin-go/README.md b/gremlin-go/README.md
index 99acef8d55..229e72e874 100644
--- a/gremlin-go/README.md
+++ b/gremlin-go/README.md
@@ -223,9 +223,24 @@ Please review the [staticcheck documentation][scheck docs] for more details on i
## Testing with Docker
-Docker allows you to test the driver without installing any dependencies. The following command can be used to run docker:
+Docker allows you to test the driver without installing any dependencies. Please make sure Docker is installed and running on your system.
+You will need to install both [Docker Engine][dengine] and [Docker Compose][dcompose], which are included in [Docker Desktop][ddesktop].
-`docker-compose up --exit-code-from gremlin-go-integration-tests`
+The docker compose environment variable `GREMLIN_SERVER` specifies the Gremlin server docker image to use, i.e. an image with the tag
+`tinkerpop/gremlin-server:$GREMLIN_SERVER`, and is a required environment variable. This also requires the specified docker image to exist,
+either locally or in [Docker Hub][dhub].
+
+If your OS Platform cannot build a local SNAPSHOT Gremlin server through `maven`, it is recommended to use the latest released server version
+from [Docker Hub][dhub] (do not use `GREMLIN_SERVER=latest`, use actual version number, e.g. `GREMLIN_SERVER=3.5.x` or `GREMLIN_SERVER=3.6.x`).
+
+There are 4 ways to launch the test suite and set the `GREMLIN_SERVER` environment variable depending on your Platform:
+- Execute tests via the `run.sh` script, which sets `GREMLIN_SERVER` by default. Run `./run.sh -h` for usage information (Unix/Linux - recommended).
+- Add `GREMLIN_SERVER=<server-image-version>` to an `.env` file inside `gremlin-go` and run `docker-compose up --exit-code-from gremlin-go-integration-tests` (Platform-agnostic).
+- Run `GREMLIN_SERVER=<server-image-version> docker-compose up --exit-code-from gremlin-go-integration-tests` in Unix/Linux.
+- Run `$env:GREMLIN_SERVER="<server-image-version>";docker-compose up --exit-code-from gremlin-go-integration-tests` in Windows PowerShell.
+
+You should see exit code 0 upon successful completion of the test suites. Run `docker-compose down` to remove the service containers (not needed
+if you executed `run.sh`), or `docker-compose down --rmi all` to remove the service containers while deleting all used images.
# Go Gremlin Language Variant
@@ -376,3 +391,7 @@ The `Go` driver supports all of the core GraphBinary data types.
[gofmt]: https://pkg.go.dev/cmd/gofmt
[goland]: https://www.jetbrains.com/go/
[fmtsave]: https://www.jetbrains.com/help/go/reformat-and-rearrange-code.html#reformat-on-save
+[ddesktop]:https://docs.docker.com/desktop/
+[dengine]:https://docs.docker.com/engine/install/
+[dcompose]:https://docs.docker.com/compose/install/
+[dhub]:https://hub.docker.com/r/tinkerpop/gremlin-server
diff --git a/gremlin-go/docker-compose.yml b/gremlin-go/docker-compose.yml
index da379abb0e..d8964b1556 100644
--- a/gremlin-go/docker-compose.yml
+++ b/gremlin-go/docker-compose.yml
@@ -15,20 +15,22 @@
# specific language governing permissions and limitations
# under the License.
-version: "3.4"
services:
gremlin-test-server:
container_name: gremlin-test-server
- image: tinkerpop:gremlin-test-server
+ image: tinkerpop:gremlin-test-server-${GREMLIN_SERVER}
build:
context: ../
dockerfile: gremlin-go/Dockerfile
args:
- NEO4J_VERSION: 3.5.3
+ - GREMLIN_SERVER=${GREMLIN_SERVER}
ports:
- "45940:45940"
- "45941:45941"
+ volumes:
+ - ${HOME}/.groovy:/root/.groovy
+ - ${HOME}/.m2:/root/.m2
gremlin-go-integration-tests:
container_name: gremlin-go-integration-tests
@@ -45,8 +47,8 @@ services:
- RUN_BASIC_AUTH_INTEGRATION_TESTS=true
- TEST_TRANSACTIONS=true
working_dir: /go_app
- command: >
- bash -c "go install github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
- && go test -v -json ./... -race -covermode=atomic -coverprofile=\"coverage.out\" -coverpkg=./... | gotestfmt"
+ command: ["./wait-for-server.sh", gremlin-test-server, "45940", "90", "bash", "-c",
+ "go install github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
+ && go test -v -json ./... -race -covermode=atomic -coverprofile=\"coverage.out\" -coverpkg=./... | gotestfmt"]
depends_on:
- gremlin-test-server
diff --git a/gremlin-go/docker/docker-entrypoint.sh b/gremlin-go/docker/docker-entrypoint.sh
index c4efb650dd..87b3d16d62 100644
--- a/gremlin-go/docker/docker-entrypoint.sh
+++ b/gremlin-go/docker/docker-entrypoint.sh
@@ -36,6 +36,8 @@ cp *.yaml ${TINKERPOP_HOME}/conf/
java -version
+/opt/gremlin-server/bin/gremlin-server.sh install org.apache.tinkerpop neo4j-gremlin "$GREMLIN_SERVER_VER"
+
/opt/gremlin-server/bin/gremlin-server.sh conf/gremlin-server-integration.yaml &
/opt/gremlin-server/bin/gremlin-server.sh conf/gremlin-server-integration-secure.yaml
diff --git a/gremlin-go/docker/gremlin-server-integration-secure.yaml b/gremlin-go/docker/gremlin-server-integration-secure.yaml
index d91b546448..8a86330826 100644
--- a/gremlin-go/docker/gremlin-server-integration-secure.yaml
+++ b/gremlin-go/docker/gremlin-server-integration-secure.yaml
@@ -18,14 +18,15 @@
host: 0.0.0.0
port: 45941
evaluationTimeout: 30000
+channelizer: org.apache.tinkerpop.gremlin.server.channel.UnifiedChannelizer
graphs: {
- graph: conf/tinkergraph-empty.properties,
- immutable: conf/tinkergraph-empty.properties,
- classic: conf/tinkergraph-empty.properties,
- modern: conf/tinkergraph-empty.properties,
- crew: conf/tinkergraph-empty.properties,
- grateful: conf/tinkergraph-empty.properties,
- sink: conf/tinkergraph-empty.properties}
+ graph: scripts/tinkergraph-empty.properties,
+ immutable: scripts/tinkergraph-empty.properties,
+ classic: scripts/tinkergraph-empty.properties,
+ modern: scripts/tinkergraph-empty.properties,
+ crew: scripts/tinkergraph-empty.properties,
+ grateful: scripts/tinkergraph-empty.properties,
+ sink: scripts/tinkergraph-empty.properties}
scriptEngines: {
gremlin-lang : {},
gremlin-groovy: {
diff --git a/gremlin-go/docker/gremlin-server-integration.yaml b/gremlin-go/docker/gremlin-server-integration.yaml
index 4318869a86..ec56c74be1 100644
--- a/gremlin-go/docker/gremlin-server-integration.yaml
+++ b/gremlin-go/docker/gremlin-server-integration.yaml
@@ -18,15 +18,16 @@
host: 0.0.0.0
port: 45940
evaluationTimeout: 30000
+channelizer: org.apache.tinkerpop.gremlin.server.channel.UnifiedChannelizer
graphs: {
- graph: conf/tinkergraph-empty.properties,
- immutable: conf/tinkergraph-empty.properties,
- classic: conf/tinkergraph-empty.properties,
- modern: conf/tinkergraph-empty.properties,
- crew: conf/tinkergraph-empty.properties,
- grateful: conf/tinkergraph-empty.properties,
- sink: conf/tinkergraph-empty.properties,
- tx: conf/neo4j-empty.properties
+ graph: scripts/tinkergraph-empty.properties,
+ immutable: scripts/tinkergraph-empty.properties,
+ classic: scripts/tinkergraph-empty.properties,
+ modern: scripts/tinkergraph-empty.properties,
+ crew: scripts/tinkergraph-empty.properties,
+ grateful: scripts/tinkergraph-empty.properties,
+ sink: scripts/tinkergraph-empty.properties,
+ tx: scripts/neo4j-empty.properties
}
scriptEngines: {
gremlin-lang : {},
diff --git a/gremlin-go/run.sh b/gremlin-go/run.sh
new file mode 100755
index 0000000000..3e482a8b72
--- /dev/null
+++ b/gremlin-go/run.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+function usage {
+ echo -e "\nUsage: $(basename "$0") <server-version> [OPTIONS]" \
+ "\n\nRun the Gremlin-Go test suite in Docker." \
+ "\n\nOptions:" \
+ "\n\t<server-version> \t Optional value, if unspecified the test suite will run with the current version
+ \t\t\t of Gremlin server and Gremlin Neo4j, as specified in the TinkerPop pom.xml file." \
+ "\n\t-h, --help \t\t Show this message." \
+ "\n\nExamples:" \
+ "\n\tRunning the default: ./run.sh" \
+ "\n\tThe default requires a SNAPSHOT server image to be built using:
+ mvn clean install -pl :gremlin-server -DskipTests -DskipIntegrationTests=true -am && mvn install -Pdocker-images -pl :gremlin-server" \
+ "\n\tRunning a prebuilt local SNAPSHOT build: ./run.sh 3.x.x-SNAPSHOT" \
+ "\n\tRunning a released version: ./run.sh 3.5.3" \
+ "\n"
+}
+
+if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
+ usage
+ exit 0
+fi
+
+# Parses current gremlin server version from project pom.xml file using perl regex
+GREMLIN_SERVER_VERSION=$(grep tinkerpop -A2 pom.xml | grep -Po '(?<=<version>)([0-9]+\.?){3}(-SNAPSHOT)?(?=<)')
+GREMLIN_SERVER="${1:-$GREMLIN_SERVER_VERSION}"
+echo "$GREMLIN_SERVER"
+
+# Passes current gremlin server version into docker compose as environment variable
+GREMLIN_SERVER="$GREMLIN_SERVER_VERSION" docker-compose up --build --exit-code-from gremlin-go-integration-tests
+# Removes all service containers
+docker-compose down
diff --git a/gremlin-go/wait-for-server.sh b/gremlin-go/wait-for-server.sh
new file mode 100755
index 0000000000..130f096c82
--- /dev/null
+++ b/gremlin-go/wait-for-server.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+function usage {
+ echo -e "\nUsage: $(basename "$0") <host> <port> <max retry> [OPTIONS]" \
+ "\n\nCurls a Gremlin Server HTTP endpoint until success to signal that the server is ready, or until max number of retries reached to signal failure." \
+ "\n\nOptions:" \
+ "\n\t<host> \t\t Gremlin server host." \
+ "\n\t<host> \t\t Gremlin server port." \
+ "\n\t<max retry> \t Number of times to retry connection, entering 0 or less means no retry limit." \
+ "\n\t-h, --help \t Show this message." \
+ "\n"
+}
+
+if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
+ usage
+ exit 0
+fi
+
+host="$1"; shift
+port="$1"; shift
+max_retries="$1"; shift
+
+# Setting max retry to 0 or less means no retry limit.
+if [ "$max_retries" -le 0 ] ; then
+ max_retries=-1
+fi
+
+counter=0
+until curl "$host:$port?gremlin=100-1"; do
+ if [ "$counter" -eq "$max_retries" ] ; then
+ echo "Gremlin server is unavailable - exiting: max retries $max_retries reached, please check server setup or increase max retry value"
+ exit 1
+ fi
+ >&2 echo "Gremlin server is unavailable - sleeping"
+ sleep 1
+ (( counter++ )) || true
+done
+
+>&2 echo "Gremlin Server is up - executing command"
+exec "$@"