You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by xi...@apache.org on 2022/09/19 21:51:16 UTC
[tinkerpop] branch 3.5-dev updated: Update docker/build.sh to work with GLV docker-compose changes.
This is an automated email from the ASF dual-hosted git repository.
xiazcy pushed a commit to branch 3.5-dev
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
The following commit(s) were added to refs/heads/3.5-dev by this push:
new 0dc2522284 Update docker/build.sh to work with GLV docker-compose changes.
new 933a01009c Merge pull request #1800 from Bit-Quill/ken/docker-build-sh
0dc2522284 is described below
commit 0dc2522284584083680956cd5ab4480bd93468ec
Author: Ken Hu <10...@users.noreply.github.com>
AuthorDate: Thu Sep 1 17:42:39 2022 -0700
Update docker/build.sh to work with GLV docker-compose changes.
The non-Java GLVs have changed to use docker-compose for integration
testing which creates a Docker in Docker issue when run through the
build.sh script. To prevent this issue, the GLVs are split away and run
individually using their docker-compose files. However, since the
docker-compose files require some artifacts from the build (including
the local Maven repository items), those directories are bind mounted so
that the files are available for the docker-compose builds.
To increase the usability of the script for developers who only work on
a portion of TinkerPop, options were added to include only a subset of
the modules to the build.
Change version parsing to use awk instead of grep with Perl regex
because it isn't installed by default on macOS.
---
docker/build.sh | 133 +++++++++++++++++++--
docker/scripts/build.sh | 59 +++++----
.../dev/developer/development-environment.asciidoc | 15 ++-
gremlin-go/run.sh | 2 +-
4 files changed, 173 insertions(+), 36 deletions(-)
diff --git a/docker/build.sh b/docker/build.sh
index 3232ce5b5c..1b299d5c34 100755
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -28,17 +28,72 @@ function cleanup {
BUILD_IMAGE=$(docker images tinkerpop | awk "{if (\$2 == \"${BUILD_TAG}\") print \$3}")
[ ! -z ${BUILD_IMAGE} ] && docker rmi ${BUILD_IMAGE}
rm -f ${PROJECT_HOME}/Dockerfile
+ docker image prune --filter "label=maintainer=dev@tinkerpop.apache.org" -f > /dev/null
}
trap cleanup EXIT
REMOVE_CONTAINER="--rm"
[[ -n ${KEEP_CONTAINER} ]] && unset REMOVE_CONTAINER
-case "$1" in
- -h | --help ) ${DIR}/scripts/build.sh -h; exit 0 ;;
-esac
+function usage {
+ echo -e "\nUsage: `basename $0` [OPTIONS]" \
+ "\nBuild the current local TinkerPop project in a Docker container." \
+ "\nBy default all modules are run. If you prefer to run a specific module then please" \
+ "\nenter it as an option. Selecting specific modules changes the build to include only" \
+ "\nthose selected modules as well as gremlin-server, gremlin-test and neo4j-gremlin (and" \
+ "\ntheir dependencies)." \
+ "\n\nOptions are:\n" \
+ "\n\t-t, --tests run standard test suite" \
+ "\n\t-i, --integration-tests run integration tests" \
+ "\n\t-n, --neo4j include Neo4j" \
+ "\n\t-go, --golang change to minimal build and add gremlin-go to build" \
+ "\n\t-py, --python change to minimal build and add gremlin-python to build" \
+ "\n\t-dn, --dotnet change to minimal build and add gremlin-dotnet to build" \
+ "\n\t-js, --javascript change to minimal build and add gremlin-javascript to build" \
+ "\n\t-c, --console change to minimal build and add gremlin-console to build" \
+ "\n\t-j, --java-docs build Java docs" \
+ "\n\t-d, --docs build user docs" \
+ "\n\t-h, --help show this message" \
+ "\n"
+}
+
+ARGS=""
+RUN_TESTS=""
+RUN_INTEGRATION_TESTS=""
+INCLUDE_GO=""
+INCLUDE_PYTHON=""
+INCLUDE_DOTNET=""
+INCLUDE_JAVASCRIPT=""
+INCLUDE_CONSOLE=""
+while [ ! -z "$1" ]; do
+ case "$1" in
+ -t | --tests ) ARGS="${ARGS} -t"; RUN_TESTS=true; shift ;;
+ -i | --integration-tests ) ARGS="${ARGS} -i"; RUN_INTEGRATION_TESTS=true; shift ;;
+ -n | --neo4j ) ARGS="${ARGS} -n"; shift ;;
+ -go | --golang ) ARGS="${ARGS} -go"; INCLUDE_GO=true; shift ;;
+ -py | --python ) ARGS="${ARGS} -py"; INCLUDE_PYTHON=true; shift ;;
+ -dn | --dotnet ) ARGS="${ARGS} -dn"; INCLUDE_DOTNET=true; shift ;;
+ -js | --javascript ) ARGS="${ARGS} -js"; INCLUDE_JAVASCRIPT=true; shift ;;
+ -c | --console ) ARGS="${ARGS} -c"; INCLUDE_CONSOLE=true; shift ;;
+ -j | --java-docs ) ARGS="${ARGS} -j"; shift ;;
+ -d | --docs ) ARGS="${ARGS} -d"; shift ;;
+ -h | --help ) usage; exit 0 ;;
+ *) usage 1>&2; exit 1 ;;
+ esac
+done
+
+# The default is for every module to be included.
+if [[ -z ${INCLUDE_GO} && -z ${INCLUDE_PYTHON} && -z ${INCLUDE_DOTNET} && -z ${INCLUDE_JAVASCRIPT} && -z ${INCLUDE_CONSOLE} ]]; then
+ INCLUDE_GO=true
+ INCLUDE_PYTHON=true
+ INCLUDE_DOTNET=true
+ INCLUDE_JAVASCRIPT=true
+ INCLUDE_CONSOLE=true
+fi
pushd ${PROJECT_HOME} > /dev/null
+export ABS_PROJECT_HOME=$(pwd) # absolute path required by some Docker variables.
+echo "ABS_PROJECT_HOME ${ABS_PROJECT_HOME}"
HADOOP_VERSION=$(cat pom.xml | grep -o '<hadoop.version>[^<]*' | head -n1 | cut -d '>' -f2)
@@ -46,12 +101,74 @@ docker/build-containers.sh -h "${HADOOP_VERSION}"
sed -e "s/HADOOP_VERSION\$/${HADOOP_VERSION}/" docker/build/Dockerfile.template > Dockerfile
cat >> Dockerfile <<EOF
-CMD ["sh", "-c", "docker/scripts/build.sh $@"]
+CMD ["sh", "-c", "docker/scripts/build.sh ${ARGS}"]
EOF
+function check_status {
+ status=$?
+ [ "$1" == "down" ] && docker-compose down
+ popd > /dev/null
+ [ $status -ne 0 ] && exit $status
+}
+
+# GREMLIN_SERVER is the project version e.g. 3.5.5-SNAPSHOT
+export GREMLIN_SERVER=$(grep tinkerpop -A2 pom.xml | sed -r -n 's/.*<version>(([0-9]+\.?){3})(-SNAPSHOT)?<\/version>/\1\3/p')
+echo "GREMLIN_SERVER ${GREMLIN_SERVER}"
+
docker build -t tinkerpop:${BUILD_TAG} .
-docker run -p 80:80 ${TINKERPOP_DOCKER_OPTS} ${REMOVE_CONTAINER} -e "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" -ti tinkerpop:${BUILD_TAG}
+docker run -p 80:80 ${TINKERPOP_DOCKER_OPTS} ${REMOVE_CONTAINER} \
+ -e "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" \
+ -ti \
+ --mount type=bind,src=${HOME}/.m2/,dst=/root/.m2/ \
+ --mount type=bind,src=$(pwd)/gremlin-server/,dst=/usr/src/tinkerpop/gremlin-server/ \
+ --mount type=bind,src=$(pwd)/gremlin-test/,dst=/usr/src/tinkerpop/gremlin-test/ \
+ --mount type=bind,src=$(pwd)/gremlin-console/,dst=/usr/src/tinkerpop/gremlin-console/ \
+ --mount type=bind,src=$(pwd)/neo4j-gremlin/,dst=/usr/src/tinkerpop/neo4j-gremlin/ \
+ tinkerpop:${BUILD_TAG}
+check_status
+
+if [ -n "${RUN_TESTS}" ]; then
+ # If testing, then build base server which is required by the following docker-compose.
+ pushd ${ABS_PROJECT_HOME}/gremlin-server > /dev/null
+ docker build -f ./Dockerfile --build-arg GREMLIN_SERVER_DIR=target/apache-tinkerpop-gremlin-server-${GREMLIN_SERVER}-standalone -t tinkerpop/gremlin-server:${GREMLIN_SERVER} .
+ check_status
+fi
+
+if [ -n "${INCLUDE_GO}" ] && [ -n "${RUN_TESTS}" ]; then
+ pushd ${ABS_PROJECT_HOME}/gremlin-go > /dev/null
+ docker-compose up --build --exit-code-from gremlin-go-integration-tests
+ check_status "down"
+fi
+
+if [ -n "${INCLUDE_PYTHON}" ] && [ -n "${RUN_TESTS}" ]; then
+ pushd ${ABS_PROJECT_HOME}/gremlin-python > /dev/null
+ export BUILD_DIR=$(pwd)/target/python3/
+ mkdir -p ${BUILD_DIR}
+ cp -r ./src/main/python/* ${BUILD_DIR}
+ docker-compose up --build --abort-on-container-exit gremlin-server-test-python gremlin-python-integration-tests
+ check_status "down"
+fi
+
+if [ -n "${INCLUDE_DOTNET}" ] && [ -n "${RUN_TESTS}" ]; then
+ pushd ${ABS_PROJECT_HOME}/gremlin-dotnet/test > /dev/null
+ docker-compose up --build --exit-code-from gremlin-dotnet-integration-tests
+ check_status "down"
+fi
+
+if [ -n "${INCLUDE_JAVASCRIPT}" ] && [ -n "${RUN_TESTS}" ]; then
+ pushd ${ABS_PROJECT_HOME}/gremlin-javascript/src/main/javascript/gremlin-javascript > /dev/null
+ docker-compose up --build --exit-code-from gremlin-js-integration-tests
+ check_status "down"
+fi
+
+if [ -n "${INCLUDE_CONSOLE}" ] && [ -n "${RUN_INTEGRATION_TESTS}" ]; then
+ pushd ${ABS_PROJECT_HOME}/gremlin-console > /dev/null
+ docker build -t gremlin-console-test:py3.8jre11 ./src/test/python/docker
+ docker run --rm \
+ --mount type=bind,src=$(pwd)/src/test/python,dst=/console_app \
+ --mount type=bind,src=$(pwd)/target/apache-tinkerpop-gremlin-console-${GREMLIN_SERVER}-standalone,dst=/console_app/gremlin-console \
+ gremlin-console-test:py3.8jre11
+ check_status
+fi
-status=$?
-popd > /dev/null
-exit $status
+exit 0
diff --git a/docker/scripts/build.sh b/docker/scripts/build.sh
index ee32fe70ec..37f948b733 100755
--- a/docker/scripts/build.sh
+++ b/docker/scripts/build.sh
@@ -23,39 +23,51 @@ RUN_INTEGRATION_TESTS=
INCLUDE_NEO4J=
BUILD_JAVA_DOCS=
BUILD_USER_DOCS=
-
-function usage {
- echo -e "\nUsage: `basename $0` [OPTIONS]" \
- "\nBuild the current local TinkerPop project in a Docker container." \
- "\n\nOptions are:\n" \
- "\n\t-t, --tests run standard test suite" \
- "\n\t-i, --integration-tests run integration tests" \
- "\n\t-n, --neo4j include Neo4j" \
- "\n\t-j, --java-docs build Java docs" \
- "\n\t-d, --docs build user docs" \
- "\n\t-h, --help show this message" \
- "\n"
-}
+INCLUDE_GO=
+INCLUDE_PYTHON=
+INCLUDE_DOTNET=
+INCLUDE_JAVASCRIPT=
+INCLUDE_CONSOLE=
while [ ! -z "$1" ]; do
case "$1" in
- -t | --tests ) RUN_TESTS=true; shift ;;
- -i | --integration-tests ) RUN_INTEGRATION_TESTS=true; shift ;;
- -n | --neo4j ) INCLUDE_NEO4J=true; shift ;;
- -j | --java-docs ) BUILD_JAVA_DOCS=true; shift ;;
- -d | --docs ) BUILD_USER_DOCS=true; shift ;;
- -h | --help ) usage; exit 0 ;;
- *) usage 1>&2; exit 1 ;;
+ -t | --tests ) RUN_TESTS=true; shift ;;
+ -i | --integration-tests ) RUN_INTEGRATION_TESTS=true; shift ;;
+ -n | --neo4j ) INCLUDE_NEO4J=true; shift ;;
+ -j | --java-docs ) BUILD_JAVA_DOCS=true; shift ;;
+ -d | --docs ) BUILD_USER_DOCS=true; shift ;;
+ -go | --golang ) INCLUDE_GO=true; shift ;;
+ -py | --python ) INCLUDE_PYTHON=true; shift ;;
+ -dn | --dotnet ) INCLUDE_DOTNET=true; shift ;;
+ -js | --javascript ) INCLUDE_JAVASCRIPT=true; shift ;;
+ -c | --console ) INCLUDE_CONSOLE=true; shift ;;
esac
done
-TINKERPOP_BUILD_OPTIONS=""
+# To prevent Docker in Docker, skip building images and deactivate Docker profiles.
+TINKERPOP_BUILD_OPTIONS="-DskipImageBuild -P -glv-js,-glv-go,-glv-python"
[ -z "${RUN_TESTS}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -DskipTests"
[ -z "${RUN_INTEGRATION_TESTS}" ] || TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -DskipIntegrationTests=false"
[ -z "${INCLUDE_NEO4J}" ] || TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -DincludeNeo4j"
[ -z "${BUILD_JAVA_DOCS}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -Dmaven.javadoc.skip=true"
+# If any of these GLVs are selected then create a minimal build to reduce build/test times
+# as the user is likely not interested in the other modules.
+if [[ -n ${INCLUDE_GO} || -n ${INCLUDE_PYTHON} || -n ${INCLUDE_DOTNET} || -n ${INCLUDE_JAVASCRIPT} || -n ${INCLUDE_CONSOLE} ]]; then
+ # gremlin-server, gremlin-test and neo4j-gremlin are minimal dependencies needed to start a server for GLV testing.
+ TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -am -pl gremlin-server,gremlin-test,neo4j-gremlin"
+
+ [ -n "${INCLUDE_GO}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-go"
+ [ -n "${INCLUDE_PYTHON}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-python"
+ [ -n "${INCLUDE_DOTNET}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-dotnet,:gremlin-dotnet-source"
+ [ -n "${INCLUDE_JAVASCRIPT}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-javascript"
+ [ -n "${INCLUDE_CONSOLE}" ] && TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS},gremlin-console"
+else
+ # When building everything, deactivate dotnet tests to prevent Docker in Docker.
+ TINKERPOP_BUILD_OPTIONS="${TINKERPOP_BUILD_OPTIONS} -pl -:gremlin-dotnet-tests"
+fi
+
# If the tmpfs (in-memory filesystem exists, use it)
TINKERMEM_PATH=$(cd .. ; echo `pwd`/tinkermem)
if [ -d "${TINKERMEM_PATH}" ]; then
@@ -67,9 +79,7 @@ if [ -d "${TINKERMEM_PATH}" ]; then
cd ${TINKERMEM_PATH}
fi
-# omitting gremlin-go for now to avoid docker in docker as it builds via docker compose
-# progress to restructure and add gremlin-go back will be tracked here https://issues.apache.org/jira/browse/TINKERPOP-2737
-touch {gremlin-dotnet,gremlin-dotnet/src,gremlin-dotnet/test,gremlin-python,gremlin-javascript}/.glv
+touch gremlin-dotnet/src/.glv
# use a custom maven settings.xml
if [ -r "settings.xml" ]; then
@@ -80,6 +90,7 @@ fi
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
mvn clean install process-resources --batch-mode ${TINKERPOP_BUILD_OPTIONS} || exit 1
+
[ -z "${BUILD_JAVA_DOCS}" ] || mvn process-resources -Djavadoc || exit 1
if [ ! -z "${BUILD_USER_DOCS}" ]; then
diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc
index 3ac74dbd4d..6b7797647a 100644
--- a/docs/src/dev/developer/development-environment.asciidoc
+++ b/docs/src/dev/developer/development-environment.asciidoc
@@ -448,9 +448,6 @@ TinkerPop provides a shell script, that can start several build tasks within a D
required Docker images will be built automatically if they don't exist yet. Thus the first invocation
of the Docker script is expected to take some time.
-NOTE: Currently, `gremlin-go` integration is not supported within this Docker container, but can be run separately with Docker.
-Please see the `Testing With Docker` section under the xref:../../../../gremlin-go/driver/README.md[Gremlin Go Driver] for details.
-
The script can be found under `PROJECT_HOME/docker/build.sh`. The following tasks are currently
supported:
@@ -462,6 +459,18 @@ supported:
A list of command line options is provided by `docker/build.sh --help`. The container will install,
configure and start all required dependencies, such as Hadoop.
+By default, this script will run every module in the project. However, if you are planning on working on just a
+small set of the modules (e.g. the GLVs) then you can use the script options to reduce the modules included by
+specifically selecting which modules you want. This behavior is currently supported for the non-Java GLVs and
+gremlin-console. This option will include only the selected modules as well as gremlin-server, gremlin-test,
+neo4j-gremlin and all their dependencies. This is the minimum set of modules required to build and test the GLVs.
+
+[source,bash]
+.Build and test gremlin-python and gremlin-go with minimal Gremlin Server dependencies
+----
+./docker/build.sh --tests --integration-tests --python --golang
+----
+
Options can be passed to Docker by setting the `TINKERPOP_DOCKER_OPTS` environment variable. A speed boost can
be gained at the expense of memory by using tmpfs and the special directory `/usr/src/tinkermem`.
diff --git a/gremlin-go/run.sh b/gremlin-go/run.sh
index e37b768711..5f62072dda 100755
--- a/gremlin-go/run.sh
+++ b/gremlin-go/run.sh
@@ -40,7 +40,7 @@ if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
fi
# Parses current gremlin server version from project pom.xml file using perl regex
-GREMLIN_SERVER_VERSION=$(grep tinkerpop -A2 pom.xml | grep -Po '(?<=<version>)([0-9]+\.?){3}(-SNAPSHOT)?(?=<)')
+GREMLIN_SERVER_VERSION=$(grep tinkerpop -A2 pom.xml | sed -r -n 's/.*<version>(([0-9]+\.?){3})(-SNAPSHOT)?<\/version>/\1\3/p')
export GREMLIN_SERVER="${1:-$GREMLIN_SERVER_VERSION}"
echo "Running server version: $GREMLIN_SERVER"