You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2020/06/25 07:47:08 UTC
[hbase] branch master updated: HBASE-23339 release scripts should
use host gpg-agent when running in container.
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 497862b HBASE-23339 release scripts should use host gpg-agent when running in container.
497862b is described below
commit 497862bb472d938879a63853b7a881064f1093c5
Author: Sean Busbey <bu...@apache.org>
AuthorDate: Sat Apr 18 00:40:19 2020 -0500
HBASE-23339 release scripts should use host gpg-agent when running in container.
* put together a docker container that can use host gpg-agent forwarded over ssh.
* use gpg-agent forwarding container on OS X and directly forward the agent on Linux
* clean up the release container on exit
* use docker mounts instead of the deprecated volume syntax
* use image names within our project namespace
* update README to walk through running on GCE with gpg-agent forwarding
closes #1620
Signed-off-by: Matt Foley <ma...@apache.org>
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
dev-support/create-release/README.txt | 82 ++++++++++--
dev-support/create-release/do-release-docker.sh | 142 ++++++++++++++++++---
dev-support/create-release/do-release.sh | 52 ++++++--
dev-support/create-release/hbase-rm/Dockerfile | 15 ++-
.../create-release/mac-sshd-gpg-agent/Dockerfile | 100 +++++++++++++++
dev-support/create-release/release-build.sh | 10 +-
dev-support/create-release/release-util.sh | 53 ++++----
7 files changed, 374 insertions(+), 80 deletions(-)
diff --git a/dev-support/create-release/README.txt b/dev-support/create-release/README.txt
index 62703a1..4a457dd 100644
--- a/dev-support/create-release/README.txt
+++ b/dev-support/create-release/README.txt
@@ -17,13 +17,29 @@ anomalies are explained up in JIRA.
See http://hbase.apache.org/book.html#maven.release
+Regardless of where your release build will run (locally, locally in docker, on a remote machine,
+etc) you will need a local gpg-agent with access to your secret keys. A quick way to tell gpg
+to clear out state and start a gpg-agent is via the following command phrase:
+
+$ gpgconf --kill all && gpg-connect-agent /bye
+
+Before starting an RC build, make sure your local gpg-agent has configs
+to properly handle your credentials, especially if you want to avoid
+typing the passphrase to your secret key.
+
+e.g. if you are going to run and step away, best to increase the TTL
+on caching the unlocked secret via ~/.gnupg/gpg-agent.conf
+ # in seconds, e.g. a day
+ default-cache-ttl 86400
+ max-cache-ttl 86400
+
Running a build on GCE is easy enough. Here are some notes if of use.
Create an instance. 4CPU/15G/10G disk seems to work well enough.
Once up, run the below to make your machine fit for RC building:
-# Presuming debian-compatible OS
-$ sudo apt-get install -y git openjdk-8-jdk maven gnupg gnupg-agent
-# Install docker
+# Presuming debian-compatible OS, do these steps on the VM
+# your VM username should be your ASF id, because it will show up in build artifacts.
+# Follow the docker install guide: https://docs.docker.com/engine/install/debian/
$ sudo apt-get install -y \
apt-transport-https \
ca-certificates \
@@ -37,15 +53,55 @@ $ sudo add-apt-repository -y \
stable"
$ sudo apt-get update
$ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
-$ sudo usermod -a -G docker $USERID
+# Follow the post installation steps: https://docs.docker.com/engine/install/linux-postinstall/
+$ sudo usermod -aG docker $USER
# LOGOUT and then LOGIN again so $USERID shows as part of docker group
-# Copy up private key for $USERID export from laptop and import on gce.
-$ gpg --import stack.duboce.net.asc
-$ export GPG_TTY=$(tty) # https://github.com/keybase/keybase-issues/issues/2798
-$ eval $(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase --default-cache-ttl=86400 --max-cache-ttl=86400)
-$ export PROJECT="${PROJECT:-hbase}"
-$ git clone https://github.com/apache/${PROJECT}.git
-$ cd "${PROJECT}"
+# Test here by running docker's hello world as your build user
+$ docker run hello-world
+
+# Follow the GPG guide for forwarding your gpg-agent from your local machine to the VM
+# https://wiki.gnupg.org/AgentForwarding
+# On the VM find out the location of the gpg agent socket and extra socket
+$ gpgconf --list-dir agent-socket
+/run/user/1000/gnupg/S.gpg-agent
+$ gpgconf --list-dir agent-extra-socket
+/run/user/1000/gnupg/S.gpg-agent.extra
+# On the VM configure sshd to remove stale sockets
+$ sudo bash -c 'echo "StreamLocalBindUnlink yes" >> /etc/ssh/sshd_config'
+$ sudo systemctl restart ssh
+# logout of the VM
+
+# Do these steps on your local machine.
+# make sure gpg-agent is running
+$ gpg-connect-agent /bye
+# Export your public key and copy it to the VM.
+# Assuming 'example.gce.host' maps to your VM's external IP (or use the IP)
+$ gpg --export example@apache.org > ~/gpg.example.apache.pub
+$ scp ~/gpg.example.apache.pub example.gce.host:
+# ssh into the VM while forwarding the remote gpg socket locations found above to your local
+# gpg-agent's extra socket (this will restrict what commands the remote node is allowed to have
+# your agent handle. Note that the gpg guide above can help you set this up in your ssh config
+# rather than typing it in ssh like this every time.
+$ ssh -i ~/.ssh/my_id \
+ -R "/run/user/1000/gnupg/S.gpg-agent:$(gpgconf --list-dir agent-extra-socket)" \
+ -R "/run/user/1000/gnupg/S.gpg-agent.extra:$(gpgconf --list-dir agent-extra-socket)" \
+ example.gce.host
+
+# now in an SSH session on the VM with the socket forwarding
+# import your public key and test signing with the forwarding to your local agent.
+$ gpg --no-autostart --import gpg.example.apache.pub
+$ echo "foo" > foo.txt
+$ gpg --no-autostart --detach --armor --sign foo.txt
+$ gpg --no-autostart --verify foo.txt.asc
+
+# install git and clone the main project on the build machine
+$ sudo apt-get install -y git
+$ git clone https://github.com/apache/hbase.git
+# finally set up an output folder and launch a dry run.
$ mkdir ~/build
-$ ./dev-resources/create-release/do-release-docker.sh -d ~/build
-# etc.
+$ cd hbase
+$ ./dev-support/create-release/do-release-docker.sh -d ~/build
+
+# for building the main repo specifically you can save an extra download by pointing the build
+# to the local clone you just made
+$ ./dev-support/create-release/do-release-docker.sh -d ~/build -r .git
diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh
index fac89e3..e863cb3 100755
--- a/dev-support/create-release/do-release-docker.sh
+++ b/dev-support/create-release/do-release-docker.sh
@@ -76,6 +76,7 @@ Options:
-s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
If none specified, runs tag, then publish-dist, and then publish-release.
'publish-snapshot' is also an allowed, less used, option.
+ -x debug. do less clean up. (env file, gpg forwarding on mac)
EOF
exit 1
}
@@ -85,7 +86,7 @@ IMGTAG=latest
JAVA=
RELEASE_STEP=
GIT_REPO=
-while getopts "d:fhj:p:r:s:t:" opt; do
+while getopts "d:fhj:p:r:s:t:x" opt; do
case $opt in
d) WORKDIR="$OPTARG" ;;
f) DRY_RUN=0 ;;
@@ -94,6 +95,7 @@ while getopts "d:fhj:p:r:s:t:" opt; do
p) PROJECT="$OPTARG" ;;
r) GIT_REPO="$OPTARG" ;;
s) RELEASE_STEP="$OPTARG" ;;
+ x) DEBUG=1 ;;
h) usage ;;
?) error "Invalid option. Run with -h for help." ;;
esac
@@ -102,6 +104,7 @@ shift $((OPTIND-1))
if (( $# > 0 )); then
error "Arguments can only be provided with option flags, invalid args: $*"
fi
+export DEBUG
if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
error "Work directory (-d) must be defined and exist. Run with -h for help."
@@ -114,12 +117,26 @@ if [ -d "$WORKDIR/output" ]; then
fi
fi
+if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ] || \
+ [ -f "${WORKDIR}/gpg-proxy.cid" ] || \
+ [ -f "${WORKDIR}/release.cid" ]; then
+ read -r -p "container/pid files from prior run exists. Overwrite and continue? [y/n] " ANSWER
+ if [ "$ANSWER" != "y" ]; then
+ error "Exiting."
+ fi
+fi
+
cd "$WORKDIR"
rm -rf "$WORKDIR/output"
+rm -rf "${WORKDIR}/gpg-proxy.ssh.pid" "${WORKDIR}/gpg-proxy.cid" "${WORKDIR}/release.cid"
mkdir "$WORKDIR/output"
+banner "Gathering release details."
+HOST_OS="$(get_host_os)"
get_release_info
+banner "Setup"
+
# Place all RM scripts and necessary data in a local directory that must be defined in the command
# line. This directory is mounted into the image. Its WORKDIR, the arg passed with -d.
for f in "$SELF"/*; do
@@ -128,25 +145,65 @@ for f in "$SELF"/*; do
fi
done
-GPG_KEY_FILE="$WORKDIR/gpg.key"
+# We need to import that public key in the container in order to use the private key via the agent.
+GPG_KEY_FILE="$WORKDIR/gpg.key.public"
+echo "Exporting public key for ${GPG_KEY}"
fcreate_secure "$GPG_KEY_FILE"
-$GPG --passphrase "$GPG_PASSPHRASE" --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
+$GPG "${GPG_ARGS[@]}" --export "${GPG_KEY}" > "${GPG_KEY_FILE}"
+
+function cleanup {
+ local id
+ banner "Release Cleanup"
+ if is_debug; then
+ echo "skipping due to debug run"
+ return 0
+ fi
+ echo "details in cleanup.log"
+ if [ -f "${ENVFILE}" ]; then
+ rm -f "$ENVFILE"
+ fi
+ rm -f "$GPG_KEY_FILE"
+ if [ -f "${WORKDIR}/gpg-proxy.ssh.pid" ]; then
+ id=$(cat "${WORKDIR}/gpg-proxy.ssh.pid")
+ echo "Stopping ssh tunnel for gpg-agent at PID ${id}" | tee -a cleanup.log
+ kill -9 "${id}" >>cleanup.log 2>&1 || true
+ rm -f "${WORKDIR}/gpg-proxy.ssh.pid" >>cleanup.log 2>&1
+ fi
+ if [ -f "${WORKDIR}/gpg-proxy.cid" ]; then
+ id=$(cat "${WORKDIR}/gpg-proxy.cid")
+ echo "Stopping gpg-proxy container with ID ${id}" | tee -a cleanup.log
+ docker kill "${id}" >>cleanup.log 2>&1 || true
+ rm -f "${WORKDIR}/gpg-proxy.cid" >>cleanup.log 2>&1
+ # TODO we should remove the gpgagent volume?
+ fi
+ if [ -f "${WORKDIR}/release.cid" ]; then
+ id=$(cat "${WORKDIR}/release.cid")
+ echo "Stopping release container with ID ${id}" | tee -a cleanup.log
+ docker kill "${id}" >>cleanup.log 2>&1 || true
+ rm -f "${WORKDIR}/release.cid" >>cleanup.log 2>&1
+ fi
+}
+
+trap cleanup EXIT
+
+echo "Host OS: ${HOST_OS}"
+if [ "${HOST_OS}" == "DARWIN" ]; then
+ run_silent "Building gpg-agent-proxy image with tag ${IMGTAG}..." "docker-proxy-build.log" \
+ docker build --build-arg "UID=${UID}" --build-arg "RM_USER=${USER}" \
+ --tag "org.apache.hbase/gpg-agent-proxy:${IMGTAG}" "${SELF}/mac-sshd-gpg-agent"
+fi
run_silent "Building hbase-rm image with tag $IMGTAG..." "docker-build.log" \
- docker build -t "hbase-rm:$IMGTAG" --build-arg UID=$UID "$SELF/hbase-rm"
+ docker build --tag "org.apache.hbase/hbase-rm:$IMGTAG" --build-arg "UID=$UID" \
+ --build-arg "RM_USER=${USER}" "$SELF/hbase-rm"
+banner "Final prep for container launch."
+echo "Writing out environment for container."
# Write the release information to a file with environment variables to be used when running the
# image.
ENVFILE="$WORKDIR/env.list"
fcreate_secure "$ENVFILE"
-function cleanup {
- rm -f "$ENVFILE"
- rm -f "$GPG_KEY_FILE"
-}
-
-trap cleanup EXIT
-
cat > "$ENVFILE" <<EOF
PROJECT=$PROJECT
DRY_RUN=$DRY_RUN
@@ -162,15 +219,15 @@ GIT_NAME=$GIT_NAME
GIT_EMAIL=$GIT_EMAIL
GPG_KEY=$GPG_KEY
ASF_PASSWORD=$ASF_PASSWORD
-GPG_PASSPHRASE=$GPG_PASSPHRASE
RELEASE_STEP=$RELEASE_STEP
API_DIFF_TAG=$API_DIFF_TAG
+HOST_OS=$HOST_OS
EOF
-JAVA_VOL=()
+JAVA_MOUNT=()
if [ -n "$JAVA" ]; then
echo "JAVA_HOME=/opt/hbase-java" >> "$ENVFILE"
- JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
+ JAVA_MOUNT=(--mount "type=bind,src=${JAVA},dst=/opt/hbase-java,readonly")
fi
#TODO some debug output would be good here
@@ -226,14 +283,61 @@ if [ -n "${GIT_REPO}" ]; then
echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
fi
-echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
+GPG_PROXY_MOUNT=()
+if [ "${HOST_OS}" == "DARWIN" ]; then
+ GPG_PROXY_MOUNT=(--mount "type=volume,src=gpgagent,dst=/home/${USER}/.gnupg/")
+ echo "Setting up GPG agent proxy container needed on OS X."
+ echo " we should clean this up for you. If that fails the container ID is below and in " \
+ "gpg-proxy.cid"
+ #TODO the key pair used should be configurable
+ docker run --rm -p 62222:22 \
+ --detach --cidfile "${WORKDIR}/gpg-proxy.cid" \
+ --mount \
+ "type=bind,src=${HOME}/.ssh/id_rsa.pub,dst=/home/${USER}/.ssh/authorized_keys,readonly" \
+ "${GPG_PROXY_MOUNT[@]}" \
+ "org.apache.hbase/gpg-agent-proxy:${IMGTAG}"
+ # gotta trust the container host
+ ssh-keyscan -p 62222 localhost 2>/dev/null | sort > "${WORKDIR}/gpg-agent-proxy.ssh-keyscan"
+ sort "${HOME}/.ssh/known_hosts" | comm -1 -3 - "${WORKDIR}/gpg-agent-proxy.ssh-keyscan" \
+ > "${WORKDIR}/gpg-agent-proxy.known_hosts"
+ if [ -s "${WORKDIR}/gpg-agent-proxy.known_hosts" ]; then
+ echo "Your ssh known_hosts does not include the entries for the gpg-agent proxy container."
+ echo "The following entry(ies) arre missing:"
+ sed -e 's/^/ /' "${WORKDIR}/gpg-agent-proxy.known_hosts"
+ read -r -p "Okay to add these entries to ${HOME}/.ssh/known_hosts? [y/n] " ANSWER
+ if [ "$ANSWER" != "y" ]; then
+ error "Exiting."
+ fi
+ cat "${WORKDIR}/gpg-agent-proxy.known_hosts" >> "${HOME}/.ssh/known_hosts"
+ fi
+ echo "Launching ssh reverse tunnel from the container to gpg agent."
+ echo " we should clean this up for you. If that fails the PID is in gpg-proxy.ssh.pid"
+ ssh -p 62222 -R "/home/${USER}/.gnupg/S.gpg-agent:$(gpgconf --list-dir agent-extra-socket)" \
+ -i "${HOME}/.ssh/id_rsa" -N -n localhost >gpg-proxy.ssh.log 2>&1 &
+ echo $! > "${WORKDIR}/gpg-proxy.ssh.pid"
+else
+ # Note that on linux we always directly mount the gpg agent's extra socket to limit what the
+ # container can ask the gpg-agent to do.
+ # When working on a remote linux machine you should be sure to forward both the remote machine's
+ # agent socket and agent extra socket to your local gpg-agent's extra socket. See the README.txt
+ # for an example.
+ GPG_PROXY_MOUNT=(--mount \
+ "type=bind,src=$(gpgconf --list-dir agent-extra-socket),dst=/home/${USER}/.gnupg/S.gpg-agent")
+fi
+
+banner "Building $RELEASE_TAG; output will be at $WORKDIR/output"
+echo "We should clean the container up when we are done. If that fails then the container ID " \
+ "is in release.cid"
+echo
# Where possible we specifcy "consistency=delegated" when we do not need host access during the
# build run. On Mac OS X specifically this gets us a big perf improvement.
-cmd=(docker run -ti \
+cmd=(docker run --rm -ti \
--env-file "$ENVFILE" \
- --mount "type=bind,src=${WORKDIR},dst=/opt/hbase-rm,consistency=delegated" \
- "${JAVA_VOL[@]}" \
+ --cidfile "${WORKDIR}/release.cid" \
+ --mount "type=bind,src=${WORKDIR},dst=/home/${USER}/hbase-rm,consistency=delegated" \
+ "${JAVA_MOUNT[@]}" \
"${GIT_REPO_MOUNT[@]}" \
- "hbase-rm:$IMGTAG")
+ "${GPG_PROXY_MOUNT[@]}" \
+ "org.apache.hbase/hbase-rm:$IMGTAG")
echo "${cmd[*]}"
"${cmd[@]}"
diff --git a/dev-support/create-release/do-release.sh b/dev-support/create-release/do-release.sh
index b359d4b..7e5f186 100755
--- a/dev-support/create-release/do-release.sh
+++ b/dev-support/create-release/do-release.sh
@@ -17,6 +17,7 @@
# limitations under the License.
#
+set -e
# Use the adjacent do-release-docker.sh instead, if you can.
# Otherwise, this runs core of the release creation.
# Will ask you questions on what to build and for logins
@@ -40,29 +41,64 @@ if (( $# > 0 )); then
error "Arguments can only be provided with option flags, invalid args: $*"
fi
+function gpg_agent_help {
+ cat <<EOF
+Trying to sign a test file using your GPG setup failed.
+
+Please make sure you have a local gpg-agent running with access to your secret keys prior to
+starting a release build. If you are creating release artifacts on a remote machine please check
+that you have set up ssh forwarding to the gpg-agent extra socket.
+
+For help on how to do this please see the README file in the create-release directory.
+EOF
+ exit 1
+}
+
# If running in docker, import and then cache keys.
if [ "$RUNNING_IN_DOCKER" = "1" ]; then
- # Run gpg agent.
- eval "$(gpg-agent --disable-scdaemon --daemon --no-grab --allow-preset-passphrase \
- --default-cache-ttl=86400 --max-cache-ttl=86400)"
- echo "GPG Version: $(gpg --version)"
- # Inside docker, need to import the GPG keyfile stored in the current directory.
- # (On workstation, assume GPG has access to keychain/cache with key_id already imported.)
- echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --import "$SELF/gpg.key"
+ # when Docker Desktop for mac is running under load there is a delay before the mounted volume
+ # becomes available. if we do not pause then we may try to use the gpg-agent socket before docker
+ # has got it ready and we will not think there is a gpg-agent.
+ if [ "${HOST_OS}" == "DARWIN" ]; then
+ sleep 5
+ fi
+ # in docker our working dir is set to where all of our scripts are held
+ # and we want default output to go into the "output" directory that should be in there.
+ if [ -d "output" ]; then
+ cd output
+ fi
+ GPG_ARGS=("${GPG_ARGS[@]}" --local-user "${GPG_KEY}")
+ echo "GPG Version: $("${GPG}" "${GPG_ARGS[@]}" --version)"
+ # Inside docker, need to import the GPG key stored in the current directory.
+ if ! $GPG "${GPG_ARGS[@]}" --import "$SELF/gpg.key.public" ; then
+ gpg_agent_help
+ fi
# We may need to adjust the path since JAVA_HOME may be overridden by the driver script.
if [ -n "$JAVA_HOME" ]; then
+ echo "Using JAVA_HOME from host."
export PATH="$JAVA_HOME/bin:$PATH"
else
# JAVA_HOME for the openjdk package.
- export JAVA_HOME=/usr
+ export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
fi
else
# Outside docker, need to ask for information about the release.
get_release_info
+ GPG_ARGS=("${GPG_ARGS[@]}" --local-user "${GPG_KEY}")
fi
+
GPG_TTY="$(tty)"
export GPG_TTY
+echo "Testing gpg signing."
+echo "foo" > gpg_test.txt
+if ! "${GPG}" "${GPG_ARGS[@]}" --detach --armor --sign gpg_test.txt ; then
+ gpg_agent_help
+fi
+# In --batch mode we have to be explicit about what we are verifying
+if ! "${GPG}" "${GPG_ARGS[@]}" --verify gpg_test.txt.asc gpg_test.txt ; then
+ gpg_agent_help
+fi
if [[ -z "$RELEASE_STEP" ]]; then
# If doing all stages, leave out 'publish-snapshot'
diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile
index 3b7afdb..b1eb76f 100644
--- a/dev-support/create-release/hbase-rm/Dockerfile
+++ b/dev-support/create-release/hbase-rm/Dockerfile
@@ -50,10 +50,15 @@ RUN wget -qO- "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&fi
tar xvz -C /opt
ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
-WORKDIR /opt/hbase-rm/output
-
ARG UID
-RUN useradd -m -s /bin/bash -p hbase-rm -u $UID hbase-rm
-USER hbase-rm:hbase-rm
+ARG RM_USER
+RUN groupadd hbase-rm && \
+ useradd --create-home --shell /bin/bash -p hbase-rm -u $UID $RM_USER && \
+ mkdir /home/$RM_USER/.gnupg && \
+ chown -R $RM_USER:hbase-rm /home/$RM_USER && \
+ chmod -R 700 /home/$RM_USER
+
+USER $RM_USER:hbase-rm
+WORKDIR /home/$RM_USER/hbase-rm/
-ENTRYPOINT [ "/opt/hbase-rm/do-release.sh" ]
+ENTRYPOINT [ "./do-release.sh" ]
diff --git a/dev-support/create-release/mac-sshd-gpg-agent/Dockerfile b/dev-support/create-release/mac-sshd-gpg-agent/Dockerfile
new file mode 100644
index 0000000..a71d867
--- /dev/null
+++ b/dev-support/create-release/mac-sshd-gpg-agent/Dockerfile
@@ -0,0 +1,100 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Image for use on Mac boxes to get a gpg agent socket available
+# within transient release building ocntainers.
+#
+# build like:
+#
+# docker build --build-arg "UID=$UID" --build-arg "RM_USER=$USER" \
+# --tag org.apache.hbase/gpg-agent-proxy mac-sshd-gpg-agent
+#
+# run like:
+#
+# docker run --rm -p 62222:22 \
+# --mount "type=bind,src=${HOME}/.ssh/id_rsa.pub,dst=/home/${USER}/.ssh/authorized_keys,readonly" \
+# --mount "type=volume,src=gpgagent,dst=/home/${USER}/.gnupg/" \
+# org.apache.hbase/gpg-agent-proxy:latest
+#
+# test like:
+#
+# ssh -p 62222 -R "/home/${USER}/.gnupg/S.gpg-agent:$(gpgconf --list-dir agent-extra-socket)" \
+# -i "${HOME}/.ssh/id_rsa" -N -n localhost
+#
+# launch a docker container to do work that shares the mount for the gpg agent
+# expressly does not need to be this same image, but needs to have defined the same user
+#
+# docker run --rm -it \
+# --mount "type=volume,src=gpgagent,dst=/home/${USER}/.gnupg/" \
+# --mount "type=bind,src=${HOME}/projects/hbase-releases/KEYS,dst=/home/${USER}/KEYS,readonly" \
+# --entrypoint /bin/bash --user "${USER}" --workdir "/home/${USER}/" \
+# org.apache.hbase/gpg-agent-proxy:latest
+#
+#
+# Make sure to import the public keys
+#
+# gpg --no-autostart --import < ${HOME}/KEYS
+# Optional?
+# gpg --no-autostart --edit-key ${YOUR_KEY}
+# trust
+# 5
+# y
+# quit
+#
+# echo "foo" > foo
+# gpg --no-autostart --armor --detach --sign foo
+# gpg --no-autostart --verify foo.asc
+#
+# For more info see
+# * gpg forwarding over ssh: https://wiki.gnupg.org/AgentForwarding
+# * example docker for sshd: https://github.com/hotblac/nginx-ssh
+# * why we have to bother with this: https://github.com/docker/for-mac/issues/483
+#
+# If the docker image changes then the host key used by sshd will change and you will get a
+# nastygram when launching ssh about host identification changing. This is expected. you should
+# remove the previous host key.
+#
+# Tested with
+# * Docker Desktop 2.2.0.5
+# * gpg 2.2.20
+# * pinentry-mac 0.9.4
+# * yubikey 5
+#
+FROM ubuntu:18.04
+
+# This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch
+# the most current package versions (instead of potentially using old versions cached by docker).
+#
+# We only need gnupg2 here if we want the ability to test out the gpg-agent forwarding by sshing
+# into the container rather than launching a new docker container.
+RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
+ && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
+ openssh-server=1:7.6p1-4ubuntu0.3 gnupg2=2.2.4-1ubuntu1.2 && mkdir /run/sshd \
+ && echo "StreamLocalBindUnlink yes" >> /etc/ssh/sshd_config \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+EXPOSE 22
+# Set up our ssh user
+ARG UID
+ARG RM_USER
+RUN groupadd sshgroup && \
+ useradd --create-home --shell /bin/bash --groups sshgroup --uid $UID $RM_USER && \
+ mkdir /home/$RM_USER/.ssh /home/$RM_USER/.gnupg && \
+ chown -R $RM_USER:sshgroup /home/$RM_USER/ && \
+ chmod -R 700 /home/$RM_USER/
+# When we run we run sshd
+ENTRYPOINT ["/usr/sbin/sshd", "-D"]
diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh
index 86cca65..e526836 100755
--- a/dev-support/create-release/release-build.sh
+++ b/dev-support/create-release/release-build.sh
@@ -66,13 +66,12 @@ Used only for 'publish':
separately define GIT_REF if RELEASE_TAG is not actually present as a tag at publish time)
If both RELEASE_TAG and GIT_REF are undefined it will default to HEAD of master.
GPG_KEY - GPG key id (usually email addr) used to sign release artifacts
- GPG_PASSPHRASE - Passphrase for GPG key
REPO - Set to full path of a directory to use as maven local repo (dependencies cache)
to avoid re-downloading dependencies for each stage. It is automatically set if you
request full sequence of stages (tag, publish-dist, publish-release) in do-release.sh.
For example:
- $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_PASSPHRASE=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist
+ $ PROJECT="hbase-operator-tools" ASF_USERNAME=NAME ASF_PASSWORD=PASSWORD GPG_KEY=stack@apache.org ./release-build.sh publish-dist
EOF
exit 1
}
@@ -165,12 +164,7 @@ fi
### Below is for 'publish-*' stages ###
check_get_passwords ASF_PASSWORD
-if [[ -z "$GPG_PASSPHRASE" ]]; then
- check_get_passwords GPG_PASSPHRASE
- GPG_TTY="$(tty)"
- export GPG_TTY
-fi
-check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY GPG_PASSPHRASE
+check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD GPG_KEY
# Commit ref to checkout when building
BASE_DIR=$(pwd)
diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh
index 3e9506b..b9b79ea 100755
--- a/dev-support/create-release/release-util.sh
+++ b/dev-support/create-release/release-util.sh
@@ -17,7 +17,9 @@
# limitations under the License.
#
DRY_RUN=${DRY_RUN:-1} #default to dry run
-GPG="gpg --pinentry-mode loopback --no-tty --batch"
+DEBUG=${DEBUG:-0}
+GPG=${GPG:-gpg}
+GPG_ARGS=(--no-autostart --batch)
# Maven Profiles for publishing snapshots and release to Maven Central and Dist
PUBLISH_PROFILES=("-P" "apache-release,release")
@@ -46,23 +48,26 @@ function parse_version {
head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1
}
+function banner {
+ local msg="$1"
+ echo "========================"
+ echo "=== ${msg}"
+ echo
+}
+
function run_silent {
local BANNER="$1"
local LOG_FILE="$2"
shift 2
- echo "========================"
- echo "=== $BANNER"
+ banner "${BANNER}"
echo "Command: $*"
echo "Log file: $LOG_FILE"
- "$@" 1>"$LOG_FILE" 2>&1
-
- local EC=$?
- if [ $EC != 0 ]; then
+ if ! "$@" 1>"$LOG_FILE" 2>&1; then
echo "Command FAILED. Check full logs for details."
tail "$LOG_FILE"
- exit $EC
+ exit 1
fi
echo "=== SUCCESS"
}
@@ -260,20 +265,17 @@ EOF
ASF_PASSWORD="***INVALID***"
fi
- if [ -z "$GPG_PASSPHRASE" ]; then
- stty -echo && printf "GPG_PASSPHRASE: " && read -r GPG_PASSPHRASE && printf '\n' && stty echo
- GPG_TTY="$(tty)"
- export GPG_TTY
- fi
-
export ASF_PASSWORD
- export GPG_PASSPHRASE
}
function is_dry_run {
[[ "$DRY_RUN" = 1 ]]
}
+function is_debug {
+ [[ "${DEBUG}" = 1 ]]
+}
+
function check_get_passwords {
for env in "$@"; do
if [ -z "${!env}" ]; then
@@ -381,8 +383,6 @@ function configure_maven {
<password>${env.ASF_PASSWORD}</password></server>
<server><id>apache.releases.https</id><username>${env.ASF_USERNAME}</username>
<password>${env.ASF_PASSWORD}</password></server>
- <server><id>gpg.passphrase</id>
- <passphrase>${env.GPG_PASSPHRASE}</passphrase></server>
</servers>
<profiles>
<profile>
@@ -436,6 +436,7 @@ function git_clone_overwrite {
}
# Writes report into cwd!
+# TODO should have option for maintenance release that include LimitedPrivate in report
function generate_api_report {
local project="$1"
local previous_tag="$2"
@@ -518,10 +519,9 @@ function update_releasenotes {
# named for 'project', the first arg passed.
# Expects the following three defines in the environment:
# - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
-# - The passphrase in the GPG_PASSPHRASE variable: no default (we don't make .asc file).
# - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
# For example:
-# $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
+# $ GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
make_src_release() {
# Tar up the src and sign and hash it.
local project="${1}"
@@ -533,9 +533,8 @@ make_src_release() {
git clean -d -f -x
git archive --format=tar.gz --output="../${tgz}" --prefix="${base_name}/" "${GIT_REF:-master}"
cd .. || exit
- echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "${tgz}.asc" \
- --detach-sig "${tgz}"
- echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --print-md SHA512 "${tgz}" > "${tgz}.sha512"
+ $GPG "${GPG_ARGS[@]}" --armor --output "${tgz}.asc" --detach-sig "${tgz}"
+ $GPG "${GPG_ARGS[@]}" --print-md SHA512 "${tgz}" > "${tgz}.sha512"
}
# Make binary release.
@@ -544,11 +543,10 @@ make_src_release() {
# named for 'project', the first arg passed.
# Expects the following three defines in the environment:
# - GPG needs to be defined, with the path to GPG: defaults 'gpg'.
-# - The passphrase in the GPG_PASSPHRASE variable: no default (we don't make .asc file).
# - GIT_REF which is the tag to create the tgz from: defaults to 'master'.
# - MVN Default is "mvn -B --settings $MAVEN_SETTINGS_FILE".
# For example:
-# $ GPG_PASSPHRASE="XYZ" GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
+# $ GIT_REF="master" make_src_release hbase-operator-tools 1.0.0
make_binary_release() {
local project="${1}"
local version="${2}"
@@ -573,8 +571,8 @@ make_binary_release() {
cp "${f_bin_prefix}"*-bin.tar.gz ..
cd .. || exit
for i in "${base_name}"*-bin.tar.gz; do
- echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "$i.asc" --detach-sig "$i"
- echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --print-md SHA512 "${i}" > "$i.sha512"
+ "${GPG}" "${GPG_ARGS[@]}" --armour --output "${i}.asc" --detach-sig "${i}"
+ "${GPG}" "${GPG_ARGS[@]}" --print-md SHA512 "${i}" > "${i}.sha512"
done
else
cd .. || exit
@@ -588,10 +586,11 @@ make_binary_release() {
# 'assembly' build (where gpg signing occurs) experiences timeout, without this "kick".
function kick_gpg_agent {
# All that's needed is to run gpg on a random file
+ # TODO could we just call gpg-connect-agent /bye
local i
i="$(mktemp)"
echo "This is a test file" > "$i"
- echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --armour --output "$i.asc" --detach-sig "$i"
+ "${GPG}" "${GPG_ARGS[@]}" --armour --output "${i}.asc" --detach-sig "${i}"
rm "$i" "$i.asc"
}