You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by on...@apache.org on 2020/04/27 07:25:09 UTC
[geode] branch develop updated: GEODE-8026: release improvements
(#5002)
This is an automated email from the ASF dual-hosted git repository.
onichols pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new edcc077 GEODE-8026: release improvements (#5002)
edcc077 is described below
commit edcc07752bc5cd1c8e800ec70a792cc699ba7143
Author: Owen Nichols <34...@users.noreply.github.com>
AuthorDate: Mon Apr 27 00:24:35 2020 -0700
GEODE-8026: release improvements (#5002)
* make release artifact names consistent
* verify tgz structure including presence of LICENSE and NOTICE and correct copyright in NOTICE
* verify file size is reasonable
* check that gfsh version --full contains correct SHA, version, and was built with an open-licensed JDK
---
dev-tools/release/deploy_rc_pipeline.sh | 129 +++++++++++++++++++++++++-------
dev-tools/release/prepare_rc.sh | 43 +++++------
2 files changed, 117 insertions(+), 55 deletions(-)
diff --git a/dev-tools/release/deploy_rc_pipeline.sh b/dev-tools/release/deploy_rc_pipeline.sh
index d21a449..2666585 100755
--- a/dev-tools/release/deploy_rc_pipeline.sh
+++ b/dev-tools/release/deploy_rc_pipeline.sh
@@ -105,9 +105,16 @@ jobs:
- -ec
- |
set -ex
+ FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+ VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
+ SHA=$(cd geode && git rev-parse HEAD)
java -version
cd geode
- ./gradlew test
+ ./gradlew test installDist
+ gfsh=geode-assembly/build/install/apache-geode/bin/gfsh
+ $gfsh version --full | grep "^Source-Revision: ${SHA}$"
+ $gfsh version --full | grep "^Product-Version: ${VERSION}$"
+ ! $gfsh version --full | grep Oracle
- name: build-geode-from-src-tgz
serial: true
plan:
@@ -133,11 +140,50 @@ jobs:
set -ex
FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
- curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}-src.tgz > src.tgz
+ SHA=$(cd geode && git rev-parse HEAD)
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}-src.tgz > src.tgz
tar xzf src.tgz
cd apache-geode-${VERSION}-src
java -version
- ./gradlew test
+ ./gradlew test installDist
+ gfsh=geode-assembly/build/install/apache-geode/bin/gfsh
+ $gfsh version --full | grep "^Source-Revision: ${SHA}$"
+ $gfsh version --full | grep "^Product-Version: ${VERSION}$"
+ ! $gfsh version --full | grep Oracle
+ - name: run-gfsh-from-tgz
+ serial: true
+ plan:
+ - aggregate:
+ - get: geode
+ trigger: true
+ - task: validate
+ timeout: 1h
+ config:
+ image_resource:
+ type: docker-image
+ source:
+ repository: openjdk
+ tag: 8
+ inputs:
+ - name: geode
+ platform: linux
+ run:
+ path: /bin/sh
+ args:
+ - -ec
+ - |
+ set -ex
+ FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+ VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
+ SHA=$(cd geode && git rev-parse HEAD)
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > bin.tgz
+ tar xzf bin.tgz
+ cd apache-geode-${VERSION}
+ java -version
+ gfsh=bin/gfsh
+ $gfsh version --full | grep "^Source-Revision: ${SHA}$"
+ $gfsh version --full | grep "^Product-Version: ${VERSION}$"
+ ! $gfsh version --full | grep Oracle
- name: run-geode-examples-jdk11
serial: true
plan:
@@ -164,7 +210,7 @@ jobs:
cd geode-examples
java -version
./gradlew runAll
- - name: run-geode-examples-from-src-tar-gz-jdk8
+ - name: run-geode-examples-from-src-tgz-jdk8
serial: true
plan:
- aggregate:
@@ -190,9 +236,9 @@ jobs:
FULL_VERSION=$(cd geode-examples && git describe --tags | sed -e 's#^rel/v##' -e 's#-.*##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
- curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}.tar.gz > src.tgz
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}-src.tgz > src.tgz
tar xzf src.tgz
- cd apache-geode-examples-${VERSION}
+ cd apache-geode-examples-${VERSION}-src
java -version
./gradlew -PgeodeReleaseUrl=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION} -PgeodeRepositoryUrl=${STAGING_MAVEN} build runAll
- name: build-geode-native-from-tag
@@ -221,7 +267,7 @@ jobs:
FULL_VERSION=$(cd geode-native && git describe --tags | sed -e 's#^rel/v##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
#use geode from binary dist
- curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > geode-bin.tgz
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > geode-bin.tgz
tar xzf geode-bin.tgz
# needed to get cmake >= 3.12
echo 'APT::Default-Release "stable";' >> /etc/apt/apt.conf.d/99defaultrelease
@@ -237,7 +283,7 @@ jobs:
cmake --build . -- -j 4
cmake --build . --target docs -- -j 4
cmake --build . --target install -- -j 4
- - name: build-geode-native-from-src-tar-gz
+ - name: build-geode-native-from-src-tgz
serial: true
plan:
- aggregate:
@@ -275,9 +321,9 @@ jobs:
echo 'deb http://security.debian.org/ stable/updates main contrib non-free' >> /etc/apt/sources.list.d/stable.list
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y cmake openssl doxygen build-essential libssl-dev zlib1g-dev
- curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-native-${VERSION}-src.tar.gz > src.tgz
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-native-${VERSION}-src.tgz > src.tgz
tar xzf src.tgz
- cd apache-geode-native*
+ cd apache-geode-native-${VERSION}-src
mkdir build
cd build
cmake .. -DGEODE_ROOT=$PWD/../../geode/geode-assembly/build/install/apache-geode
@@ -315,7 +361,7 @@ jobs:
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
cd upthewaterspout-tests
- curl -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
gpg --import KEYS
java -version
./gradlew build -PmavenURL=${STAGING_MAVEN} -PdownloadURL=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/ -Pversion=${FULL_VERSION}
@@ -343,7 +389,7 @@ jobs:
set -ex
FULL_VERSION=$(cd geode-benchmarks && git describe --tags | sed -e 's#^rel/v##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
- curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-benchmarks-${VERSION}-src.tgz > src.tgz
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-benchmarks-${VERSION}-src.tgz > src.tgz
tar xzf src.tgz
cd apache-geode-benchmarks-${VERSION}-src
java -version
@@ -380,30 +426,54 @@ jobs:
set -ex
FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
- curl -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
+ curl -L -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
gpg --import KEYS
url=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}
- function verifyArtifactSignature {
- file=$1
+ function verifyArtifactSizeSignatureLicenseNoticeAndCopyright {
+ tld=$1
+ file=${tld}.tgz
+ minfilesize=$2
+ maxfilesize=$3
echo Verifying $file...
asc=${file}.asc
- sha=${file}.sha$2
- sum=sha${2}sum
- curl -s $url/$file > $file
- curl -s $url/$asc > $asc
- curl -s $url/$sha > $sha
+ sha=${file}.sha256
+ sum=sha256sum
+ curl -L -s $url/$file > $file
+ actualfilesize=$(wc -c < $file)
+ if [ $actualfilesize -lt $minfilesize ] ; then
+ echo "File size of $file is only $actualfilesize bytes, expected at least $minfilesize"
+ return 1
+ fi
+ if [ $actualfilesize -gt $maxfilesize ] ; then
+ echo "File size of $file is $actualfilesize, expected no more than $maxfilesize bytes"
+ return 1
+ fi
+ curl -L -s $url/$asc > $asc
+ curl -L -s $url/$sha > $sha
gpg --verify $asc
$sum -c $sha
echo $file >> exp
echo $asc >> exp
echo $sha >> exp
+ #check that each archive contains all content below a top-level-directory with the same name as the file (sans .tgz)
+ ! tar tvzf $file | grep -v " ${tld}/"
+ #check that each archive contains LICENSE and NOTICE
+ tar tvzf $file | grep " ${tld}/LICENSE"
+ tar tvzf $file | grep " ${tld}/NOTICE"
+ #check that NOTICE contains current copyright year and correctly assigns copyright to ASF
+ tar xzf $file "${tld}/NOTICE"
+ year=$(date +%Y)
+ grep "Copyright" "${tld}/NOTICE"
+ grep -q "Copyright.*${year}.*Apache Software Foundation" "${tld}/NOTICE"
+ #check that the declared license is of the correct type
+ head -1 "${tld}/LICENSE" | grep -q "Apache License"
}
- verifyArtifactSignature apache-geode-${VERSION}-src.tgz 256
- verifyArtifactSignature apache-geode-${VERSION}.tgz 256
- verifyArtifactSignature apache-geode-examples-${VERSION}.tar.gz 256
- verifyArtifactSignature apache-geode-native-${VERSION}-src.tar.gz 512
- verifyArtifactSignature apache-geode-benchmarks-${VERSION}-src.tgz 256
- curl -s ${url}/ | awk '/>..</{next}/<li>/{gsub(/ *<[^>]*>/,"");print}' | sort > actual-file-list
+ verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION}-src 10000000 30000000
+ verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION} 100000000 150000000
+ verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-examples-${VERSION}-src 50000 2000000
+ verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-native-${VERSION}-src 2000000 4000000
+ verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-benchmarks-${VERSION}-src 50000 500000
+ curl -L -s ${url}/ | awk '/>..</{next}/<li>/{gsub(/ *<[^>]*>/,"");print}' | sort > actual-file-list
sort < exp > expected-file-list
set +x
echo ""
@@ -453,11 +523,11 @@ jobs:
file=$1
echo ""
echo Checking $file...
- curl -s $url/$file | tar tvzf - | egrep '\.('"${BINARY_EXTENSIONS}"')$' | tee -a bins
+ curl -L -s $url/$file | tar tvzf - | egrep '\.('"${BINARY_EXTENSIONS}"')$' | tee -a bins
}
verifyNoBinaries apache-geode-${VERSION}-src.tgz
- verifyNoBinaries apache-geode-examples-${VERSION}.tar.gz
- verifyNoBinaries apache-geode-native-${VERSION}-src.tar.gz
+ verifyNoBinaries apache-geode-examples-${VERSION}-src.tgz
+ verifyNoBinaries apache-geode-native-${VERSION}-src.tgz
verifyNoBinaries apache-geode-benchmarks-${VERSION}-src.tgz
echo ""
echo ""
@@ -470,4 +540,5 @@ jobs:
EOF
fly -t concourse.apachegeode-ci.info-main login --team-name main --concourse-url https://concourse.apachegeode-ci.info/
fly -t concourse.apachegeode-ci.info-main set-pipeline -p apache-support-${VERSION_MM//./-}-rc -c $PIPEYML
+fly -t concourse.apachegeode-ci.info-main unpause-pipeline -p apache-support-${VERSION_MM//./-}-rc
rm $PIPEYML
diff --git a/dev-tools/release/prepare_rc.sh b/dev-tools/release/prepare_rc.sh
index f3c4118..8039293 100755
--- a/dev-tools/release/prepare_rc.sh
+++ b/dev-tools/release/prepare_rc.sh
@@ -104,6 +104,11 @@ GEODE_NATIVE=$WORKSPACE/geode-native
GEODE_BENCHMARKS=$WORKSPACE/geode-benchmarks
BREW_DIR=$WORKSPACE/homebrew-core
SVN_DIR=$WORKSPACE/dist/dev/geode
+if which shasum >/dev/null; then
+ SHASUM="shasum -a 256"
+else
+ SHASUM=sha256sum
+fi
set +x
@@ -193,25 +198,18 @@ which brew >/dev/null && OPENSSL_ROOT_DIR=$(brew --prefix openssl) || OPENSSL_RO
cd ${GEODE_NATIVE}/build
cmake .. -DPRODUCT_VERSION=${VERSION} -DOPENSSL_ROOT_DIR=$OPENSSL_ROOT_DIR -DGEODE_ROOT=${GEODE}/geode-assembly/build/install/apache-geode
cpack -G TGZ --config CPackSourceConfig.cmake
-NCTAR=apache-geode-native-${VERSION}-src.tar.gz
+NCOUT=apache-geode-native-${VERSION}-src.tar.gz
+NCTGZ=apache-geode-native-${VERSION}-src.tgz
mkdir repkg-temp
cd repkg-temp
-tar xzf ../${NCTAR}
-rm ../${NCTAR}
-mv apache-geode-native apache-geode-native-${VERSION}
-tar czf ../${NCTAR} *
+tar xzf ../${NCOUT}
+rm ../${NCOUT}*
+mv apache-geode-native apache-geode-native-${VERSION}-src
+tar czf ../${NCTGZ} *
cd ..
rm -Rf repkg-temp
-gpg --armor -u ${SIGNING_KEY} -b ${NCTAR}
-
-if which shasum >/dev/null; then
- SHASUM=shasum
- SHASUM_OPTS="-a 512"
-else
- SHASUM=sha512sum
- SHASUM_OPTS=""
-fi
-${SHASUM} ${SHASUM_OPTS} ${NCTAR} > ${NCTAR}.sha512
+gpg --armor -u ${SIGNING_KEY} -b ${NCTGZ}
+${SHASUM} ${NCTGZ} > ${NCTGZ}.sha256
set +x
@@ -229,14 +227,7 @@ cp -r .travis.yml * ../${BMDIR}
tar czf ${BMTAR} -C .. ${BMDIR}
rm -Rf ../${BMDIR}
gpg --armor -u ${SIGNING_KEY} -b ${BMTAR}
-if which shasum >/dev/null; then
- SHASUM=shasum
- SHASUM_OPTS="-a 256"
-else
- SHASUM=sha256sum
- SHASUM_OPTS=""
-fi
-${SHASUM} ${SHASUM_OPTS} ${BMTAR} > ${BMTAR}.sha256
+${SHASUM} ${BMTAR} > ${BMTAR}.sha256
set +x
@@ -276,9 +267,9 @@ cp ${GEODE_BENCHMARKS}/apache-geode-benchmarks-${VERSION}* ${FULL_VERSION}
set +x
# verify all files are signed. sometimes gradle "forgets" to make the .asc file
-for f in ${FULL_VERSION}/*.tgz ${FULL_VERSION}/*.tar.gz ; do
- if ! [ -r $f.sha256 ] && ! [ -r $f.sha512 ] ; then
- echo missing $f.sha256 or $f.sha512
+for f in ${FULL_VERSION}/*.tgz ; do
+ if ! [ -r $f.sha256 ] ; then
+ echo missing $f.sha256
exit 1
fi
if ! [ -r $f.asc ] ; then