You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@httpd.apache.org by ic...@apache.org on 2021/09/30 11:24:59 UTC
svn commit: r1893751 - in /httpd/dev-tools/release: README common-lib.sh
cve-change.sh r0-make-candidate.sh r4-stage-release.sh
Author: icing
Date: Thu Sep 30 11:24:59 2021
New Revision: 1893751
URL: http://svn.apache.org/viewvc?rev=1893751&view=rev
Log:
* using CVE.json for CHANGES entry generation
* adding CVE.json to site/content/security/jsopn
* adding release version and date to copied CVE.json,
so site can do proper sorting
* freezing ready CVE information when candidate is
created
Added:
httpd/dev-tools/release/cve-change.sh (with props)
Modified:
httpd/dev-tools/release/README
httpd/dev-tools/release/common-lib.sh
httpd/dev-tools/release/r0-make-candidate.sh
httpd/dev-tools/release/r4-stage-release.sh
Modified: httpd/dev-tools/release/README
URL: http://svn.apache.org/viewvc/httpd/dev-tools/release/README?rev=1893751&r1=1893750&r2=1893751&view=diff
==============================================================================
--- httpd/dev-tools/release/README (original)
+++ httpd/dev-tools/release/README Thu Sep 30 11:24:59 2021
@@ -1,8 +1,8 @@
httpd release scripts
STATUS
- - implemented up to and including pushing staging release changes to
- be commited to the repositories and website
+ - New, untested changes for CVE integration of CVE.json data
+ - needs 'jq' installed
Usage overview:
- have a checkout of https://svn.apache.org/repos/asf/httpd/dev-tools in $DEV_TOOLS
Modified: httpd/dev-tools/release/common-lib.sh
URL: http://svn.apache.org/viewvc/httpd/dev-tools/release/common-lib.sh?rev=1893751&r1=1893750&r2=1893751&view=diff
==============================================================================
--- httpd/dev-tools/release/common-lib.sh (original)
+++ httpd/dev-tools/release/common-lib.sh Thu Sep 30 11:24:59 2021
@@ -38,6 +38,20 @@ ask_yes_no() {
return 1
}
+assure_commands() {
+ # check that all commands that we need are available
+ ok=1
+ for cmd in svn git jq sed curl perl awk; do
+ if ! type "$cmd" >/dev/null 2>&1; then
+ echo "ERROR: needed command not found: $cmd" >&2
+ ok=0
+ fi
+ done
+ if test "$ok" != 1; then
+ fail "please install the missing components."
+ fi
+}
+
# Inspect the local directory and populate
# SVN_URL the absolute url of the local checkout
# SVN_BASE the base url of the project
@@ -80,6 +94,7 @@ assure_clean_checkout() {
# v_patch the patch integer
# v_suffix the optional suffix, e.g. 'rc1' or so
detect_version() {
+ assure_commands
version=$1
case "$version" in
rc*)
@@ -356,13 +371,44 @@ fixup_httpd_export() {
find "${export_dir}"/docs/manual -name \*.xsl -o -name \*.xsl.\* | xargs rm -rf
}
+indent_entry() {
+ indent="$1"
+ indent="${indent:- *) }"
+ while read line; do
+ echo "${indent}${line}"
+ indent=" "
+ done
+}
+
+cve_changes() {
+ CVE_JSON="$1"
+ test -f "${CVE_JSON}" || fail "cve_changes: ${CVE_JSON} not found"
+ CVE_ID=`jq -r .CVE_data_meta.ID "${CVE_JSON}"`
+ CVE_TITLE=`jq -r .CVE_data_meta.TITLE "${CVE_JSON}"`
+ test -n "${CVE_ID}" -a "${CVE_ID}" != "null" || fail "CVE ID missing in ${CVE_JSON}"
+ CVE_DESCR=`jq -r '.description.description_data[].value' "${CVE_JSON}"`
+ cat << EOF | fold -sw 65 | indent_entry
+SECURITY: ${CVE_ID}: ${CVE_TITLE} (cve.mitre.org)
+${CVE_DESCR}
+
+EOF
+ CVE_CREDIT=`jq -r '.credit[].value' "${CVE_JSON}"`
+ if test -n "${CVE_CREDIT}" -a "${CVE_CREDIT}" != "null"; then
+ cat << EOF | fold -sw 65 | indent_entry " "
+Credits: ${CVE_CREDIT}
+
+EOF
+ fi
+}
+
add_changes_entry() {
CHANGES="$1"
- ENTRY="$2"
+ CVE_JSON="$2"
test -f "${CHANGES}" || fail "add_changes_entry: changes ${CHANGES} not found"
- test -f "${ENTRY}" || fail "add_changes_entry: entry ${ENTRY} not found"
- cp "${CHANGES}" "${CHANGES}".tmp ;
- awk -v fname="${ENTRY}" \
+
+ cp "${CHANGES}" "${CHANGES}.tmp" ;
+ cve_changes "${CVE_JSON}" > "${CHANGES}.entry.tmp"
+ awk -v fname="${CHANGES}.entry.tmp" \
'BEGIN{done = 0; active = 0} done == 0 && active == 0 && /^Changes with Apache /{
active = 1; print; next};
/^( *\*|Changes with Apache )/ && active == 1 && done == 0 {
@@ -371,7 +417,7 @@ add_changes_entry() {
}
printf "\n"; print rec; active = 0; done = 1; next
} //;' "${CHANGES}".tmp > "${CHANGES}"
- rm "${CHANGES}".tmp
+ rm -f "${CHANGES}".tmp "${CHANGES}.entry.tmp"
}
stage_checkout_release() {
@@ -451,19 +497,45 @@ calc_ready_CVE_DIRS() {
dest="$1"
checkout_pmc "${dest}"
pushd "${dest}/SECURITY" >/dev/null
- CVE_DIRS=`./tools/readiness.sh -n | fgrep '[OK]'| awk '{print $2}' | sort `
+ CVE_SUBDIRS=`./tools/readiness.sh -n | fgrep '[OK]'| awk '{print $2}' | sort `
popd >/dev/null
cves_ok=1
- for CVE in $CVE_DIRS; do
- if test -f "${dest}/SECURITY/${CVE}/CHANGES"; then
- :
- elif test -f "${dest}/SECURITY/${CVE}/CHANGES.diff"; then
- echo "ERROR: cannot apply old-style diff in ${dest}/SECURITY/${CVE}/CHANGES.diff." \
- " Please convert this to a CHANGES files with just the text."
+ CVE_DIRS=""
+ for CVE in $CVE_SUBDIRS; do
+ CVE=`basename "${CVE}"`
+ if test -f "${dest}/SECURITY/${CVE}/CVE.json"; then
+ CVE_DIRS="$CVE_DIRS ${CVE}"
+ else
+ echo "ERROR: CVE ${CVE} does not have a CVE.json file."
cves_ok=0
+ fi
+ done
+ if test "${cves_ok}" != 1; then
+ fail "Please correct the files mentioned."
+ fi
+}
+
+get_version_CVE_DIRS() {
+ dest="$1"
+ checkout_pmc "${dest}"
+ cve_version_dir="${dest}/SECURITY/${PROJECT}-${FULL_VERSION}"
+ if ! test -d "${cve_version_dir}"; then
+ echo "CVE Directory '${cve_version_dir}' does not exist."
+ echo "It should have been created and committed during creation of the candidate."
+ fail "something is wrong with the state of ${cve_version_dir}"
+ fi
+ cves_ok=1
+ CVE_DIRS=""
+ for CVE in "${cve_version_dir}"/*; do
+ if ! test -d "${CVE}"; then
+ continue
+ fi
+ CVE=`basename "${CVE}"`
+ if test -f "${dest}/SECURITY/${CVE}/CVE.json"; then
+ CVE_DIRS="$CVE_DIRS ${CVE}"
else
- echo "ERROR: CVE ${CVE} does not have a CHANGES file."
+ echo "ERROR: CVE ${CVE} does not have a CVE.json file."
cves_ok=0
fi
done
Added: httpd/dev-tools/release/cve-change.sh
URL: http://svn.apache.org/viewvc/httpd/dev-tools/release/cve-change.sh?rev=1893751&view=auto
==============================================================================
--- httpd/dev-tools/release/cve-change.sh (added)
+++ httpd/dev-tools/release/cve-change.sh Thu Sep 30 11:24:59 2021
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# Check which CVEs are ready for a release right now.
+#
+
+#Useful for debugging
+#set -x
+
+#Bail when non-zero return codes are encountered
+set -e
+
+
+usage () {
+ cat <<EOF 1>&2
+usage: $0 [options] file [file...]
+ Print CHANGES entry for a CVE.json file.
+ Options:
+ -h print usage information
+EOF
+ exit 1
+}
+
+while getopts "h" opt; do
+ case $opt in
+ h) usage
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+
+source `dirname $0`/common-lib.sh
+source `dirname $0`/env.sh
+
+for CVE_JSON in "$@"; do
+ cve_changes "${CVE_JSON}"
+done
Propchange: httpd/dev-tools/release/cve-change.sh
------------------------------------------------------------------------------
svn:executable = *
Modified: httpd/dev-tools/release/r0-make-candidate.sh
URL: http://svn.apache.org/viewvc/httpd/dev-tools/release/r0-make-candidate.sh?rev=1893751&r1=1893750&r2=1893751&view=diff
==============================================================================
--- httpd/dev-tools/release/r0-make-candidate.sh (original)
+++ httpd/dev-tools/release/r0-make-candidate.sh Thu Sep 30 11:24:59 2021
@@ -51,15 +51,21 @@ detect_version $1
SVN_DEST="tags/candidate-${FULL_VERSION}"
DEST_DIR="${DIST_DIR}/candidate-${FULL_VERSION}"
-cat <<EOF
-creating release candidate:
- PROJECT: $PROJECT
- VERSION: $FULL_VERSION
- URL: $SVN_URL
- SOURCE: $SVN_SRC@$SVN_REV
- DEST: $SVN_DEST
- LOCAL: $DEST_DIR
+# find out which CVEs are ready and will be reporting in this candidate
+calc_ready_CVE_DIRS "${DIST_DIR}/pmc"
+
+# if a directory of the candidate already exists at pmc/SECURITY
+# something is fishy and we require someone to decide what to fix.
+if test -d "${DIST_DIR}/pmc/SECURITY/${PROJECT}-${FULL_VERSION}"; then
+ cat <<EOF >&2
+${DIST_DIR}/pmc/SECURITY/${PROJECT}-${FULL_VERSION} already exists.
+This indicates that the candidate has already been built before
+and not been fully removed again. Please verify if the version
+number used is correct or if the removal of SECURITY/${PROJECT}-${FULL_VERSION}
+was just omitted by mistake.
EOF
+ fail "Please correct this and try again."
+fi
if test -f include/ap_release.h; then
if ! grep -e "^#define *AP_SERVER_MAJORVERSION_NUMBER *${v_major}\$" include/ap_release.h >/dev/null 2>&1;then
@@ -79,6 +85,20 @@ if test -d changes-entries; then
fi
fi
+cat <<EOF
+creating release candidate:
+ PROJECT: $PROJECT
+ VERSION: $FULL_VERSION
+ URL: $SVN_URL
+ SOURCE: $SVN_SRC@$SVN_REV
+ DEST: $SVN_DEST
+ LOCAL: $DEST_DIR
+ CVEs: ${CVE_DIRS:-none}
+EOF
+if ! ask_yes_no "Is this as expected?"; then
+ exit 1
+fi
+
if svn ls "$SVN_BASE/$SVN_DEST" >/dev/null 2>&1; then
warn "release candidate tag already exists at $SVN_DEST"
if ! ask_yes_no "Do you want to continue anyway?"; then
@@ -151,10 +171,23 @@ fi
popd >/dev/null
-echo "changes for candidate-${FULL_VERSION} in ${DEST_DIR}"
+echo "changes for ${PROJECT}-${FULL_VERSION} in ${DEST_DIR}"
svn stat "${DEST_DIR}"
+echo ""
if ask_yes_no "Do you want to commit these?"; then
svn commit -m "Post $FULL_VERSION tag updates" "$DEST_DIR"
echo "release candidate sources created in $SVN_DEST"
save_version
+
+ if test -n "${CVE_DIRS}"; then
+ echo "Saving CVE information for ${PROJECT}-${FULL_VERSION}"
+ mkdir "${DIST_DIR}/pmc/SECURITY/${PROJECT}-${FULL_VERSION}"
+ pushd "${DIST_DIR}/pmc/SECURITY/${PROJECT}-${FULL_VERSION}"
+ for CVE in $CVE_DIRS; do
+ ln -s ../"${CVE}" "${CVE}"
+ done
+ popd
+ svn add "${DIST_DIR}/pmc/SECURITY/${PROJECT}-${FULL_VERSION}"
+ svn commit -m "CVEs included in ${PROJECT}-${FULL_VERSION}" "${DIST_DIR}/pmc"
+ fi
fi
Modified: httpd/dev-tools/release/r4-stage-release.sh
URL: http://svn.apache.org/viewvc/httpd/dev-tools/release/r4-stage-release.sh?rev=1893751&r1=1893750&r2=1893751&view=diff
==============================================================================
--- httpd/dev-tools/release/r4-stage-release.sh (original)
+++ httpd/dev-tools/release/r4-stage-release.sh Thu Sep 30 11:24:59 2021
@@ -97,23 +97,18 @@ if test ! -f "${AO_DIST_RELEASE_PATH}/CH
fi
-# checkout/update pmc and get dir dir names that are ready
-# add the CVE CHANGES to all relevant files
-calc_ready_CVE_DIRS "${DIST_DIR}/pmc"
+# checkout/update pmc and get dir dir names that have
+# been recorded there for this release
+get_version_CVE_DIRS "${DIST_DIR}/pmc"
if test -n "${CVE_DIRS}"; then
echo "adding CVEs: ${CVE_DIRS}"
for changes in CHANGES \
"${AO_DIST_RELEASE_PATH}/CHANGES_${v_major}.${v_minor}" \
"${AO_DIST_RELEASE_PATH}/CHANGES_${VERSION}"; do
for CVE in $CVE_DIRS; do
- add_changes_entry "$changes" "${DIST_DIR}/pmc/SECURITY/${CVE}/CHANGES"
+ add_changes_entry "$changes" "${DIST_DIR}/pmc/SECURITY/${CVE}/CVE.json"
done
done
-
- echo "moving CVEs to pmc/resolved ..."
- for CVE in $CVE_DIRS; do
- svn mv "${DIST_DIR}/pmc/SECURITY/${CVE}" "${DIST_DIR}/pmc/resolved/" >/dev/null
- done
fi
@@ -144,8 +139,30 @@ perl -pi -e "s/\\d{4}-\\d{2}-\\d{2}/$iso
if ! grep -e "# Apache HTTP Server ${VERSION} (httpd): ${VERSION}.*<span>$iso_date</span>" "${DIST_SITEPATH}"/content/download.md >/dev/null;then
fail "Failed to update the release version and/or date in download.md"
fi
+
+if test -n "${CVE_DIRS}"; then
+ TIMELINE_ENTRY='{"lang":"eng","time":"'"${iso_date}"'","value":"'"${VERSION}"' released"}'
+ for CVE in $CVE_DIRS; do
+ CVE_JSON="${DIST_DIR}/pmc/SECURITY/${CVE}/CVE.json"
+ test -f "$CVE_JSON" || fail "not found: ${CVE_JSON}"
+ CVE_ID=`jq -r .CVE_d`a`ta_meta.ID "${CVE_JSON}"`
+ test -n "${CVE_ID}" -a "${CVE_ID}" != "null" || fail "CVE ID missing in ${CVE_JSON}"
+ # add a timeline entry for the release and copy to website
+ jq '.timeline[.timeline|length] |= . + '"${TIMELINE_ENTRY}" "${CVE_JSON}"\
+ >"${DIST_SITEPATH}"/content/security/json/${CVE_ID}.json
+ done
+fi
+
(cd "${DIST_SITEPATH}" && git add .)
+# finally, move CVEs we handled to resolved
+if test -n "${CVE_DIRS}"; then
+ echo "moving CVEs to pmc/resolved ..."
+ for CVE in $CVE_DIRS; do
+ svn mv "${DIST_DIR}/pmc/SECURITY/${CVE}" "${DIST_DIR}/pmc/resolved/" >/dev/null
+ done
+fi
+
########################################
# stage changes in the local branch checkout
# step the patch level and update local files for the next version