You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ew...@apache.org on 2018/02/28 18:29:16 UTC
[kafka] branch trunk updated: MINOR: Extend release.py with a
subcommand for staging docs into the kafka-site repo
This is an automated email from the ASF dual-hosted git repository.
ewencp pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new f582a75 MINOR: Extend release.py with a subcommand for staging docs into the kafka-site repo
f582a75 is described below
commit f582a7515c9303589a7cf964e53bd12992c66254
Author: Ewen Cheslack-Postava <me...@ewencp.org>
AuthorDate: Wed Feb 28 10:28:55 2018 -0800
MINOR: Extend release.py with a subcommand for staging docs into the kafka-site repo
Author: Ewen Cheslack-Postava <me...@ewencp.org>
Reviewers: Ismael Juma <is...@juma.me.uk>, Damian Guy <da...@gmail.com>
Closes #3917 from ewencp/stage-docs
---
release.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 135 insertions(+), 29 deletions(-)
diff --git a/release.py b/release.py
index b008794..0917184 100755
--- a/release.py
+++ b/release.py
@@ -20,14 +20,31 @@
"""
Utility for creating release candidates and promoting release candidates to a final relase.
-Usage: release.py
+Usage: release.py [subcommand]
-The utility is interactive; you will be prompted for basic release information and guided through the process.
+release.py stage
+
+ Builds and stages an RC for a release.
+
+ The utility is interactive; you will be prompted for basic release information and guided through the process.
+
+ This utility assumes you already have local a kafka git folder and that you
+ have added remotes corresponding to both:
+ (i) the github apache kafka mirror and
+ (ii) the apache kafka git repo.
+
+release.py stage-docs [kafka-site-path]
+
+ Builds the documentation and stages it into an instance of the Kafka website repository.
+
+ This is meant to automate the integration between the main Kafka website repository (https://github.com/apache/kafka-site)
+ and the versioned documentation maintained in the main Kafka repository. This is useful both for local testing and
+ development of docs (follow the instructions here: https://cwiki.apache.org/confluence/display/KAFKA/Setup+Kafka+Website+on+Local+Apache+Server)
+ as well as for committers to deploy docs (run this script, then validate, commit, and push to kafka-site).
+
+ With no arguments this script assumes you have the Kafka repository and kafka-site repository checked out side-by-side, but
+ you can specify a full path to the kafka-site repository if this is not the case.
-This utility assumes you already have local a kafka git folder and that you
-have added remotes corresponding to both:
-(i) the github apache kafka mirror and
-(ii) the apache kafka git repo.
"""
from __future__ import print_function
@@ -145,11 +162,113 @@ def get_pref(prefs, name, request_fn):
prefs[name] = val
return val
-# Load saved preferences
-prefs = {}
-if os.path.exists(PREFS_FILE):
- with open(PREFS_FILE, 'r') as prefs_fp:
- prefs = json.load(prefs_fp)
+def load_prefs():
+ """Load saved preferences"""
+ prefs = {}
+ if os.path.exists(PREFS_FILE):
+ with open(PREFS_FILE, 'r') as prefs_fp:
+ prefs = json.load(prefs_fp)
+ return prefs
+
+def save_prefs(prefs):
+ """Save preferences"""
+ print("Saving preferences to %s" % PREFS_FILE)
+ with open(PREFS_FILE, 'w') as prefs_fp:
+ prefs = json.dump(prefs, prefs_fp)
+
+def get_jdk(prefs, version):
+ """
+ Get settings for the specified JDK version.
+ """
+ jdk_java_home = get_pref(prefs, 'jdk%d' % version, lambda: raw_input("Enter the path for JAVA_HOME for a JDK%d compiler (blank to use default JAVA_HOME): " % version))
+ jdk_env = dict(os.environ) if jdk_java_home.strip() else None
+ if jdk_env is not None: jdk_env['JAVA_HOME'] = jdk_java_home
+ if "1.%d.0" % version not in cmd_output("java -version", env=jdk_env):
+ fail("JDK %s is required" % version)
+ return jdk_env
+
+def get_version(repo=REPO_HOME):
+ """
+ Extracts the full version information as a str from gradle.properties
+ """
+ with open(os.path.join(repo, 'gradle.properties')) as fp:
+ for line in fp:
+ parts = line.split('=')
+ if parts[0].strip() != 'version': continue
+ return parts[1].strip()
+ fail("Couldn't extract version from gradle.properties")
+
+def docs_version(version):
+ """
+ Detects the major/minor version and converts it to the format used for docs on the website, e.g. gets 0.10.2.0-SNAPSHOT
+ from gradle.properties and converts it to 0102
+ """
+ version_parts = version.strip().split('.')
+ # 1.0+ will only have 3 version components as opposed to pre-1.0 that had 4
+ major_minor = version_parts[0:3] if version_parts[0] == '0' else version_parts[0:2]
+ return ''.join(major_minor)
+
+def docs_release_version(version):
+ """
+ Detects the version from gradle.properties and converts it to a release version number that should be valid for the
+ current release branch. For example, 0.10.2.0-SNAPSHOT would remain 0.10.2.0-SNAPSHOT (because no release has been
+ made on that branch yet); 0.10.2.1-SNAPSHOT would be converted to 0.10.2.0 because 0.10.2.1 is still in development
+ but 0.10.2.0 should have already been released. Regular version numbers (e.g. as encountered on a release branch)
+ will remain the same.
+ """
+ version_parts = version.strip().split('.')
+ if '-SNAPSHOT' in version_parts[-1]:
+ bugfix = int(version_parts[-1].split('-')[0])
+ if bugfix > 0:
+ version_parts[-1] = str(bugfix - 1)
+ return '.'.join(version_parts)
+
+def command_stage_docs():
+ kafka_site_repo_path = sys.argv[2] if len(sys.argv) > 2 else os.path.join(REPO_HOME, '..', 'kafka-site')
+ if not os.path.exists(kafka_site_repo_path) or not os.path.exists(os.path.join(kafka_site_repo_path, 'powered-by.html')):
+ sys.exit("%s doesn't exist or does not appear to be the kafka-site repository" % kafka_site_repo_path)
+
+ prefs = load_prefs()
+ jdk8_env = get_jdk(prefs, 8)
+ save_prefs(prefs)
+
+ version = get_version()
+ # We explicitly override the version of the project that we normally get from gradle.properties since we want to be
+ # able to run this from a release branch where we made some updates, but the build would show an incorrect SNAPSHOT
+ # version due to already having bumped the bugfix version number.
+ gradle_version_override = docs_release_version(version)
+
+ cmd("Building docs", "./gradlew -Pversion=%s clean releaseTarGzAll aggregatedJavadoc" % gradle_version_override, cwd=REPO_HOME, env=jdk8_env)
+
+ docs_tar = os.path.join(REPO_HOME, 'core', 'build', 'distributions', 'kafka_2.11-%s-site-docs.tgz' % gradle_version_override)
+
+ versioned_docs_path = os.path.join(kafka_site_repo_path, docs_version(version))
+ if not os.path.exists(versioned_docs_path):
+ os.mkdir(versioned_docs_path, 0755)
+
+ # The contents of the docs jar are site-docs/<docs dir>. We need to get rid of the site-docs prefix and dump everything
+ # inside it into the docs version subdirectory in the kafka-site repo
+ cmd('Extracting site-docs', 'tar xf %s --strip-components 1' % docs_tar, cwd=versioned_docs_path)
+
+ javadocs_src_dir = os.path.join(REPO_HOME, 'build', 'docs', 'javadoc')
+
+ cmd('Copying javadocs', 'cp -R %s %s' % (javadocs_src_dir, versioned_docs_path))
+
+ sys.exit(0)
+
+
+# Dispatch to subcommand
+subcommand = sys.argv[1] if len(sys.argv) > 1 else None
+if subcommand == 'stage-docs':
+ command_stage_docs()
+elif not (subcommand is None or subcommand == 'stage'):
+ fail("Unknown subcommand: %s" % subcommand)
+# else -> default subcommand stage
+
+
+## Default 'stage' subcommand implementation isn't isolated to its own function yet for historical reasons
+
+prefs = load_prefs()
if not user_ok("""Requirements:
1. Updated docs to reference the new release version where appropriate.
@@ -221,7 +340,7 @@ except ValueError:
rc = raw_input("Release candidate number: ")
dev_branch = '.'.join(release_version_parts[:2])
-docs_version = ''.join(release_version_parts[:2])
+docs_release_version = docs_version(release_version[:2])
# Validate that the release doesn't already exist and that the
cmd("Fetching tags from upstream", 'git fetch --tags %s' % PUSH_REMOTE_NAME)
@@ -244,18 +363,8 @@ if not rc:
# Prereq checks
apache_id = get_pref(prefs, 'apache_id', lambda: raw_input("Enter your apache username: "))
-
-jdk7_java_home = get_pref(prefs, 'jdk7', lambda: raw_input("Enter the path for JAVA_HOME for a JDK7 compiler (blank to use default JAVA_HOME): "))
-jdk7_env = dict(os.environ) if jdk7_java_home.strip() else None
-if jdk7_env is not None: jdk7_env['JAVA_HOME'] = jdk7_java_home
-if "1.7.0" not in cmd_output("java -version", env=jdk7_env):
- fail("You must be able to build artifacts with JDK7 for Scala 2.10 and 2.11 artifacts")
-
-jdk8_java_home = get_pref(prefs, 'jdk8', lambda: raw_input("Enter the path for JAVA_HOME for a JDK8 compiler (blank to use default JAVA_HOME): "))
-jdk8_env = dict(os.environ) if jdk8_java_home.strip() else None
-if jdk8_env is not None: jdk8_env['JAVA_HOME'] = jdk8_java_home
-if "1.8.0" not in cmd_output("java -version", env=jdk8_env):
- fail("You must be able to build artifacts with JDK8 for Scala 2.12 artifacts")
+jdk7_env = get_jdk(prefs, 7)
+jdk8_env = get_jdk(prefs, 8)
def select_gpg_key():
@@ -275,10 +384,7 @@ with tempfile.NamedTemporaryFile() as gpg_test_tempfile:
gpg_test_tempfile.write("abcdefg")
cmd("Testing GPG key & passphrase", ["gpg", "--batch", "--pinentry-mode", "loopback", "--passphrase-fd", "0", "-u", key_name, "--armor", "--output", gpg_test_tempfile.name + ".asc", "--detach-sig", gpg_test_tempfile.name], stdin=gpg_passphrase)
-# Save preferences
-print("Saving preferences to %s" % PREFS_FILE)
-with open(PREFS_FILE, 'w') as prefs_fp:
- prefs = json.dump(prefs, prefs_fp)
+save_prefs(prefs)
# Generate RC
try:
@@ -400,7 +506,7 @@ release_notification_props = { 'release_version': release_version,
'rc_tag': rc_tag,
'rc_githash': rc_githash,
'dev_branch': dev_branch,
- 'docs_version': docs_version,
+ 'docs_version': docs_release_version,
'apache_id': apache_id,
}
--
To stop receiving notification emails like this one, please contact
ewencp@apache.org.