You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ch...@apache.org on 2017/12/19 15:42:45 UTC
[1/2] hbase git commit: HBASE-18548 Move sources of website gen and
check jobs into source control
Repository: hbase
Updated Branches:
refs/heads/branch-2 35296e590 -> 61ce86399
HBASE-18548 Move sources of website gen and check jobs into source control
Signed-off-by: Chia-Ping Tsai <ch...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ed98ee1c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ed98ee1c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ed98ee1c
Branch: refs/heads/branch-2
Commit: ed98ee1c195f85db167fcec13ef8c4ea6b6a487f
Parents: 35296e5
Author: Misty Stanley-Jones <mi...@apache.org>
Authored: Wed Aug 9 14:34:46 2017 -0700
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Tue Dec 19 23:40:46 2017 +0800
----------------------------------------------------------------------
CHANGES.txt | 0
LICENSE.txt | 0
NOTICE.txt | 0
README.txt | 0
.../jenkins-scripts/check-website-links.sh | 47 ++++++
.../jenkins-scripts/generate-hbase-website.sh | 152 +++++++++++++++++++
pom.xml | 0
.../appendix_contributing_to_documentation.adoc | 34 ++---
8 files changed, 216 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/README.txt
----------------------------------------------------------------------
diff --git a/README.txt b/README.txt
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/dev-support/jenkins-scripts/check-website-links.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/check-website-links.sh b/dev-support/jenkins-scripts/check-website-links.sh
new file mode 100755
index 0000000..c23abbb
--- /dev/null
+++ b/dev-support/jenkins-scripts/check-website-links.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# This script is designed to run as a Jenkins job, such as at
+# https://builds.apache.org/view/All/job/HBase%20Website%20Link%20Checker/
+#
+# It generates artifacts which the Jenkins job then can mail out and/or archive.
+#
+# We download a specific version of linklint because the original has bugs and
+# is not well maintained.
+#
+# See http://www.linklint.org/doc/inputs.html for linklint options
+
+# Clean up the workspace
+rm -rf *.zip > /dev/null
+rm -rf linklint > /dev/null
+rm -Rf link_report
+
+# This version of linklint fixes some bugs in the now-unmaintained 2.3.5 version
+wget http://ingo-karkat.de/downloads/tools/linklint/linklint-2.3.5_ingo_020.zip
+unzip linklint-2.3.5_ingo_020.zip
+chmod +x linklint/linklint.pl
+
+# Run the checker
+echo "Checking http://hbase.apache.org and saving report to link_report/"
+echo "Excluding /testapidocs/ because some tests use private classes not published in /apidocs/."
+# Check internal structure
+linklint/linklint.pl -http \
+ -host hbase.apache.org \
+ /@ \
+ -skip /testapidocs/@ \
+ -skip /testdevapidocs/@ \
+ -net \
+ -redirect \
+ -no_query_string \
+ -htmlonly \
+ -timeout 30 \
+ -delay 1 \
+ -limit 100000 \
+ -doc link_report
+
+# Detect whether we had errors and act accordingly
+if ! grep -q 'ERROR' link_report/index.html; then
+ echo "Errors found. Sending email."
+ exit 1
+else
+ echo "No errors found. Warnings might be present."
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/dev-support/jenkins-scripts/generate-hbase-website.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh
new file mode 100644
index 0000000..a3f7823
--- /dev/null
+++ b/dev-support/jenkins-scripts/generate-hbase-website.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+
+# This script is meant to run as part of a Jenkins job such as
+# https://builds.apache.org/job/hbase_generate_website/
+#
+# It needs to be built on a Jenkins server with the label git-websites
+#
+# It expects to have the hbase repo cloned to the directory hbase
+#
+# If there is a build error, the Jenkins job is configured to send an email
+
+LOCAL_REPO=${WORKSPACE}/.m2/repo
+# Nuke the local maven repo each time, to start with a known environment
+rm -Rf "${LOCAL_REPO}"
+mkdir -p "${LOCAL_REPO}"
+
+# Clean any leftover files in case we are reusing the workspace
+rm -Rf -- *.patch *.patch.zip hbase/target target *.txt hbase-site
+
+# Set up the environment
+export JAVA_HOME=$JDK_1_8_LATEST__HOME
+export PATH=$JAVA_HOME/bin:$MAVEN_3_3_3_HOME/bin:$PATH
+export MAVEN_OPTS="-XX:MaxPermSize=256m -Dmaven.repo.local=${LOCAL_REPO}"
+
+# Verify the Maven version
+mvn -version
+
+# Save and print the SHA we are building
+CURRENT_HBASE_COMMIT="$(git log --pretty=format:%H -n1)"
+echo "Current HBase commit: $CURRENT_HBASE_COMMIT"
+
+# Clone the hbase-site repo manually so it doesn't trigger spurious
+# commits in Jenkins.
+git clone --depth 1 --branch asf-site https://git-wip-us.apache.org/repos/asf/hbase-site.git
+
+# Figure out the last commit we built the site from, and bail if the build
+# still represents the SHA of HBase master
+cd "${WORKSPACE}/hbase-site" || exit -1
+git log --pretty=%s | grep ${CURRENT_HBASE_COMMIT}
+PUSHED=$?
+echo "PUSHED is $PUSHED"
+
+if [ $PUSHED -eq 0 ]; then
+ echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building."
+ exit 0
+else
+ echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet. $PUSHED"
+fi
+
+# Go to the hbase directory so we can build the site
+cd "${WORKSPACE}/hbase" || exit -1
+
+# This will only be set for builds that are triggered by SCM change, not manual builds
+if [ "$CHANGE_ID" ]; then
+ echo -n " ($CHANGE_ID - $CHANGE_TITLE)"
+fi
+
+# Build and install HBase, then build the site
+echo "Building HBase"
+mvn \
+ -DskipTests \
+ -Dmaven.javadoc.skip=true \
+ --batch-mode \
+ -Dcheckstyle.skip=true \
+ -Dfindbugs.skip=true \
+ --log-file="${WORKSPACE}/hbase-build-log-${CURRENT_HBASE_COMMIT}.txt" \
+ clean install \
+&& mvn clean site \
+ --batch-mode \
+ -DskipTests \
+ --log-file="${WORKSPACE}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt"
+
+status=$?
+if [ $status -ne 0 ]; then
+ echo "Failure: mvn clean site"
+ exit $status
+fi
+
+# Stage the site
+echo "Staging HBase site"
+mvn \
+ --batch-mode \
+ --log-file="${WORKSPACE}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \
+ site:stage
+status=$?
+if [ $status -ne 0 ] || [ ! -d target/staging ]; then
+ echo "Failure: mvn site:stage"
+ exit $status
+fi
+
+# Get ready to update the hbase-site repo with the new artifacts
+cd "${WORKSPACE}/hbase-site" || exit -1
+
+#Remove previously-generated files
+FILES_TO_REMOVE=("hbase-*"
+ "apidocs"
+ "devapidocs"
+ "testapidocs"
+ "testdevapidocs"
+ "xref"
+ "xref-test"
+ "*book*"
+ "*.html"
+ "*.pdf*"
+ "css"
+ "js"
+ "images")
+
+for FILE in "${FILES_TO_REMOVE[@]}"; do
+ echo "Removing ${WORKSPACE}/hbase-site/$FILE"
+ rm -Rf "${FILE}"
+done
+
+# Copy in the newly-built artifacts
+cp -au "${WORKSPACE}"/hbase/target/staging/* .
+
+# If the index.html is missing, bail because this is serious
+if [ ! -f index.html ]; then
+ echo "The index.html is missing. Aborting."
+ exit 1
+else
+ # Add all the changes
+ echo "Adding all the files we know about"
+ git add .
+ # Create the commit message and commit the changes
+ WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT."
+ echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG"
+ git commit -m "${WEBSITE_COMMIT_MSG}" -a
+ # Dump a little report
+ echo "This commit changed these files (excluding Modified files):"
+ git diff --name-status --diff-filter=ADCRTXUB origin/asf-site
+ # Create a patch, which Jenkins can save as an artifact and can be examined for debugging
+ git format-patch --stdout origin/asf-site > "${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch"
+ echo "Change set saved to patch ${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch"
+ # Push the real commit
+ git push origin asf-site || (echo "Failed to push to asf-site. Website not updated." && exit -1)
+ # Create an empty commit to work around INFRA-10751
+ git commit --allow-empty -m "INFRA-10751 Empty commit"
+ # Push the empty commit
+ git push origin asf-site || (echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" && exit -1)
+ echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes."
+ git fetch origin
+ git reset --hard origin/asf-site
+
+ # Zip up the patch so Jenkins can save it
+ cd "${WORKSPACE}" || exit -1
+ zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch"
+fi
+
+#echo "Dumping current environment:"
+#env
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/hbase/blob/ed98ee1c/src/main/asciidoc/_chapters/appendix_contributing_to_documentation.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/appendix_contributing_to_documentation.adoc b/src/main/asciidoc/_chapters/appendix_contributing_to_documentation.adoc
index a11108b..6570c9c 100644
--- a/src/main/asciidoc/_chapters/appendix_contributing_to_documentation.adoc
+++ b/src/main/asciidoc/_chapters/appendix_contributing_to_documentation.adoc
@@ -138,23 +138,23 @@ When you are satisfied with your changes, follow the procedure in
[[website_publish]]
=== Publishing the HBase Website and Documentation
-HBase uses the ASF's `gitpubsub` mechanism.
-. After generating the website and documentation
-artifacts using `mvn clean site site:stage`, check out the `asf-site` repository.
-
-. Remove previously-generated content using the following command:
-+
-----
-rm -rf rm -rf *apidocs* *book* *.html *.pdf* css js
-----
-+
-WARNING: Do not remove the `0.94/` directory. To regenerate them, you must check out
-the 0.94 branch and run `mvn clean site site:stage` from there, and then copy the
-artifacts to the 0.94/ directory of the `asf-site` branch.
-
-. Copy the contents of `target/staging` to the branch.
-
-. Add and commit your changes, and submit a patch for review.
+HBase uses the ASF's `gitpubsub` mechanism. A Jenkins job runs the
+`dev-support/jenkins-scripts/generate-hbase-website.sh` script, which runs the
+`mvn clean site site:stage` against the `master` branch of the `hbase`
+repository and commits the built artifacts to the `asf-site` branch of the
+`hbase-site` repository. When the commit is pushed, the website is redeployed
+automatically. If the script encounters an error, an email is sent to the
+developer mailing list. You can run the script manually or examine it to see the
+steps involved.
+
+[[website_check_links]]
+=== Checking the HBase Website for Broken Links
+
+A Jenkins job runs periodically to check HBase website for broken links, using
+the `dev-support/jenkins-scripts/check-website-links.sh` script. This script
+uses a tool called `linklint` to check for bad links and create a report. If
+broken links are found, an email is sent to the developer mailing list. You can
+run the script manually or examine it to see the steps involved.
=== HBase Reference Guide Style Guide and Cheat Sheet
[2/2] hbase git commit: HBASE-18563 Fix RAT License complaint about
website jenkins scripts
Posted by ch...@apache.org.
HBASE-18563 Fix RAT License complaint about website jenkins scripts
Signed-off-by: Chia-Ping Tsai <ch...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/61ce8639
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/61ce8639
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/61ce8639
Branch: refs/heads/branch-2
Commit: 61ce86399ada0f05886e13f3bdc6cce316d5e562
Parents: ed98ee1
Author: Esteban Gutierrez <es...@apache.org>
Authored: Thu Aug 10 20:08:03 2017 -0500
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Tue Dec 19 23:42:24 2017 +0800
----------------------------------------------------------------------
.../jenkins-scripts/check-website-links.sh | 20 +++++++++++++++++++-
.../jenkins-scripts/generate-hbase-website.sh | 18 ++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/61ce8639/dev-support/jenkins-scripts/check-website-links.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/check-website-links.sh b/dev-support/jenkins-scripts/check-website-links.sh
index c23abbb..1f7cd1c 100755
--- a/dev-support/jenkins-scripts/check-website-links.sh
+++ b/dev-support/jenkins-scripts/check-website-links.sh
@@ -1,4 +1,22 @@
#!/bin/bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
# This script is designed to run as a Jenkins job, such as at
# https://builds.apache.org/view/All/job/HBase%20Website%20Link%20Checker/
@@ -44,4 +62,4 @@ if ! grep -q 'ERROR' link_report/index.html; then
exit 1
else
echo "No errors found. Warnings might be present."
-fi
\ No newline at end of file
+fi
http://git-wip-us.apache.org/repos/asf/hbase/blob/61ce8639/dev-support/jenkins-scripts/generate-hbase-website.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh
index a3f7823..06d160a 100644
--- a/dev-support/jenkins-scripts/generate-hbase-website.sh
+++ b/dev-support/jenkins-scripts/generate-hbase-website.sh
@@ -1,4 +1,22 @@
#!/bin/bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
# This script is meant to run as part of a Jenkins job such as
# https://builds.apache.org/job/hbase_generate_website/