You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by yu...@apache.org on 2023/05/17 08:02:01 UTC
[incubator-devlake-website] branch main updated: fix: link checker always fails on imgur (#530)
This is an automated email from the ASF dual-hosted git repository.
yumeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake-website.git
The following commit(s) were added to refs/heads/main by this push:
new 702d9f67f7 fix: link checker always fails on imgur (#530)
702d9f67f7 is described below
commit 702d9f67f7a1a505d7597ec0ad70e55d50e98c6b
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Wed May 17 16:01:57 2023 +0800
fix: link checker always fails on imgur (#530)
---
.github/workflows/link-schedule.yaml | 20 +++++------
.github/workflows/pr_ci.yaml | 67 ++++++++++++++----------------------
.lycheeignore | 6 ++++
3 files changed, 41 insertions(+), 52 deletions(-)
diff --git a/.github/workflows/link-schedule.yaml b/.github/workflows/link-schedule.yaml
index e8199587ac..4712c772d4 100644
--- a/.github/workflows/link-schedule.yaml
+++ b/.github/workflows/link-schedule.yaml
@@ -3,13 +3,13 @@ name: Check links daily
on:
workflow_dispatch:
schedule:
- - cron: "0 0 * * *"
+ - cron: "18 0 * * *"
permissions:
issues: write # to create an issue when link check fails
env:
- OUTPUT_FILE: lychee/out.md
+ OUTPUT_FILE: lychee.md
jobs:
build-and-check-links:
@@ -23,10 +23,6 @@ jobs:
uses: actions/setup-node@main
with:
node-version: 17.8.x
- - name: Build Docusaurus website
- run: |
- npm install
- npm run build
- name: Install lychee
env:
LYCHEEVERSION: 0.12.0
@@ -59,13 +55,15 @@ jobs:
LYCHEE_TMP="$(mktemp)"
GITHUB_WORKFLOW_URL="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}?check_suite_focus=true"
- lychee --output ${LYCHEE_TMP} -E -i -n -t 45 --max-concurrency 64 -a 429,401,403 -m 10 -- 'build' '*.md'
- exit_code=$?
- if [ $exit_code -ne 0 ]; then
- mkdir -p "$(dirname -- "${OUTPUT_FILE}")"
- cat "${LYCHEE_TMP}" > "${OUTPUT_FILE}"
+ if ! lychee --output ${OUTPUT_FILE} \
+ -E -i -n -t 5 --max-concurrency 64 -a 429,401,403 -m 10 -f markdown -s http -s https -X HEAD \
+ '**/*.md'
+ then
echo "" >> "${OUTPUT_FILE}" # add a new line
echo "[Full Github Actions output](${GITHUB_WORKFLOW_URL})" >> "${OUTPUT_FILE}"
+ exit 1
+ else
+ rm -f "${OUTPUT_FILE}"
fi
# if lychee run failed, create an issue
- name: Create Issue From File
diff --git a/.github/workflows/pr_ci.yaml b/.github/workflows/pr_ci.yaml
index 7d3ed22073..bd4f7facf6 100644
--- a/.github/workflows/pr_ci.yaml
+++ b/.github/workflows/pr_ci.yaml
@@ -8,32 +8,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- - name: Checkout code
- uses: actions/checkout@main
-
- - name: Use Node.js 17.8.x
- uses: actions/setup-node@main
+ - uses: actions/checkout@v2
with:
- node-version: 17.8.x
- - name: Replace base url
- run: |
- # replace base url to localhost to support change of doc structure
- sed -i 's#https://devlake.apache.org#http://local.devlake.apache.org#g' docusaurus.config.js
- - name: Edit hosts
- run: |
- # why we need this:
- # lychee will ignore links to localhost, so we need to add a new domain to avoid this
- echo "127.0.0.1 local.devlake.apache.org" | sudo tee -a /etc/hosts
- - name: Build Docusaurus website
- run: |
- npm install
- npm run build
- - name: Install http-server
- run: |
- sudo npm install -g http-server
+ fetch-depth: 0
- name: Install lychee
env:
- LYCHEEVERSION: 0.12.0
+ LYCHEEVERSION: 0.13.0
# due to the limitation of Apache, we can't use lycheeverse/lychee-action directly,
# so we re-use some core code from it(install)
run: |
@@ -47,22 +27,27 @@ jobs:
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
run: |
- # to run a local server, so that lychee can check links on it rather than on devlake.apache.org
- sudo http-server ./build -s -p 80 -a 0.0.0.0 &
- # wait for http-server to start
- npm install wait-on
- npx wait-on http://local.devlake.apache.org -t 60000
-
# For parameter description, see https://github.com/lycheeverse/lychee#commandline-parameters
- # -E, --exclude-all-private Exclude all private IPs from checking.
- # -i, --insecure Proceed for server connections considered insecure (invalid TLS)
- # -n, --no-progress Do not show progress bar.
- # -t, --timeout <timeout> Website timeout in seconds from connect to response finished [default:20]
- # --max-concurrency <max-concurrency> Maximum number of concurrent network requests [default: 128]
- # -a --accept <accept> Comma-separated list of accepted status codes for valid links
- # -m, --max-redirects <MAX_REDIRECTS>
-
- # 'build': the site directory to check
- # './*.md': all markdown files in the root directory
-
- lychee -E -i -n -t 45 --max-concurrency 64 -a 401,403 -m 10 -- 'build' '*.md'
+ # test external link for modified files only, the rest are tested by the docusaurus already
+ git diff --name-only ${{ github.event.pull_request.base.sha }}... | grep -P '\.md$' > modified_files || true
+ if [ -s modified_files ]; then
+ # run lychee
+ cat modified_files | xargs -d '\n' -- lychee -E -i -n -t 45 --max-concurrency 64 -a 409,401,403 -m 10 -s http -s https -X HEAD --
+ fi
+ - name: Use Node.js 17.8.x
+ uses: actions/setup-node@main
+ with:
+ node-version: 17.8.x
+ - name: Replace base url
+ run: |
+ # replace base url to localhost to support change of doc structure
+ sed -i 's#https://devlake.apache.org#http://local.devlake.apache.org#g' docusaurus.config.js
+ - name: Edit hosts
+ run: |
+ # why we need this:
+ # lychee will ignore links to localhost, so we need to add a new domain to avoid this
+ echo "127.0.0.1 local.devlake.apache.org" | sudo tee -a /etc/hosts
+ - name: Build Docusaurus website
+ run: |
+ npm install
+ npm run build
diff --git a/.lycheeignore b/.lycheeignore
index 70a3951a4c..b0d2a2d3a6 100644
--- a/.lycheeignore
+++ b/.lycheeignore
@@ -10,3 +10,9 @@ https://merico.feishu.cn.*
.*sample-url.*
https://github.com/dailidong
+
+https://i.imgur.com/*
+https://example.org/*
+https://*.example.org/*
+http://example.org/*
+http://*.example.org/*
\ No newline at end of file