You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@echarts.apache.org by sh...@apache.org on 2021/04/23 09:05:49 UTC
[echarts-website] branch asf-site updated: chore: add link check
workflow
This is an automated email from the ASF dual-hosted git repository.
shenyi pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/echarts-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new bec4e7e chore: add link check workflow
bec4e7e is described below
commit bec4e7e64a8a2d5c7f60c8daf633791eb2261235
Author: pissang <bm...@gmail.com>
AuthorDate: Fri Apr 23 17:05:36 2021 +0800
chore: add link check workflow
---
.github/workflows/check-links.yml | 27 +++++++++++
.gitignore | 1 +
.scripts/checkLinks.js | 99 +++++++++++++++++++++++++++++++++++++++
.scripts/package.json | 10 ++++
.scripts/task.js | 51 ++++++++++++++++++++
5 files changed, 188 insertions(+)
diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml
new file mode 100644
index 0000000..f1efaf4
--- /dev/null
+++ b/.github/workflows/check-links.yml
@@ -0,0 +1,27 @@
+name: Check Links
+
+on:
+ schedule:
+ - cron: '0 10 * * *'
+ repository_dispatch:
+ types: check-links
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ strategy:
+ matrix:
+ node-version: [12.x]
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v1
+ with:
+ registry-url: https://registry.npmjs.org/
+ - name: Check links
+ run: |
+ npm i
+ node checkLinks.js
+ working-directory: .scripts
diff --git a/.gitignore b/.gitignore
index e9491c1..4ac938b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
_*.html
.DS_Store
en/_*.html
+node_modules
package-lock.json
/examples/data/option
/examples/data-gl/option
\ No newline at end of file
diff --git a/.scripts/checkLinks.js b/.scripts/checkLinks.js
new file mode 100644
index 0000000..4364656
--- /dev/null
+++ b/.scripts/checkLinks.js
@@ -0,0 +1,99 @@
+// jsdelivr CDN may have possibility that having 403 links. This scripts check if any resource is 403 and use purge API to update it.
+
+const pathTool = require('path');
+const globby = require('globby');
+const {runTasks} = require('./task');
+const fetch = require('node-fetch').default;
+
+const rootDir = pathTool.resolve(__dirname, '../');
+
+function getCdnUrl(fileUrl) {
+ return `https://cdn.jsdelivr.net/gh/apache/echarts-website@asf-site/${fileUrl}`;
+}
+
+function getPurgeUrl(fileUrl) {
+ return `https://purge.jsdelivr.net/gh/apache/echarts-website@asf-site/${fileUrl}`;
+}
+// Only reject when url is 403. Don't check if url is 404 or other status
+function checkIsUrl403(url) {
+ return fetch(url, {
+ method: 'HEAD'
+ }).then(response => {
+ console.log('Check', url, response.status);
+ if (response.status === 403) {
+ throw '403 URL';
+ }
+ });
+}
+
+function purgeUrl(url) {
+ return fetch(url).then(response => response.json())
+ .then(json => {
+ if (json.success) {
+ console.log('Purge Success', url);
+ }
+ else {
+ console.error('Purge Fail', url);
+ }
+ });
+}
+
+async function find403Urls(files) {
+ const totalLen = files.length;
+ let finished = 0;
+ return (await runTasks(files, async (filePath) => {
+ let is403 = false;
+ try {
+ await checkIsUrl403(getCdnUrl(filePath));
+ }
+ catch {
+ is403 = true;
+ }
+ finished++;
+ console.log(`${finished} / ${totalLen} (${(finished / totalLen * 100).toFixed(0)}%)`);
+
+ return is403 ? filePath : '';
+ }, 10)).filter(a => !!a);
+}
+
+async function run() {
+ const files = await globby([
+ 'zh/**/*',
+ 'examples/**/*',
+ '!examples/en/**/*', // Not check english content because they don't use jsdelivr
+ '!zh/**/*.html',
+ '!examples/**/*.html',
+
+ // It's won't be uploaded to github
+ '!examples/data/option',
+ '!examples/data-gl/option'
+ ], {
+ cwd: rootDir
+ });
+
+ const files403 = await find403Urls(files);
+ let successPurged = 0;
+ if (files403.length) {
+ console.log(`Purging ${files403.length} 403 Urls`);
+
+ const totalLen = files403.length;
+ let finished = 0;
+ await runTasks(files403, async(filePath) => {
+ try {
+ await purgeUrl(getPurgeUrl(filePath));
+ successPurged++;
+ }
+ catch (e) {
+ console.log('Purge Error', filePath);
+ console.log(e);
+ }
+ finished++;
+ console.log(`${finished} / ${totalLen} (${(finished / totalLen * 100).toFixed(0)}%)`)
+ }, 10);
+ }
+ console.log('Finished');
+ console.log(`Found ${files403.length} in ${files.length} is 403`);
+ console.log(`Successed purged ${successPurged}`);
+}
+
+run();
\ No newline at end of file
diff --git a/.scripts/package.json b/.scripts/package.json
new file mode 100644
index 0000000..befa8b9
--- /dev/null
+++ b/.scripts/package.json
@@ -0,0 +1,10 @@
+{
+ "name": "echarts-website",
+ "version": "1.0.0",
+ "dependencies": {
+ "globby": "^11.0.3"
+ },
+ "devDependencies": {
+ "node-fetch": "^2.6.1"
+ }
+}
diff --git a/.scripts/task.js b/.scripts/task.js
new file mode 100644
index 0000000..f2e5f1c
--- /dev/null
+++ b/.scripts/task.js
@@ -0,0 +1,51 @@
+function runTasks(
+ taskParamsLists, createTask, concurrency
+) {
+ concurrency = Math.min(taskParamsLists.length, concurrency);
+ return new Promise((resolve, reject) => {
+ let runningTaskCount = 0;
+ let cursor = 0;
+ let rets = [];
+
+ function finishTask(res, idx) {
+ rets[idx] = res;
+ processNext();
+ }
+
+ function failTask(e) {
+ console.error(e);
+ processNext();
+ }
+
+ function processNext() {
+ runningTaskCount--;
+ addTask();
+
+ if (runningTaskCount === 0) {
+ resolve(rets);
+ }
+ }
+
+ function addTask() {
+ const param = taskParamsLists[cursor];
+ if (param) {
+ const currentTaskIdx = cursor;
+ runningTaskCount++;
+ createTask(param)
+ .then((res) => finishTask(res, currentTaskIdx))
+ .catch(failTask);
+ cursor++;
+ }
+ }
+
+ for (let i = 0; i < concurrency; i++) {
+ addTask();
+ }
+
+ if (!runningTaskCount) {
+ resolve(rets);
+ }
+ });
+}
+
+module.exports.runTasks = runTasks;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@echarts.apache.org
For additional commands, e-mail: commits-help@echarts.apache.org