You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by ju...@apache.org on 2022/04/18 09:38:30 UTC
[apisix-website] branch master updated: refactor: reduce build time (#1030)
This is an automated email from the ASF dual-hosted git repository.
juzhiyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix-website.git
The following commit(s) were added to refs/heads/master by this push:
new d7abc6b757b refactor: reduce build time (#1030)
d7abc6b757b is described below
commit d7abc6b757bf2a914a79ad81d960b52061504330
Author: Young <is...@outlook.com>
AuthorDate: Mon Apr 18 17:38:26 2022 +0800
refactor: reduce build time (#1030)
---
.github/workflows/deploy.yml | 31 ++++
.github/workflows/link-check.yml | 2 +-
package.json | 3 +-
scripts/generate-repos-info.js | 13 +-
scripts/sync-docs.js | 270 ++++++++++++++++-----------------
website/config/apisix-versions.js | 49 ++++++
website/src/theme/DocSidebar/index.tsx | 3 +-
yarn.lock | 2 +-
8 files changed, 224 insertions(+), 149 deletions(-)
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 933938393df..03c936868a1 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -28,6 +28,28 @@ jobs:
with:
node-version: "14"
+ - name: Get node version
+ id: node-version
+ run: |
+ echo "::set-output name=ver::$(node --version)"
+
+ - name: Get yarn cache directory path
+ id: yarn-cache-dir-path
+ run: echo "::set-output name=dir::$(yarn cache dir)"
+
+ # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+ - uses: actions/cache@v3
+ id: yarn-cache
+ with:
+ path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+ key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+
+ - name: Cache install
+ uses: actions/cache@v3
+ with:
+ path: ./node_modules
+ key: ${{ runner.os }}-dep-${{ steps.node-version.outputs.ver }}-${{ hashFiles('**/yarn.lock') }}
+
- name: Install Dependencies
run: |
yarn install
@@ -36,6 +58,15 @@ jobs:
run: |
yarn sync-doc && yarn generate-repos-info && git status
+ - name: Apply docusaurus cache
+ id: docusaurus-cache
+ uses: actions/cache@v3
+ with:
+ path: |
+ ./website/.docusaurus
+ ./website/build
+ key: ${{ runner.os }}-dep-${{ steps.node-version.outputs.ver }}-${{ hashFiles('./website/docs/apisix/**') }}
+
- name: Build
run: |
yarn build && cp ./.asf.yaml ./website/build
diff --git a/.github/workflows/link-check.yml b/.github/workflows/link-check.yml
index 0ceba87707c..b6a60f38a57 100644
--- a/.github/workflows/link-check.yml
+++ b/.github/workflows/link-check.yml
@@ -26,7 +26,7 @@ jobs:
- uses: actions/checkout@v3.0.1
- uses: actions/setup-node@v3
with:
- node-version: "12"
+ node-version: "14"
- name: Install Dependencies
run: |
diff --git a/package.json b/package.json
index 13e5ba0a2cf..44135af153b 100644
--- a/package.json
+++ b/package.json
@@ -14,6 +14,7 @@
"start": "yarn workspace website start",
"build": "yarn workspace website build",
"docusaurus": "yarn workspace website docusaurus",
+ "serve": "yarn docusaurus serve",
"prepare": "husky install"
},
"devDependencies": {
@@ -36,4 +37,4 @@
"*.{yml,yaml}": "eslint --cache --fix",
"*.css": "stylelint --cache --fix"
}
-}
+}
\ No newline at end of file
diff --git a/scripts/generate-repos-info.js b/scripts/generate-repos-info.js
index cf665a42f4a..f0f82262931 100644
--- a/scripts/generate-repos-info.js
+++ b/scripts/generate-repos-info.js
@@ -1,4 +1,4 @@
-const fs = require('fs');
+const fs = require('fs/promises');
const axios = require('axios');
const Listr = require('listr');
@@ -59,16 +59,15 @@ const tasks = new Listr([
},
]),
})),
+ { concurrent: repoList.length },
),
},
{
title: `Save repos' info and good first issues to json file`,
- task: () => {
- fs.writeFileSync(
- '../website/config/repos-info.json',
- JSON.stringify(res),
- );
- },
+ task: () => fs.writeFile(
+ '../website/config/repos-info.json',
+ JSON.stringify(res),
+ ),
},
]);
diff --git a/scripts/sync-docs.js b/scripts/sync-docs.js
index 79f4bddad81..53b00b46c5a 100644
--- a/scripts/sync-docs.js
+++ b/scripts/sync-docs.js
@@ -1,26 +1,64 @@
const childProcess = require('child_process');
-const fs = require('fs');
+const fs = require('fs/promises');
const path = require('path');
const process = require('process');
const os = require('os');
const Listr = require('listr');
const simpleGit = require('simple-git');
const semver = require('semver');
+const replace = require('replace-in-file');
+const { promisify } = require('util');
const common = require('./common.js');
+const { versions } = require('../website/config/apisix-versions.js');
-const git = simpleGit();
const { projects, languages, projectPaths } = common;
+const exec = promisify(childProcess.exec);
const tempPath = './tmp';
const websitePath = '../website';
-
+const gitMap = {};
const projectReleases = {};
+
+const extractDocsHeadTasks = (project, version) => ([
+ {
+ title: `Checkout ${project.name} version: ${version}`,
+ task: () => gitMap[project.name].cwd(`${tempPath}/${project.name}/`).checkout(`remotes/origin/release/${version}`, ['-f']),
+ },
+ {
+ title: 'Replace elements inside MD files',
+ task: async () => {
+ const branchName = `release/${version}`;
+ await replaceMDElements(project.name, [`${tempPath}/${project.name}/docs`], branchName);
+ await copyAllDocs(project);
+ },
+ },
+]);
+
+const extractDocsTailTasks = (project, version) => ([
+ {
+ title: 'Add English & Chinese documents',
+ task: () => Promise.all([
+ exec(
+ `yarn docusaurus docs:version:docs-${project.name} ${version}`,
+ { cwd: websitePath },
+ ),
+ isDirExisted(`./${tempPath}/${project.name}/docs/zh/latest`)
+ .then(
+ (exist) => exist && copyFolder(
+ project.latestDocs.zh,
+ `${websitePath}/i18n/zh/docusaurus-plugin-content-docs-docs-${project.name}/version-${version}`,
+ ),
+ ),
+ ]),
+ },
+]);
+
const tasks = new Listr([
{
title: 'Start documents sync',
- task: () => {
- removeFolder(tempPath);
- fs.mkdirSync(tempPath);
+ task: async () => {
+ await removeFolder(tempPath);
+ await fs.mkdir(tempPath);
},
},
{
@@ -29,19 +67,20 @@ const tasks = new Listr([
const gitTasks = projects.map((project) => ({
title: `Clone ${project.name} repository`,
task: async () => {
- await git.clone(`https://github.com/apache/${project.name}.git`, `${tempPath}/${project.name}/`);
+ gitMap[project.name] = simpleGit();
+ await gitMap[project.name].clone(`https://github.com/apache/${project.name}.git`, `${tempPath}/${project.name}/`);
},
}));
- return new Listr(gitTasks, { concurrent: 3 });
+ return new Listr(gitTasks, { concurrent: projects.length });
},
},
{
title: 'Find project release',
- task: async () => {
+ task: () => {
const findReleaseTasks = projects.map((project) => ({
title: `Find ${project.name} releases`,
task: async () => {
- const ret = await git.cwd(`${tempPath}/${project.name}/`).branch();
+ const ret = await gitMap[project.name].cwd(`${tempPath}/${project.name}/`).branch();
if (ret.all) {
projectReleases[project.name] = ret.all
.filter((release) => release.includes('remotes/origin/release/'))
@@ -50,7 +89,7 @@ const tasks = new Listr([
}
},
}));
- return new Listr(findReleaseTasks);
+ return new Listr(findReleaseTasks, { concurrent: projects.length });
},
},
{
@@ -59,54 +98,31 @@ const tasks = new Listr([
const extractDocumentTasks = projectPaths.map((project) => ({
title: `Extract ${project.name} documents`,
task: () => {
- const extractProjectTasks = projectReleases[project.name].map((version) => ({
- title: `Extract ${project.name} ${version} documents`,
- task: () => new Listr([
- {
- title: `Checkout ${project.name} version: ${version}`,
- task: async () => {
- await git.cwd(`${tempPath}/${project.name}/`).checkout(`remotes/origin/release/${version}`, ['-f']);
- },
- },
- {
- title: 'Replace elements inside MD files',
- task: () => {
- const branchName = `release/${version}`;
- replaceMDElements(project.name, [`${tempPath}/${project.name}/docs`], branchName);
- copyAllDocs(project);
+ const extractProjectTasks = project.name === 'apisix'
+ ? versions.map((version) => ({
+ title: `Extract ${project.name} ${version} documents`,
+ task: () => new Listr([
+ ...extractDocsHeadTasks(project, version),
+ {
+ title: 'Generate API docs for APISIX',
+ enabled: () => os.platform() === 'linux' && isFileExisted(`./${tempPath}/${project.name}/autodocs`),
+ task: () => generateAPIDocs(project),
},
- },
- {
- title: 'Generate API docs for APISIX',
- enabled: () => os.platform() === 'linux' && project.name === 'apisix' && isFileExisted(`./${tempPath}/${project.name}/autodocs`),
- task: () => generateAPIDocs(project),
- },
- {
- title: 'Add English documents',
- task: () => {
- childProcess.execSync(
- `npm run docusaurus docs:version:docs-${project.name} ${version}`,
- { cwd: websitePath },
- );
- },
- },
- {
- title: 'Add Chinese documents',
- task: () => {
- if (isFileExisted(`./${tempPath}/${project.name}/docs/zh/latest`) !== false) {
- copyFolder(
- project.latestDocs.zh,
- `${websitePath}/i18n/zh/docusaurus-plugin-content-docs-docs-${project.name}/version-${version}`,
- );
- }
- },
- },
- ]),
- }));
+ ...extractDocsTailTasks(project, version),
+ ]),
+ }))
+ : projectReleases[project.name].map((version) => ({
+ title: `Extract ${project.name} ${version} documents`,
+ task: () => new Listr([
+ ...extractDocsHeadTasks(project, version),
+ ...extractDocsTailTasks(project, version),
+ ]),
+ }));
return new Listr(extractProjectTasks);
},
}));
- return new Listr(extractDocumentTasks);
+
+ return new Listr(extractDocumentTasks, { concurrent: projects.length });
},
},
{
@@ -119,17 +135,13 @@ const tasks = new Listr([
const steps = [
{
title: `Checkout ${project.name} next version`,
- task: () => new Promise((resolve) => {
- git.cwd(`${tempPath}/${project.name}/`).checkout(`remotes/origin/${project.branch}`, ['-f']).then(() => {
- resolve();
- });
- }),
+ task: () => gitMap[project.name].cwd(`${tempPath}/${project.name}/`).checkout(`remotes/origin/${project.branch}`, ['-f']),
},
{
title: 'Replace elements inside MD files',
- task: () => {
- replaceMDElements(project.name, [`${tempPath}/${project.name}/docs`], project.branch);
- copyAllDocs(project);
+ task: async () => {
+ await replaceMDElements(project.name, [`${tempPath}/${project.name}/docs`], project.branch);
+ await copyAllDocs(project);
},
},
{
@@ -141,14 +153,12 @@ const tasks = new Listr([
return new Listr(steps);
},
}));
- return new Listr(nextVersionTasks);
+ return new Listr(nextVersionTasks, { concurrent: projects.length });
},
},
{
title: 'Clean temporary files',
- task: () => {
- removeFolder(tempPath);
- },
+ task: () => removeFolder(tempPath),
},
]);
@@ -163,14 +173,10 @@ tasks.run()
// eslint-disable-next-line @typescript-eslint/no-unused-vars
function log(text) {
+ // console.log(text);
}
-function isFileExisted(p) {
- return fs.existsSync(p);
-}
-
-function replaceMDElements(project, path, branch = 'master') {
- const replace = require('replace-in-file');
+async function replaceMDElements(project, path, branch = 'master') {
const allMDFilePaths = path.map((p) => `${p}/**/*.md`);
// replace the image urls inside markdown files
@@ -182,7 +188,7 @@ function replaceMDElements(project, path, branch = 'master') {
to: (match) => {
const imgPath = match.replace(/\(|\)|\.\.\/*/g, '');
const newUrl = `https://raw.githubusercontent.com/apache/${project}/${branch}/docs/${imgPath}`;
- // console.log(`${project}: ${match} 👉 ${newUrl}`);
+ log(`${project}: ${match} 👉 ${newUrl}`);
return newUrl;
},
};
@@ -204,97 +210,85 @@ function replaceMDElements(project, path, branch = 'master') {
const projectNameWithoutPrefix = project === 'apisix' ? 'apisix' : project.replace('apisix-', '');
const newUrl = match.replace(
/\]\(.*\)/g,
- `](https://apisix.apache.org${lang !== 'en' ? `/${lang}` : ''
- }/docs/${projectNameWithoutPrefix}/${urlPath})`,
+ `](https://apisix.apache.org${lang === 'en' ? '' : `/${lang}`}/docs/${projectNameWithoutPrefix}/${urlPath})`,
);
log(`${project}: ${match} 👉 ${newUrl}`);
return newUrl;
},
};
- try {
- replace.sync(imageOptions);
- replace.sync(markdownOptions);
- } catch (error) {
- console.error(`${project} - Error occurred:`, error);
- }
+ await replace(imageOptions);
+ await replace(markdownOptions);
}
-function removeFolder(tarDir) {
- if (!fs.existsSync(tarDir)) return;
+async function isFileExisted(p) {
+ return fs.stat(p).then((v) => v.isFile()).catch(() => false);
+}
- const files = fs.readdirSync(tarDir);
- files.forEach((file) => {
- const tarPath = path.join(tarDir, file);
- const stats = fs.statSync(tarPath);
- if (stats.isDirectory()) {
- removeFolder(tarPath);
- } else {
- fs.unlinkSync(tarPath);
- }
- });
+async function isDirExisted(p) {
+ return fs.stat(p).then((v) => v.isDirectory()).catch(() => false);
+}
- fs.rmdirSync(tarDir);
+async function removeFolder(tarDir) {
+ if (await isDirExisted()) return;
+ await fs.rm(tarDir, { recursive: true, force: true });
}
-function copyFolder(srcDir, tarDir) {
- const files = fs.readdirSync(srcDir);
- if (isFileExisted(tarDir) === false) {
- fs.mkdirSync(tarDir, () => log(`create directory ${tarDir}`));
- }
- files.forEach((file) => {
+async function copyFolder(srcDir, tarDir) {
+ const [files] = await Promise.all([
+ fs.readdir(srcDir),
+ fs.mkdir(tarDir, { recursive: true }),
+ ]);
+
+ return Promise.all(files.map(async (file) => {
const srcPath = path.join(srcDir, file);
const tarPath = path.join(tarDir, file);
+ const stats = await fs.stat(srcPath);
- const stats = fs.statSync(srcPath);
- if (stats.isDirectory()) {
- if (!fs.existsSync(tarPath)) {
- fs.mkdirSync(tarPath);
- }
- copyFolder(srcPath, tarPath);
- } else {
- fs.copyFileSync(srcPath, tarPath);
- }
- });
+ return stats.isDirectory()
+ ? copyFolder(srcPath, tarPath)
+ : fs.copyFile(srcPath, tarPath);
+ }));
}
-function copyDocs(source, target, projectName, locale) {
- if (isFileExisted(`${source}/${locale}/latest`) === false) {
+async function copyDocs(source, target, projectName, locale) {
+ if (!await isDirExisted(`${source}/${locale}/latest`)) {
log(`[${projectName}] can not find ${locale} latest folder, skip.`);
return;
}
log(`[${projectName}] load ${locale} latest docs config.json`);
const configLatest = JSON.parse(
- fs.readFileSync(`${source}/${locale}/latest/config.json`),
+ await fs.readFile(`${source}/${locale}/latest/config.json`),
);
log(`[${projectName}] delete ${locale} docs config.json`);
- fs.unlinkSync(`${source}/${locale}/latest/config.json`);
-
log(`[${projectName}] copy latest ${locale} docs to ${target}`);
- copyFolder(`${source}/${locale}/latest/`, target);
-
log(`[${projectName}] write sidebar.json`);
- const sidebar = {
- docs: [...(configLatest.sidebar || [])],
- };
- fs.writeFileSync(`${target}/sidebars.json`, JSON.stringify(sidebar, null, 2));
+ await Promise.all([
+ fs.unlink(`${source}/${locale}/latest/config.json`),
+ copyFolder(`${source}/${locale}/latest/`, target),
+ fs.writeFile(`${target}/sidebars.json`, JSON.stringify({
+ docs: [...(configLatest.sidebar || [])],
+ }, null, 2)),
+ ]);
}
-function copyAllDocs(project) {
- copyDocs(
- `${tempPath}/${project.name}/docs`,
- project.latestDocs.en,
- project.name,
- 'en',
- );
- copyDocs(
- `${tempPath}/${project.name}/docs`,
- project.latestDocs.zh,
- project.name,
- 'zh',
- );
+async function copyAllDocs(project) {
+ await Promise.all([
+ copyDocs(
+ `${tempPath}/${project.name}/docs`,
+ project.latestDocs.en,
+ project.name,
+ 'en',
+ ),
+ copyDocs(
+ `${tempPath}/${project.name}/docs`,
+ project.latestDocs.zh,
+ project.name,
+ 'zh',
+ ),
+ ]);
}
/**
@@ -315,9 +309,9 @@ function generateAPIDocs(project) {
},
{
title: 'Copy API docs',
- task: () => {
- if (isFileExisted(`./${tempPath}/${project.name}/autodocs/output`) !== false) {
- copyFolder(`${tempPath}/${project.name}/autodocs/output`, `${project.latestDocs.en}/pdk-docs`);
+ task: async () => {
+ if (await isFileExisted(`./${tempPath}/${project.name}/autodocs/output`)) {
+ await copyFolder(`${tempPath}/${project.name}/autodocs/output`, `${project.latestDocs.en}/pdk-docs`);
}
},
},
diff --git a/website/config/apisix-versions.js b/website/config/apisix-versions.js
new file mode 100644
index 00000000000..d3cb603a0cc
--- /dev/null
+++ b/website/config/apisix-versions.js
@@ -0,0 +1,49 @@
+/* eslint-disable quote-props */
+
+/**
+ * @type {Array<string>} version list
+ */
+const versions = [
+ '2.12',
+ '2.13',
+];
+
+/**
+ * @type {Array<{label: string, href: string}>}
+ */
+const archivedVersions = [
+ {
+ label: '2.11',
+ href: 'https://625a9090d04b9a6953165811--2-11-old-docs-apache-apisix.netlify.app/docs/apisix/getting-started/',
+ },
+ {
+ label: '2.10',
+ href: 'https://625a9090d04b9a6953165811--2-11-old-docs-apache-apisix.netlify.app/docs/apisix/2.10/getting-started/',
+ },
+ {
+ label: '2.9',
+ href: 'https://625a57e513f19e48ae3a4468--old-docs-apache-apisix.netlify.app/docs/apisix/getting-started/',
+ },
+ {
+ label: '2.8',
+ href: 'https://625a57e513f19e48ae3a4468--old-docs-apache-apisix.netlify.app/docs/apisix/2.8/getting-started/',
+ },
+ {
+ label: '2.7',
+ href: 'https://625a57e513f19e48ae3a4468--old-docs-apache-apisix.netlify.app/docs/apisix/2.7/getting-started/',
+ },
+ {
+ label: '2.6',
+ href: 'https://625a57e513f19e48ae3a4468--old-docs-apache-apisix.netlify.app/docs/apisix/2.6/getting-started/',
+ },
+ {
+ label: '2.5',
+ href: 'https://625a57e513f19e48ae3a4468--old-docs-apache-apisix.netlify.app/docs/apisix/2.5/getting-started/',
+ },
+ {
+ label: '2.4',
+ href: 'https://625a57e513f19e48ae3a4468--old-docs-apache-apisix.netlify.app/docs/apisix/2.4/getting-started/',
+ },
+];
+
+module.exports = { versions, archivedVersions };
diff --git a/website/src/theme/DocSidebar/index.tsx b/website/src/theme/DocSidebar/index.tsx
index 127ce92528f..c52c333e292 100644
--- a/website/src/theme/DocSidebar/index.tsx
+++ b/website/src/theme/DocSidebar/index.tsx
@@ -23,6 +23,7 @@ import { translate } from '@docusaurus/Translate';
import { DocSidebarItems } from '@theme/DocSidebarItem';
import DocsVersionDropdownNavbarItem from '@theme/NavbarItem/DocsVersionDropdownNavbarItem';
import type { Props } from '@theme/DocSidebar';
+import { archivedVersions } from '../../../config/apisix-versions';
import styles from './styles.module.css';
@@ -68,7 +69,7 @@ const DocsVersionWrapper = (props: {docsPluginId: string}) => {
<DocsVersionDropdownNavbarItem
docsPluginId={docsPluginId}
dropdownItemsBefore={[]}
- dropdownItemsAfter={[]}
+ dropdownItemsAfter={archivedVersions}
items={[]}
/>
</div>
diff --git a/yarn.lock b/yarn.lock
index 33e2992506f..3b5e23fc216 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -9627,7 +9627,7 @@ style-to-object@0.3.0, style-to-object@^0.3.0:
dependencies:
inline-style-parser "0.1.1"
-styled-components@^5, styled-components@^5.3.3:
+styled-components@^5.3.3:
version "5.3.5"
resolved "https://registry.yarnpkg.com/styled-components/-/styled-components-5.3.5.tgz#a750a398d01f1ca73af16a241dec3da6deae5ec4"
integrity sha512-ndETJ9RKaaL6q41B69WudeqLzOpY1A/ET/glXkNZ2T7dPjPqpPCXXQjDFYZWwNnE5co0wX+gTCqx9mfxTmSIPg==