You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ur...@apache.org on 2022/02/11 05:49:02 UTC
[pulsar-site] 01/02: feat: docs migration scripts
This is an automated email from the ASF dual-hosted git repository.
urfree pushed a commit to branch refactor/migrate-scripts
in repository https://gitbox.apache.org/repos/asf/pulsar-site.git
commit 68929d4342943b7f3dd19ca01b0cd32c44854ec0
Author: LiLi <ur...@apache.org>
AuthorDate: Fri Feb 11 11:07:48 2022 +0800
feat: docs migration scripts
Signed-off-by: LiLi <ur...@apache.org>
---
site2/website-next/.gitignore | 1 -
site2/website-next/migration-scripts/.gitignore | 2 +
site2/website-next/migration-scripts/auto.js | 98 ++++++++++++
site2/website-next/migration-scripts/fix-code.js | 41 +++++
site2/website-next/migration-scripts/fix-jsx.js | 7 +
site2/website-next/migration-scripts/fix-kop.js | 44 +++++
site2/website-next/migration-scripts/fix-md.js | 54 +++++++
site2/website-next/migration-scripts/fix-miss.js | 59 +++++++
site2/website-next/migration-scripts/fix-space.js | 54 +++++++
site2/website-next/migration-scripts/fix-tab.js | 97 +++++++++++
site2/website-next/migration-scripts/fix-table.js | 72 +++++++++
.../migration-scripts/fix-tip-note-in-list.js | 27 ++++
.../website-next/migration-scripts/fix-tip-note.js | 81 ++++++++++
.../migration-scripts/migrate-chapter.js | 178 +++++++++++++++++++++
site2/website-next/migration-scripts/migrate-md.js | 45 ++++++
.../migration-scripts/migrate-missing.js | 3 +
.../migration-scripts/migrate-override.js | 3 +
site2/website-next/migration-scripts/migration.js | 8 +
site2/website-next/migration-scripts/migration.sh | 5 +
site2/website-next/migration-scripts/override.js | 44 +++++
20 files changed, 922 insertions(+), 1 deletion(-)
diff --git a/site2/website-next/.gitignore b/site2/website-next/.gitignore
index 2f55ed0..25890cc 100644
--- a/site2/website-next/.gitignore
+++ b/site2/website-next/.gitignore
@@ -20,6 +20,5 @@ yarn-debug.log*
yarn-error.log*
bak/
-migration-scripts/
.latest
.versions
\ No newline at end of file
diff --git a/site2/website-next/migration-scripts/.gitignore b/site2/website-next/migration-scripts/.gitignore
new file mode 100644
index 0000000..89db65e
--- /dev/null
+++ b/site2/website-next/migration-scripts/.gitignore
@@ -0,0 +1,2 @@
+.cache
+/node_modules
\ No newline at end of file
diff --git a/site2/website-next/migration-scripts/auto.js b/site2/website-next/migration-scripts/auto.js
new file mode 100644
index 0000000..84bce81
--- /dev/null
+++ b/site2/website-next/migration-scripts/auto.js
@@ -0,0 +1,98 @@
+const fs = require("fs");
+const path = require("path");
+const lodash = require("lodash");
+const child_process = require("child_process");
+const migrate = require("./migrate-chapter");
+const override = require("./override");
+
+const args = process.argv.slice(2);
+const version = args[0];
+let rerun = version == "rerun";
+
+function run(rerun) {
+ if (rerun) {
+ let cache = fs.readFileSync(path.join(__dirname, ".cache"), "utf8");
+ cache = JSON.parse(cache);
+ console.log(
+ "auto migrate docs for version: ",
+ cache.version,
+ "chapter: ",
+ cache.chapter
+ );
+ migrate(cache.version, cache.chapter, "");
+ return;
+ }
+
+ let version_full = "version-" + version;
+ let src = "../../website/versioned_docs/" + version_full;
+ let dest = "../../website-next/versioned_docs/" + version_full;
+ if (version == "next") {
+ src = "../../docs";
+ dest = "../../website-next/docs";
+ }
+ src = path.join(__dirname, src);
+ dest = path.join(__dirname, dest);
+
+ let sidebar_file = path.join(
+ __dirname,
+ "../../website/versioned_sidebars/" + version_full + "-sidebars.json"
+ );
+ if (version == "next") {
+ sidebar_file = path.join(__dirname, "../../website/sidebars.json");
+ }
+ let sidebar = fs.readFileSync(sidebar_file, "utf8");
+ sidebar = JSON.parse(sidebar);
+
+ sidebar = sidebar[version == "next" ? "docs" : version_full + "-docs"];
+
+ let new_sidebar_file = path.join(
+ __dirname,
+ "../../website-next/versioned_sidebars/" + version_full + "-sidebars.json"
+ );
+ if (version == "next") {
+ new_sidebar_file = path.join(__dirname, "../../website-next/sidebars.json");
+ }
+
+ let chapters = Object.keys(sidebar);
+
+ function migrateChapter() {
+ let new_sidebar = [];
+ let new_sidebar_map = {};
+ if (fs.existsSync(new_sidebar_file)) {
+ new_sidebar = JSON.parse(fs.readFileSync(new_sidebar_file, "utf8"));
+ let key =
+ version == "next" ? "docsSidebar" : version_full + "/docsSidebar";
+ new_sidebar = new_sidebar[key];
+ new_sidebar_map = lodash.keyBy(new_sidebar, "label");
+ }
+
+ let migrated_chapters = Object.keys(new_sidebar_map);
+
+ let need_migration_chapers = lodash.difference(chapters, migrated_chapters);
+
+ let current_chapter =
+ need_migration_chapers.length > 0 ? need_migration_chapers[0] : null;
+ if (current_chapter) {
+ console.log(
+ "auto migrate docs for version: ",
+ version,
+ "chapter: ",
+ current_chapter
+ );
+ migrate(version, current_chapter, "");
+ fs.writeFileSync(
+ path.join(__dirname, ".cache"),
+ JSON.stringify({ version: version, chapter: current_chapter })
+ );
+ } else {
+ console.log(
+ "所有章节已迁移完成，现在执行一次完整的【" + version + "】copy fix"
+ );
+ override(version);
+ }
+ }
+
+ migrateChapter();
+}
+
+run(rerun);
diff --git a/site2/website-next/migration-scripts/fix-code.js b/site2/website-next/migration-scripts/fix-code.js
new file mode 100644
index 0000000..b297473
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-code.js
@@ -0,0 +1,41 @@
+module.exports = (data) => {
+ let nData = data;
+
+ let reg = /^([\t ]*)(>*)([\t ]*)(```.*)((((?!```).)*\n*)+)```/gm; //代码块
+ while ((codeGroup = reg.exec(data))) {
+ let _match = codeGroup[0];
+ let space1 = codeGroup[1];
+ let arrow = codeGroup[2];
+ let space2 = codeGroup[3];
+ let begin = codeGroup[4];
+ let code = codeGroup[5];
+ let prefix = space1 + arrow + space2;
+ if ((code = /[\t ]*(.+\n*)*\S/.exec(code))) {
+ code = code[0]; //剔除纯空白的行首和行尾, 得到纯代码
+ if (code.substr(0, prefix.length) != prefix) {
+ code = prefix + code.replace(/\n(.*)/g, "\n" + prefix + "$1"); //行对齐
+ }
+ }
+ let newCodeBlock =
+ "\n" +
+ prefix +
+ begin +
+ "\n" +
+ prefix +
+ "\n" +
+ code +
+ "\n" +
+ prefix +
+ "\n" +
+ prefix +
+ "```" +
+ "\n";
+ nData = nData.replace(_match, newCodeBlock);
+ }
+ return nData
+ .replace(
+ /(.*)(\s*\n){3,}(\s*```.*)((((?!```).)*\n*)+)```.*(\s*\n)+/g,
+ "$1\n\n$3$4```\n\n"
+ ) //解决多个空行
+ .replace(/^(\s*```.*(((?!```).)*\n*)+)```.*\n+/gm, "$1```\n\n"); //解决多个空行 //解决代码块结束标识符后无空行
+};
diff --git a/site2/website-next/migration-scripts/fix-jsx.js b/site2/website-next/migration-scripts/fix-jsx.js
new file mode 100644
index 0000000..48bce1c
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-jsx.js
@@ -0,0 +1,7 @@
+module.exports = (data) => {
+ return data
+ .replace(/(import Tabs from '@theme\/Tabs';)/g, "````mdx-code-block\n$1")
+ .replace(/(import TabItem from '@theme\/TabItem';)/g, "$1\n````")
+ .replace(/(<Tabs>?)/g, "````mdx-code-block\n$1")
+ .replace(/(<\/Tabs>)/g, "$1\n````");
+};
diff --git a/site2/website-next/migration-scripts/fix-kop.js b/site2/website-next/migration-scripts/fix-kop.js
new file mode 100644
index 0000000..cad71c1
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-kop.js
@@ -0,0 +1,44 @@
+const fs = require("fs");
+const path = require("path");
+const axios = require("axios");
+
+const versionReg = /^v(\d+\.?){3,4}$/gm;
+const dir = "/Users/leo/space/sn/pulsar-hub/protocol-handlers/kop";
+const tags_url =
+ "https://urfreespace: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@api.github.com/repos/streamnative/kop/tags";
+const kop_md_url =
+ "https://urfreespace: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@raw.githubusercontent.com/streamnative/kop/{{version}}/docs/kop.md";
+
+async function fix() {
+ let { data } = await axios({
+ method: "get",
+ url: tags_url,
+ proxy: {
+ host: "127.0.0.1",
+ port: 7890,
+ },
+ });
+ let tags = data
+ .map((item) => item.name)
+ .filter((item) => versionReg.test(item));
+ // console.log(tags);
+ for (let tag of tags) {
+ let url = kop_md_url.replace(/{{version}}/, tag);
+ let { data } = await axios({
+ method: "get",
+ url: url,
+ proxy: {
+ host: "127.0.0.1",
+ port: 7890,
+ },
+ });
+ let _path = path.join(dir, tag.substr(1));
+ if (!fs.existsSync(_path)) {
+ fs.mkdirSync(_path);
+ }
+ data = data.replace(/kop\/blob\/master/g, "kop/blob/" + tag);
+ fs.writeFileSync(path.join(_path, "kop.md"), data);
+ }
+}
+
+fix();
diff --git a/site2/website-next/migration-scripts/fix-md.js b/site2/website-next/migration-scripts/fix-md.js
new file mode 100644
index 0000000..14f15bc
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-md.js
@@ -0,0 +1,54 @@
+const fixTab = require("./fix-tab");
+const fixTipNote = require("./fix-tip-note");
+const fixTable = require("./fix-table");
+const fixCode = require("./fix-code");
+const fixTipNoteInList = require("./fix-tip-note-in-list");
+const fixJSX = require("./fix-jsx");
+
+function fix(data, version) {
+ let reg = new RegExp("id:\\s*version-" + version + "-(incubating-)?");
+ data = fixTable(data);
+ data = fixTab(data);
+ data = data.replace(/(<TabItem.*)\n(.*{@inject:\s*.+?})/g, "$1\n\n$2");
+ data = data.replace(/^(>\s*```\w*)(\s*\n){2,}>/gm, "$1\n>\n>");
+ data = fixTipNote(data);
+ data = fixTipNoteInList(data);
+ data = fixCode(data);
+
+ data = data
+ .replace(reg, "id: ")
+ .replace("id: deploy-docs", "id: deploy-dcos")
+ .replace(/`{4,}/g, "```")
+ .replace(/^(id:\sstandalone)\s*$/gm, "slug: /\n$1")
+ .replace(/<style[\s\S]*?<\/style>/g, "")
+ .replace(/\[(.*)\]\s*\((.*)\.md\)/g, "[$1]($2)")
+ .replace(/\*\*(((?!\*).)+)\*/g, "**$1**") //**macOS* => **macOS**
+ .replace(/\*\*(((?!\*).)+)\*{3}/g, "**$1**") //**macOS*** => **macOS**
+ .replace(/(\w)[\t ]*\n\[/g, "$1 [")
+ .replace(/sidebar_label:\s*(.*)/, 'sidebar_label: "$1"')
+ .replace(/""(.*)""/, '"$1"')
+ .replace(/\[\[pulsar:version_number\]\]/g, "@pulsar:version_number@")
+ .replace(/{{(pulsar:.*?)}}/g, "@$1@")
+ .replace(/(@inject:\s*\w+:)`(\w+)`/g, "$1$2")
+ .replace(/<empty string>/g, "|")
+ .replace(/<li>(((?!<\/?li>|\|).)+)/g, "<li>$1</li>")
+ .replace(/<\/li>\s*<\/li>/g, "</li>")
+ .replace(/\|(((?!<\/?li>|\|).)+)<\/li>/g, "|<li>$1</li>")
+ .replace(/<\/li>\s*<\/li>/g, "</li>") //sometimes needed, no idea for now...
+ .replace(/<\/br>/g, "@AAA@")
+ .replace(/<br>/g, "@AAA@")
+ .replace(/<br\/>/g, "@AAA@")
+ .replace(/<>/g, "")
+ .replace(/@AAA@/g, "<br />")
+ .replace(
+ /<span style="(((?!:).)+):(((?!>).)+);"/g,
+ '<span style={{color: "$3"}}'
+ )
+ .replace(/\s?style=".*?"/g, "")
+ .replace(/\]\(assets\//g, "](/assets/");
+
+ return data;
+ // return fixJSX(data);
+}
+
+module.exports = fix;
diff --git a/site2/website-next/migration-scripts/fix-miss.js b/site2/website-next/migration-scripts/fix-miss.js
new file mode 100644
index 0000000..f20574b
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-miss.js
@@ -0,0 +1,59 @@
+const fs = require("fs");
+const path = require("path");
+
+function fix(docId, versions) {
+ console.log("Fix missing for: ", docId, versions);
+
+ let fixed = false;
+
+ function travel(dir, callback) {
+ fs.readdirSync(dir)
+ .sort(() => -1)
+ .forEach((file) => {
+ var pathname = path.join(dir, file);
+ if (fs.statSync(pathname).isDirectory()) {
+ if (!fixed) {
+ travel(pathname, callback);
+ }
+ } else {
+ if (!fixed) {
+ fixed = callback(pathname);
+ }
+ }
+ });
+ }
+
+ const vdocs = path.join(__dirname, "../../website/versioned_docs");
+ travel(vdocs, (pathname) => {
+ let data = fs.readFileSync(pathname, "utf8");
+ let _match = /^id:\s*version-[\d\.]+-(incubating-)?(.*)/gm.exec(data);
+ // if (_match[1].indexOf(docId) > -1) {
+ if (_match && _match[2] == docId) {
+ console.log(pathname, _match[0]);
+
+ let vs = versions.split(",");
+ vs.forEach((v) => {
+ if (v == "next") {
+ fs.writeFileSync(
+ path.join(vdocs, "../../docs/", docId + ".md"),
+ data.replace(/^id:\s*version-[\d\.]+-(.*)/gm, "id: " + docId),
+ "utf8"
+ );
+ } else {
+ fs.writeFileSync(
+ path.join(vdocs, "version-" + v, docId + ".md"),
+ data.replace(
+ /^id:\s*version-[\d\.]+-(.*)/gm,
+ "id: version-" + v + "-" + docId
+ ),
+ "utf8"
+ );
+ }
+ });
+ return true;
+ }
+ return false;
+ });
+}
+
+module.exports = fix;
diff --git a/site2/website-next/migration-scripts/fix-space.js b/site2/website-next/migration-scripts/fix-space.js
new file mode 100644
index 0000000..56dfd99
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-space.js
@@ -0,0 +1,54 @@
+const fs = require("fs");
+const path = require("path");
+
+const code_map = {};
+let index = 0;
+
+function fixSpace(data) {
+ const code_reg = /(```\w*)((((?!```).)*\n*)+)```/;
+ const _match = code_reg.exec(data);
+ if (_match) {
+ const code = _match[0];
+ const TMP_KEY = "_TMP_CODE_KEY_" + index + "_";
+ code_map[TMP_KEY] = code;
+ index++;
+ data = data.replace(code_reg, TMP_KEY);
+ return fixSpace(data);
+ } else {
+ return data;
+ }
+}
+
+function fixTmp(data) {
+ const tmp_reg = /_TMP_CODE_KEY_(\d+)_/;
+ const _match = tmp_reg.exec(data);
+ if (_match) {
+ data = data.replace(tmp_reg, code_map[_match[0]]);
+ return fixTmp(data);
+ } else {
+ return data;
+ }
+}
+
+function fix(data) {
+ data = fixSpace(data);
+ data = data.replace(/^[\t ]+/gm, "");
+ data = fixTmp(data);
+ return data;
+}
+
+function test() {
+ let data = fs.readFileSync(
+ path.join(__dirname, "../bak/io-quickstart.md"),
+ "utf8"
+ );
+ data = fixSpace(data);
+ data = data.replace(/^[\t ]+/gm, "");
+ data = fixTmp(data);
+ // console.log(data);
+ fs.writeFileSync(path.join(__dirname, "../bak/io-quickstart-fixed.md"), data);
+}
+
+// test();
+
+module.exports = fix;
diff --git a/site2/website-next/migration-scripts/fix-tab.js b/site2/website-next/migration-scripts/fix-tab.js
new file mode 100644
index 0000000..3dd751a
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-tab.js
@@ -0,0 +1,97 @@
+const fs = require("fs");
+const path = require("path");
+const lodash = require("lodash");
+const TAB_REG =
+ /([\t ]*)<!--DOCUSAURUS_CODE_TABS-->(((?!<!--END_DOCUSAURUS_CODE_TABS-->).)*\n*)*<!--END_DOCUSAURUS_CODE_TABS-->/;
+
+function removeTabTag(tab) {
+ return tab
+ .replace(/<!--DOCUSAURUS_CODE_TABS-->/, "")
+ .replace(/<!--END_DOCUSAURUS_CODE_TABS-->/, "");
+}
+
+function findTabItemNameList(tab) {
+ const _match = tab.match(/<!--(.*)-->/g);
+ return _match.map((item) => {
+ return item.replace("<!--", "").replace("-->", "");
+ });
+}
+
+function replaceTabItemTag(tab) {
+ const tab_item_reg = /([\t ]*)<!--(.*)-->((((?![\t ]*<!--).)*\n*)*)/;
+ const _match = tab_item_reg.exec(tab);
+ if (_match) {
+ const space = _match[1];
+ const tab_item_name = _match[2];
+ const tab_item_content = lodash.trimEnd(_match[3]);
+ let tab_item_str = space + '<TabItem value="';
+ tab_item_str +=
+ tab_item_name + '">' + tab_item_content + "\n\n" + space + "</TabItem>\n";
+ tab = tab.replace(tab_item_reg, tab_item_str);
+ return replaceTabItemTag(tab);
+ } else {
+ return tab;
+ }
+}
+
+function importTabComponent(data) {
+ if (!/import Tabs from '@theme\/Tabs';/g.exec(data)) {
+ return data.replace(
+ /---((((?!---).)*\n*)*)---/,
+ "---$1---" +
+ "\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n"
+ );
+ }
+ return data;
+}
+
+function fixTab(data) {
+ const _match = TAB_REG.exec(data);
+ if (_match) {
+ data = importTabComponent(data);
+ let tab = _match[0];
+ tab = removeTabTag(tab);
+ const names = findTabItemNameList(tab);
+ tab = replaceTabItemTag(tab);
+ const names_map = names.map((item) => {
+ return {
+ label: item,
+ value: item,
+ };
+ });
+
+ const space = _match[1];
+ const tab_tag_begin =
+ space +
+ "<Tabs \n" +
+ space +
+ ' defaultValue="' +
+ names[0] +
+ '"\n' +
+ space +
+ " values={" +
+ JSON.stringify(names_map) +
+ "}>";
+
+ const tab_tag_end = "\n" + space + "</Tabs>";
+ tab = tab_tag_begin + tab + tab_tag_end;
+ return fixTab(data.replace(TAB_REG, tab));
+ } else {
+ return data
+ .replace(/(<TabItem.*)\n[\t ]*\n/g, "$1\n")
+ .replace(/(<TabItem.*)/g, "$1\n");
+ }
+}
+
+function test() {
+ let data = fs.readFileSync(
+ path.join(__dirname, "../bak/schema-manage.md"),
+ "utf8"
+ );
+ data = fixTab(data);
+ console.log(data);
+}
+
+// test()
+
+module.exports = fixTab;
diff --git a/site2/website-next/migration-scripts/fix-table.js b/site2/website-next/migration-scripts/fix-table.js
new file mode 100644
index 0000000..ef629f8
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-table.js
@@ -0,0 +1,72 @@
+const fs = require("fs");
+const path = require("path");
+const NL = "\n";
+
+//Convert HTML table to markdown table
+function convertTableElementToMarkdown(table) {
+ let rows = [];
+ let trEls = table.match(/[<]tr[\s\S]*?[/]tr[>]/g);
+ for (let i = 0; i < trEls.length; i++) {
+ let tableRow = trEls[i];
+ let markdownRow = convertTableRowElementToMarkdown(tableRow, i);
+ rows.push(markdownRow);
+ }
+ return rows.join(NL);
+}
+
+//Convert the rows of the HTML table to the rows of the markdown table
+function convertTableRowElementToMarkdown(tableRow, rowNumber) {
+ let cells = [];
+ let cellEls = [];
+ cellEls = tableRow.match(/[<]th[\s\S]*?[/]th[>]/g);
+ if (!cellEls || cellEls.length == 0) {
+ cellEls = tableRow.match(/[<]td[\s\S]*?[/]td[>]/g);
+ }
+ let dividerCells = [];
+ if (rowNumber == 0) {
+ for (i = 0; i < cellEls.length; i++) {
+ dividerCells.push("---" + " |");
+ }
+ dividerCells = "| " + dividerCells.join(" ");
+ }
+ for (let i = 0; i < cellEls.length; i++) {
+ let cell = cellEls[i];
+ cell.indexOf(">") + 1;
+
+ cells.push(
+ cell
+ .substring(cell.indexOf(">") + 1, cell.indexOf("<", cell.indexOf(">")))
+ .replace(/\*\s+([^\|\*]*)/g, "<li>$1</li>") + " | "
+ );
+ }
+ let row = "| " + cells.join(" ");
+ if (rowNumber == 0) {
+ row = row + NL + dividerCells;
+ }
+
+ return row;
+}
+
+function fix(data) {
+ let content = data;
+ let patt1 = /[<]table[\s\S]*?[/]table[>]/g;
+ let tables = content.match(patt1);
+ if (tables) {
+ for (let e of tables) {
+ let et = e.replace(/\s+/g, " ");
+ let markdownTable = convertTableElementToMarkdown(et);
+ content = content.replace(e, markdownTable);
+ }
+ }
+ return content;
+}
+
+function test() {
+ let data = fs.readFileSync(path.join(__dirname, "../bak/txn-how.md"), "utf8");
+ data = fix(data);
+ fs.writeFileSync(path.join(__dirname, "../bak/txn-how-fixed.md"), data);
+}
+
+// test();
+
+module.exports = fix;
diff --git a/site2/website-next/migration-scripts/fix-tip-note-in-list.js b/site2/website-next/migration-scripts/fix-tip-note-in-list.js
new file mode 100644
index 0000000..81a3cba
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-tip-note-in-list.js
@@ -0,0 +1,27 @@
+module.exports = (data) => {
+ let nData = data;
+ let block = null;
+
+ let uListReg = /^([\*-][\t ]+.*)\n+((\n*[\t ]+.*)*)*/gm;
+
+ while ((block = uListReg.exec(data))) {
+ let content = block[2];
+ if (!content) {
+ continue;
+ }
+ let nContent = content.replace(/^ (?= *)/gm, " ");
+ nData = nData.replace(content, nContent);
+ }
+
+ let listReg = /^(\d+\.[\t ]+.*)\n+((\n*[\t ]+.*)*)*/gm;
+ while ((block = listReg.exec(data))) {
+ let content = block[2];
+ if (!content) {
+ continue;
+ }
+ let nContent = content.replace(/^ (?= *)/gm, " ");
+ nData = nData.replace(content, nContent);
+ }
+
+ return nData;
+};
diff --git a/site2/website-next/migration-scripts/fix-tip-note.js b/site2/website-next/migration-scripts/fix-tip-note.js
new file mode 100644
index 0000000..6becf1a
--- /dev/null
+++ b/site2/website-next/migration-scripts/fix-tip-note.js
@@ -0,0 +1,81 @@
+const fs = require("fs");
+const path = require("path");
+
+function fixTip(data) {
+ const _reg =
+ /^([\t ]*)>[\t ]*#*[\t ]*\**(Note|Tip)+s?:?\**[\t ]*\n(([\t ]*)>[\t ]*[^><].*\n)+/gm;
+ const _type_reg = /#*[\t ]*\**(Note|Tip)+s?:?\**[\t ]*/i;
+
+ let type = "note";
+ let _tips = data.match(_reg);
+ if (!_tips) {
+ return data;
+ }
+ for (let _tip of _tips) {
+ let tip = _tip.replace(/^[ ]>/gm, ">").replace(/^([\t ]*>)[\t ]*/gm, "$1");
+ let type_match = _type_reg.exec(tip);
+ if (type_match) {
+ type = type_match[1] + "";
+ type = type.toLowerCase();
+ }
+ let space = /^([\t ]*)>/.exec(tip)[1];
+ // if (space.length % 2 == 1) {
+ // space = space.substr(1);
+ // }
+ tip = tip
+ .replace(_type_reg, "")
+ .replace(/^([\t ]*)>[\t ]*/gm, space)
+ .replace(/^\s*\n/gm, "");
+ tip = space + ":::" + type + "\n\n" + tip + "\n" + space + ":::\n";
+ data = data.replace(_tip, tip);
+ }
+ return data
+ .replace(
+ /(.*)(\s*\n){3,}(\s*:::.*)((((?!:::).)*\n*)+):::.*(\s*\n)+/g,
+ "$1\n\n$3$4:::\n\n"
+ ) //解决多个空行
+ .replace(/(\s*:::.*(((?!:::).)*\n*)+):::.*\n+/g, "$1:::\n\n"); //解决多个空行 //解决代码块结束标识符后无空行
+}
+
+// function fixListWithTip(data) {
+// const _reg = /^([*-]\s+.*)([\s\S]*?)(:::[\s\S]*?:::)/gm;
+// let _match = data.match(_reg);
+// if (!_match) {
+// return data;
+// }
+// for (let _m of _match) {
+// let __m = _m.match(/^([*-]\s+.*)([\s\S]*?)(:::[\s\S]*?:::)/m);
+// let top = __m[1];
+// let seconds = __m[2];
+// let tip = __m[3];
+// // seconds = seconds.replace(/^([\t ]*)(\w+)/gm, "$1* $2");
+// tip = tip.replace(/^[\t ]+/gm, "");
+// data = data
+// .replace(_m, top + seconds + "\n" + tip)
+// .replace(/(\s*\n){3,}:::/g, "\n\n:::")
+// .replace(/:::(\s*\n){3,}/g, ":::\n\n");
+// }
+// return data;
+// }
+
+function fix(data) {
+ data = fixTip(data);
+ // data = fixListWithTip(data);
+ return data;
+}
+
+function test() {
+ let data = fs.readFileSync(
+ path.join(__dirname, "../bak/tiered-storage-overview.md"),
+ "utf8"
+ );
+ data = fix(data);
+ fs.writeFileSync(
+ path.join(__dirname, "../bak/tiered-storage-overview-fixed.md"),
+ data
+ );
+}
+
+// test();
+
+module.exports = fix;
diff --git a/site2/website-next/migration-scripts/migrate-chapter.js b/site2/website-next/migration-scripts/migrate-chapter.js
new file mode 100644
index 0000000..554f1d5
--- /dev/null
+++ b/site2/website-next/migration-scripts/migrate-chapter.js
@@ -0,0 +1,178 @@
+//Usage:
+//Example
+//node scripts/migration.js 2.7.3 "Concepts and Architecture" concepts
+//node scripts/migration.js 2.7.3 "Concepts and Architecture" concepts fix
+
+const fs = require("fs");
+const path = require("path");
+const nextSidebar = require("../sidebars");
+const _ = require("lodash");
+const fixMd = require("./fix-md");
+const fixMissing = require("./fix-miss");
+
+function travel(dir, callback) {
+ fs.readdirSync(dir).forEach((file) => {
+ var pathname = path.join(dir, file);
+ if (fs.statSync(pathname).isDirectory()) {
+ // travel(pathname, callback);
+ } else {
+ callback(pathname);
+ }
+ });
+}
+
+module.exports = function migrate(version, category, prefix) {
+ try {
+ console.log(
+ "migration docs for ",
+ "version: ",
+ version,
+ "category: ",
+ category,
+ "prefix: ",
+ prefix
+ );
+
+ let version_full = "version-" + version;
+ let src = "../../website/versioned_docs/" + version_full;
+ let dest = "../../website-next/versioned_docs/" + version_full;
+ if (version == "next") {
+ src = "../../docs";
+ dest = "../../website-next/docs";
+ }
+ src = path.join(__dirname, src);
+ dest = path.join(__dirname, dest);
+
+ let sidebar_file = path.join(
+ __dirname,
+ "../../website/versioned_sidebars/" + version_full + "-sidebars.json"
+ );
+ if (version == "next") {
+ sidebar_file = path.join(__dirname, "../../website/sidebars.json");
+ }
+ let sidebar = fs.readFileSync(sidebar_file, "utf8");
+ sidebar = JSON.parse(sidebar);
+
+ const _key = version == "next" ? "docs" : version_full + "-docs";
+ sidebar = sidebar[_key][category];
+ if (version != "next") {
+ sidebar = sidebar.map((item) => {
+ return item.substr(version_full.length + 1);
+ });
+ }
+ if (!sidebar) {
+ return;
+ }
+ let new_sidebar_file = path.join(
+ __dirname,
+ "../../website-next/versioned_sidebars/" + version_full + "-sidebars.json"
+ );
+ let new_sidebar = nextSidebar;
+ if (version == "next") {
+ new_sidebar_file = path.join(
+ __dirname,
+ "../../website-next/sidebars.json"
+ );
+ if (!_.keyBy(new_sidebar.docsSidebar, "label")[category]) {
+ new_sidebar.docsSidebar.push({
+ type: "category",
+ label: category,
+ items: sidebar,
+ });
+ }
+ } else {
+ try {
+ new_sidebar = fs.readFileSync(new_sidebar_file, "utf8");
+ new_sidebar = JSON.parse(new_sidebar);
+ } catch {
+ new_sidebar = {
+ [version_full + "/docsSidebar"]: [],
+ };
+ }
+ if (
+ !_.keyBy(new_sidebar[version_full + "/docsSidebar"], "label")[category]
+ ) {
+ new_sidebar[version_full + "/docsSidebar"].push({
+ type: "category",
+ label: category,
+ items: sidebar.map((item) => {
+ return {
+ type: "doc",
+ id:
+ version_full +
+ "/" +
+ (item == "deploy-docs" ? "deploy-dcos" : item),
+ };
+ }),
+ // collapsible: true,
+ // collapsed: true,
+ });
+ }
+ }
+ fs.writeFileSync(new_sidebar_file, JSON.stringify(new_sidebar, null, 2));
+
+ // console.log("path: ", src, dest);
+ let migrated_docs = [];
+ travel(src, function (pathname) {
+ let filename = path.basename(pathname);
+ try {
+ let stat = fs.statSync(dest);
+ if (!stat.isDirectory()) {
+ fs.mkdirSync(dest);
+ }
+ } catch {
+ fs.mkdirSync(dest);
+ }
+ if (
+ sidebar.includes(filename.substr(0, filename.length - 3)) ||
+ (prefix && filename.startsWith(prefix))
+ ) {
+ console.log(filename);
+ migrated_docs.push(filename.substr(0, filename.length - 3));
+ // if (fix) {
+ let data = fs.readFileSync(pathname, "utf8");
+ data = fixMd(data, version);
+ fs.writeFileSync(
+ path.join(
+ dest,
+ filename
+ // filename == "deploy-dcos.md" ? "deploy-docs.md" : filename
+ ),
+ data
+ );
+ // } else {
+ // fs.copyFileSync(pathname, path.join(dest, filename));
+ // }
+ } else {
+ let data = fs.readFileSync(pathname, "utf8");
+ let _match = /^id:\s*version-[\d\.]+-(.*)/gm.exec(data);
+ if (_match && sidebar.includes(_match[1])) {
+ // filename = _match[1] + '.md';
+ // if (fix) {
+ console.log(filename);
+ migrated_docs.push(_match[1]);
+ data = fixMd(data, version);
+ fs.writeFileSync(path.join(dest, filename), data);
+ // } else {
+ // fs.copyFileSync(pathname, path.join(dest, filename));
+ // }
+ }
+ }
+ });
+ let missing_docs = _.difference(sidebar, migrated_docs);
+ if (missing_docs.length > 0) {
+ console.log("fix missing docs:", missing_docs);
+ missing_docs.forEach((docId) => {
+ fixMissing(docId, version);
+ let filename = docId + ".md";
+ let _src = path.join(src, filename);
+ let data = fs.readFileSync(_src, "utf8");
+ data = fixMd(data, version);
+ fs.writeFileSync(path.join(dest, filename), data);
+ fs.unlinkSync(_src);
+ });
+ }
+ } catch (e) {
+ console.error(e);
+ }
+};
diff --git a/site2/website-next/migration-scripts/migrate-md.js b/site2/website-next/migration-scripts/migrate-md.js
new file mode 100644
index 0000000..de1d977
--- /dev/null
+++ b/site2/website-next/migration-scripts/migrate-md.js
@@ -0,0 +1,45 @@
+const fs = require("fs");
+const path = require("path");
+const _ = require("lodash");
+const fixMd = require("./fix-md");
+
+const args = process.argv.slice(2);
+const src = args[0];
+const dest = args[1];
+
+function travel(dir, callback) {
+ fs.readdirSync(dir).forEach((file) => {
+ var pathname = path.join(dir, file);
+ if (fs.statSync(pathname).isDirectory()) {
+ // travel(pathname, callback);
+ } else {
+ callback(pathname);
+ }
+ });
+}
+
+if (!fs.existsSync(src)) {
+ console.log(src, "is not exists.");
+ return;
+}
+
+let stat = fs.statSync(src);
+if (stat.isDirectory()) {
+ stat = fs.statSync(dest);
+ if (!stat.isDirectory()) {
+ fs.mkdirSync(dest);
+ }
+ travel(src, (pathname) => {
+ let data = fs.readFileSync(pathname, "utf8");
+ data = fixMd(data, "blog");
+ data = data.replace(/!\[\]\(\.\.\/img\//g, "![](/img/");
+ fs.writeFileSync(path.join(dest, path.basename(pathname)), data);
+ });
+} else {
+ let data = fs.readFileSync(src, "utf8");
+ data = fixMd(data, "blog");
+ data = data.replace(/!\[\]\(\.\.\/img\//g, "![](/img/");
+ fs.writeFileSync(dest, data);
+}
+
+console.log("Done.");
diff --git a/site2/website-next/migration-scripts/migrate-missing.js b/site2/website-next/migration-scripts/migrate-missing.js
new file mode 100644
index 0000000..9db85a6
--- /dev/null
+++ b/site2/website-next/migration-scripts/migrate-missing.js
@@ -0,0 +1,3 @@
+const fix = require("./fix-missing");
+const args = process.argv.slice(2);
+fix(args[0], args[1]);
diff --git a/site2/website-next/migration-scripts/migrate-override.js b/site2/website-next/migration-scripts/migrate-override.js
new file mode 100644
index 0000000..cb13964
--- /dev/null
+++ b/site2/website-next/migration-scripts/migrate-override.js
@@ -0,0 +1,3 @@
+const fix = require("./override");
+const args = process.argv.slice(2);
+fix(args[0]);
diff --git a/site2/website-next/migration-scripts/migration.js b/site2/website-next/migration-scripts/migration.js
new file mode 100644
index 0000000..39302e2
--- /dev/null
+++ b/site2/website-next/migration-scripts/migration.js
@@ -0,0 +1,8 @@
+const migrate = require("./migrate-chapter");
+
+const args = process.argv.slice(2);
+const version = args[0];
+const category = args[1];
+const prefix = args[2];
+
+migrate(version, category, prefix);
diff --git a/site2/website-next/migration-scripts/migration.sh b/site2/website-next/migration-scripts/migration.sh
new file mode 100644
index 0000000..69b4f3a
--- /dev/null
+++ b/site2/website-next/migration-scripts/migration.sh
@@ -0,0 +1,5 @@
+version=2.7.3
+chapter="Cookbooks"
+prefix="cookbooks"
+
+node bak/migration.js $version "$chapter" $prefix fix
\ No newline at end of file
diff --git a/site2/website-next/migration-scripts/override.js b/site2/website-next/migration-scripts/override.js
new file mode 100644
index 0000000..b75d118
--- /dev/null
+++ b/site2/website-next/migration-scripts/override.js
@@ -0,0 +1,44 @@
+const fs = require("fs");
+const path = require("path");
+const _ = require("lodash");
+const fixMd = require("./fix-md");
+
+function travel(dir, callback) {
+ fs.readdirSync(dir).forEach((file) => {
+ var pathname = path.join(dir, file);
+ if (fs.statSync(pathname).isDirectory()) {
+ // travel(pathname, callback);
+ // } else if (file == "tiered-storage-filesystem.md") {
+ } else {
+ callback(pathname);
+ }
+ });
+}
+
+function fix(version) {
+ let version_full = "version-" + version;
+ let src = "../../website/versioned_docs/" + version_full;
+ let dest = "../../website-next/versioned_docs/" + version_full;
+ if (version == "next") {
+ src = "../../docs";
+ dest = "../../website-next/docs";
+ }
+ src = path.join(__dirname, src);
+ dest = path.join(__dirname, dest);
+
+ travel(src, function (pathname) {
+ let filename = path.basename(pathname);
+ try {
+ fs.statSync(dest);
+ } catch {
+ fs.mkdirSync(dest);
+ }
+ console.log(filename);
+ let data = fs.readFileSync(pathname, "utf8");
+ data = fixMd(data, version);
+ // filename = filename == "deploy-dcos.md" ? "deploy-docs.md" : filename;
+ fs.writeFileSync(path.join(dest, filename), data);
+ });
+}
+
+module.exports = fix;