You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shenyu.apache.org by de...@apache.org on 2022/06/26 04:57:01 UTC
[incubator-shenyu-website] branch main updated: [type:feature] Update ci for pdf generation (#619)
This is an automated email from the ASF dual-hosted git repository.
dengliming pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-shenyu-website.git
The following commit(s) were added to refs/heads/main by this push:
new 25bf0d5d66 [type:feature] Update ci for pdf generation (#619)
25bf0d5d66 is described below
commit 25bf0d5d66e47e33164e09c4ca6376a96a1b2fac
Author: Jiageng <sk...@outlook.com>
AuthorDate: Sun Jun 26 12:56:55 2022 +0800
[type:feature] Update ci for pdf generation (#619)
* feat: add .py script to support index.rst generation
* fix: enable generate-pdf.sh(OTF download & project structure)
* fix: delete unnecessary mkdir(public/pdf)
* add: ignore pdf folder
* ci: add pdf generation
* ci: fix about SourceHanSansHC, origin zip format not valid
* fix: invoke prepare
---
.github/scripts/generate-pdf.py | 192 ++++++++++++++++++++++++++++++++++++++++
.github/scripts/generate-pdf.sh | 84 ++++--------------
.github/workflows/deploy.yml | 26 ++++--
.gitignore | 1 +
4 files changed, 232 insertions(+), 71 deletions(-)
diff --git a/.github/scripts/generate-pdf.py b/.github/scripts/generate-pdf.py
new file mode 100644
index 0000000000..48507d8558
--- /dev/null
+++ b/.github/scripts/generate-pdf.py
@@ -0,0 +1,192 @@
+
+from io import TextIOWrapper
+import os
+import json
+from functools import reduce
+from typing import Dict, List, Tuple
+
+class Document:
+ def __init__(self, path: str) -> None:
+ self.path = path
+ if len(self.path.split("/")) == 1:
+ self.relative_path = None
+ elif len(self.path.split("/")) == 2:
+ self.relative_path = self.path.split("/")[-1]
+ else:
+ self.relative_path = reduce(lambda x,y : f"{x}/{y}", self.path.split("/")[1:-1])
+ self.name = self.path.split("/")[-1]
+
+ @property
+ def rankFromFile(self) -> int:
+ return 1
+
+ def dumps(self) -> Dict:
+ return {}
+
+ def writeIndexRst(self) -> None:
+ pass
+
+
+class Md(Document):
+ def __init__(self, path: str, rank: int) -> None:
+ super().__init__(path)
+ self.rank = rank
+
+ @property
+ def rankFromFile(self) -> int:
+ with open(self.path, mode="r") as f:
+ lines = f.readlines()
+ lines = [line for line in lines if line.startswith("sidebar_position: ")]
+ if not lines:
+ raise Exception(f"{self.path} do not have sidebar_position")
+ # if "sidebar_position: 1", return 1
+ return int(lines[0].split(":")[-1].strip())
+
+ def dumps(self) -> Dict:
+ return {
+ "name": self.name,
+ "path": self.path,
+ "rank": self.rank
+ }
+
+ def writeIndexRst(self) -> None:
+ pass
+
+
+class Folder(Document):
+ def __init__(self, path: str) -> None:
+ super().__init__(path)
+ self.props = None
+
+ files_name: List[str] = os.listdir(self.path)
+ files_full_name = [os.path.join(self.path, file_name) for file_name in files_name]
+
+ folders_name = [name for name in files_full_name if os.path.isdir(name)]
+ md_files_name = [name for name in files_full_name if name.endswith(".md") and os.path.isfile(name)]
+
+ self.folders: List[Folder] = []
+ self.md_files: List[Md] = []
+ self.sequence: List[Document] = [None] * (len(folders_name) + len(md_files_name))
+
+ for folder_name in folders_name:
+ # append folder
+ folder = Folder(folder_name)
+ self.folders.append(folder)
+ self.sequence[folder.rankFromFile - 1] = folder
+
+ for md_file_name in md_files_name:
+ idx = Folder.getFirstPosEmpty(self.sequence)
+ file = Md(md_file_name, rank=idx+1)
+ self.md_files.append(file)
+ if idx < len(self.sequence):
+ self.sequence[idx] = file
+
+ @staticmethod
+ def getFirstPosEmpty(l: List):
+ for idx, elem in enumerate(l):
+ if not elem:
+ return idx
+ return len(l)
+
+ @property
+ def rankFromFile(self) -> int:
+ if not self.props:
+ self.loadCategory()
+ if "position" not in self.props:
+ raise Exception(f"{self.path}/_category_.json do not contain position")
+ return int(self.props["position"])
+
+ @property
+ def labelFromFile(self) -> str:
+ if not self.props:
+ self.loadCategory()
+ if "label" not in self.props:
+ raise Exception(f"{self.path}/_category_.json do not contain label")
+ return self.props["label"]
+
+ def loadCategory(self):
+ try:
+ with open(f"{self.path}/_category_.json", mode="r") as f:
+ res = f.read()
+ self.props = json.loads(res)
+ except FileNotFoundError as e:
+ # only top don't include _category_.json
+ self.props = {"label": "Top", "position": 1}
+
+ def dumps(self) -> Dict:
+ return {
+ "name": self.name,
+ "path": self.path,
+ "rank": self.rankFromFile,
+ "label": self.labelFromFile,
+ "contents": [doc.dumps() for doc in self.sequence]
+ }
+
+ def writeIndexRst(self) -> None:
+ docs_name = []
+ for doc in self.sequence:
+ if type(doc) == Folder:
+ docs_name.append(f"./{doc.name}/index")
+ print(doc.dumps())
+ doc.writeIndexRst()
+ elif type(doc) == Md:
+ docs_name.append(f"./{doc.name}")
+ docs_name = [f" {doc_name}\n" for doc_name in docs_name]
+
+ print(docs_name, self.labelFromFile)
+ index_pst_path = f"source/{self.relative_path}/index.rst" if self.relative_path else "source/index.rst"
+ with open(index_pst_path, mode="w") as f:
+ f.writelines([
+ "============================\n",
+ self.labelFromFile + "\n",
+ "============================\n",
+ ".. toctree::\n",
+ " :maxdepth: 1\n",
+ " :titlesonly:\n"
+ "\n"
+ ] + docs_name)
+
+
+class TopDocument:
+ def __init__(self, path: str) -> None:
+ self.path = path
+ self.sections: List[Document] = []
+ self.documents: List[str] = []
+
+ files_name: List[str] = os.listdir(self.path)
+ files_full_name = [os.path.join(self.path, file_name) for file_name in files_name]
+ for full_name in files_full_name:
+ if os.path.isdir(full_name):
+ section = Folder(full_name)
+ self.sections.append(section)
+
+
+class DocGenerator:
+ """
+ DocGenerator consists of several tools about document generation.
+ """
+
+ @classmethod
+ def getDocStructure(cls, root_path: str) -> Document:
+ """
+ Now document complies with a rule:
+ docs/
+ - chapter1/_category_.json
+ - section1/_category_.json
+ - md files
+ - md files
+ - chapter2/_category_.json
+ - md files
+ document is identical to Document
+ Dict: str(file name) -> TextIOWrapper(file descriptor)
+ """
+ os.walk()
+
+ @classmethod
+ def createFileIndexRst(cls, fin: TextIOWrapper):
+ fin.name
+
+
+doc = Folder("docs")
+print(json.dumps(doc.dumps(), indent=2))
+doc.writeIndexRst()
diff --git a/.github/scripts/generate-pdf.sh b/.github/scripts/generate-pdf.sh
old mode 100644
new mode 100755
index ea1a154f8b..d0d26a1e30
--- a/.github/scripts/generate-pdf.sh
+++ b/.github/scripts/generate-pdf.sh
@@ -6,10 +6,10 @@ function prepare {
sudo apt-get install texlive-xetex latex-cjk-all
sudo apt-get install pandoc
sudo apt install dos2unix
- wget https://github.com/adobe-fonts/source-han-sans/raw/release/OTF/SourceHanSansSC.zip
- wget https://github.com/adobe-fonts/source-han-serif/raw/release/OTF/SourceHanSerifSC_SB-H.zip
- wget https://github.com/adobe-fonts/source-han-serif/raw/release/OTF/SourceHanSerifSC_EL-M.zip
- unzip SourceHanSansSC.zip -d SourceHanSansSC
+ wget https://github.com/wweir/source-han-sans-sc/archive/refs/heads/master.zip
+ wget https://github.com/adobe-fonts/source-han-serif/raw/11f5ed33a61971e175f68df52ebd5951013ba643/OTF/SourceHanSerifSC_SB-H.zip
+ wget https://github.com/adobe-fonts/source-han-serif/raw/11f5ed33a61971e175f68df52ebd5951013ba643/OTF/SourceHanSerifSC_EL-M.zip
+ unzip master.zip -d SourceHanSansSC
unzip SourceHanSerifSC_EL-M.zip -d SourceHanSerifSC_EL-M
unzip SourceHanSerifSC_SB-H.zip -d SourceHanSerifSC_SB-H
sudo mv SourceHanSansSC SourceHanSerifSC_EL-M SourceHanSerifSC_SB-H /usr/share/fonts/opentype/
@@ -28,88 +28,40 @@ function prepare {
function generate_pdf {
localDir="source"
cp .github/scripts/Makefile .
- mkdir -p ./public/pdf
for lang in en zh
do
mkdir -p $localDir/img
- cp -r ./content/$lang/projects/$1/* $localDir/
- cp -r static/img/* $localDir/img
- cp .github/scripts/conf.py $localDir/
- cd $localDir
if [[ "$lang" == "en" ]] ;then
+ cp -r ./docs/* $localDir/
+ cp -r static/img/* $localDir/img
+ cp .github/scripts/conf.py $localDir/
+ cd $localDir
sed -i "s/language = 'zh_CN'/language = 'en_US'/" conf.py
echo "printing English version PDF"
else
+ cp -r ./i18n/zh/docusaurus-plugin-content-docs/current/* $localDir/
+ cp -r static/img/* $localDir/img
+ cp .github/scripts/conf.py $localDir/
+ cd $localDir
sed -i "s/language = 'en_US'/language = 'zh_CN'/" conf.py
echo "printing Chinese version PDF"
fi
- echo -e ".. toctree::\n :maxdepth: 1\n :titlesonly:\n" >> index.rst
- for f in `find . -type f -name "_index.md"`
- do
- fileName=${f##*/}
- path=${f%/*}
- dos2unix $f
- OLD_IFS="$IFS"
- IFS=
- title=
- link=
- subTitle=
- subLink=
- cat $f | while read line
- do
- if [[ "$line" =~ ^#.* ]]; then
- continue
- fi
-
- if [[ "$line" = " - title: "* ]]; then
- title=`echo "${line/ - title: /}"|sed -e 's/^[ \t]*//g' -e 's/[ \t]*$//g' -e $'s/\'//g'`
- fi
-
- if [[ "$line" = " - title: "* ]]; then
- subTitle=`echo "${line/ - title: /}"|sed -e 's/^[ \t]*//g' -e 's/[ \t]*$//g' -e $'s/\'//g'`
- fi
+ touch index.rst
- if [[ "$line" = " link:"* ]]; then
- link=`echo "${line/ link:/}"|sed -e 's/^[ \t]*//g' -e 's/[ \t]*$//g' -e $'s/\'//g'`
- indexFile=`echo "$title"|sed 's/[ ][ ]*/_/g'`
- echo -e " $path/$link/index" >> index.rst
- title=
- link=
- fi
-
- if [[ "$line" = " link:"* ]]; then
- subLink=`echo "${line/ link:/}"|sed -e 's/^[ \t]*//g' -e 's/[ \t]*$//g' -e $'s/\'//g'`
- subIndexFile=`echo "$title"|sed 's/[ ][ ]*/_/g'`
- if [ ! -f "${subIndexFile}.rst" ]; then
- echo "============================" >> "${path}/${subIndexFile}.rst"
- echo $title >> "$path/$subIndexFile.rst"
- echo "============================" >> "${path}/${subIndexFile}.rst"
- echo -e ".. toctree::\n :maxdepth: 1\n :titlesonly:\n" >> "$path/${subIndexFile}.rst"
- echo -e " $path/$subIndexFile" >> "$path/index.rst"
- fi
- echo -e " $path/$subLink/index" >> "$path/$subIndexFile.rst"
- subTitle=
- subLink=
- fi
- done
- IFS=$OLD_IFS
- done
-
- for f in `find . -type f -name "index.md"`
+ for f in `find . -type f -name "*.md"`
do
+ fileName=${f##*/}
path=${f%/*}
- echo "============================" >> "${path}/index.rst"
- echo `cat $f|grep -oP '^title: .*'|awk -F ": " '{print $2}'` >> "${path}/index.rst"
- echo "============================" >> "${path}/index.rst"
- echo -e ".. toctree::\n :maxdepth: 1\n :titlesonly:\n\n index.md" >> "${path}/index.rst"
+ touch ${path}/index.rst
pandoc $f --wrap=none -o "$f.rst"
rm $f
done
cd ..
+ python .github/scripts/generate-pdf.py
make latexpdf
mkdir -p pdf
- cp _build/latex/*.pdf ./public/pdf/apache_$1_docs_$lang.pdf
+ cp _build/latex/*.pdf ./pdf/apache_$1_docs_$lang.pdf
echo "apache_$1_docs_$lang.pdf"
make clean
rm -rf {_build,source}
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 2efd2290d9..fe9b7a0570 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -11,18 +11,34 @@ jobs:
runs-on: ubuntu-latest
name: Build and Deploy Job
steps:
- - uses: actions/checkout@v2
+ -
+ name: Checkout Code
+ uses: actions/checkout@v2
with:
submodules: recursive
- - name: Setup Node
+ -
+ name: Setup Node
uses: actions/setup-node@v2.4.0
with:
node-version: "16"
- - name: Setup Dependencies
+ -
+ name: Setup Dependencies
run: yarn install
- - name: Build Site
+ -
+ name: Setup Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.8
+ -
+ name: Generate PDF
+ run: |
+ bash .github/scripts/generate-pdf.sh
+ cp -r pdf/ static/
+ -
+ name: Build Site
run: yarn build
- - name: Deploy
+ -
+ name: Deploy
uses: peaceiris/actions-gh-pages@v3
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
diff --git a/.gitignore b/.gitignore
index b844e06303..de417b4ec3 100755
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
# Production
/build
+/pdf
# Generated files
.docusaurus