You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ti...@apache.org on 2022/12/04 14:24:15 UTC
[pulsar-site] branch main updated: Generate CLI docs on sync (#316)
This is an automated email from the ASF dual-hosted git repository.
tison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-site.git
The following commit(s) were added to refs/heads/main by this push:
new dcc8826a7f9 Generate CLI docs on sync (#316)
dcc8826a7f9 is described below
commit dcc8826a7f9748b95faa46a8e53c81efd0cd3af0
Author: tison <wa...@gmail.com>
AuthorDate: Sun Dec 4 22:24:11 2022 +0800
Generate CLI docs on sync (#316)
Signed-off-by: tison <wa...@gmail.com>
---
.github/workflows/ci-build-site.yml | 4 +-
.github/workflows/ci-sync-content.yml | 28 +++++-
.gitignore | 37 +-------
site2/tools/build-site.sh | 7 --
site2/tools/pulsar-admin-doc-gen.sh | 35 -------
site2/tools/pulsar-config-doc-gen.sh | 52 -----------
site2/tools/pulsar-doc-gen.sh | 44 ---------
site2/website-next/.gitignore | 4 +-
tmp/.gitignore | 1 +
tools/pytools/README.md | 34 ++++++-
tools/pytools/bin/reference-doc-generator.py | 60 ++++++++++++
tools/pytools/bin/site-updater.py | 26 +++---
tools/pytools/lib/command/__init__.py | 23 ++++-
tools/pytools/lib/execute/config_doc_generator.py | 102 +++++++++++++++++++++
.../lib/execute/pulsar_admin_clidoc_generator.py | 53 +++++++++++
.../pytools/lib/execute/pulsar_clidoc_generator.py | 47 ++++++++++
.../lib/execute/pulsar_client_clidoc_generator.py | 19 ++--
.../lib/execute/pulsar_perf_clidoc_generator.py | 19 ++--
18 files changed, 382 insertions(+), 213 deletions(-)
diff --git a/.github/workflows/ci-build-site.yml b/.github/workflows/ci-build-site.yml
index cc804b23793..d74117eeadb 100644
--- a/.github/workflows/ci-build-site.yml
+++ b/.github/workflows/ci-build-site.yml
@@ -53,7 +53,7 @@ jobs:
uses: ./.github/actions/tune-runner-vm
- name: Cache local Maven repository
- uses: actions/cache@v2
+ uses: actions/cache@v3
with:
path: |
~/.m2/repository/*/*/*
@@ -65,7 +65,7 @@ jobs:
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK 17
- uses: actions/setup-java@v2
+ uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 17
diff --git a/.github/workflows/ci-sync-content.yml b/.github/workflows/ci-sync-content.yml
index 411ecd26271..7dc728af856 100644
--- a/.github/workflows/ci-sync-content.yml
+++ b/.github/workflows/ci-sync-content.yml
@@ -39,8 +39,34 @@ jobs:
with:
python-version: '3.10'
cache: 'poetry'
+
+ - uses: actions/checkout@v3
+ with:
+ repository: apache/pulsar
+ path: tmp/pulsar
+ - name: Cache local Maven repository
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-website-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+ ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ ${{ runner.os }}-m2-dependencies-core-modules-
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: 17
+ - name: Run install by skip tests
+ working-directory: tmp/pulsar
+ run: mvn -B -ntp install -Pcore-modules,-main -DskipTests -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true
+
- name: Update site sources
working-directory: tools/pytools
run: |
poetry install
- poetry run bin/site-updater.py
+ poetry run bin/reference-doc-generator.py --master-path=$GITHUB_WORKSPACE/tmp/pulsar
+ poetry run bin/site-updater.py --master-path=$GITHUB_WORKSPACE/tmp/pulsar
diff --git a/.gitignore b/.gitignore
index 0621265338a..eefbfe3cdf9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,44 +2,17 @@
node_modules
-lib/core/metadata.js
-lib/core/MetadataBlog.js
-
-*/website/translated_docs
-*/website/build/
-*/website/yarn.lock
-*/website/node_modules
-*/website/i18n/*
-*/website/static/swagger/master/*
-*/website/static/swagger/restApiVersions.json
-*/website/static/swagger/*/v*/*.json
-*/website/i18n/*
-*/website/translated_docs*
-*/website/brodocs/index.html
-*/website/brodocs/navData.js
-*/website/brodocs/documents/*.md
-
*/website-next/translated_docs
-*/website-next/build/
+*/website-next/build
*/website-next/yarn.lock
*/website-next/node_modules
-*/website-next/i18n/*
-# */website-next/static/swagger/master/*
-# */website-next/static/swagger/restApiVersions.json
-# */website-next/static/swagger/*/v*/*.json
-*/website-next/i18n/*
-*/website-next/translated_docs*
-*/website-next/brodocs/index.html
-*/website-next/brodocs/navData.js
-*/website-next/brodocs/documents/*.md
-build/
-bak/
-website-bak/
-
-# IntelliJ
+# JetBrains IDEs
.idea/
*.iml
*.iws
apache-pulsar-*/
+
+# CI
+tmp-master
diff --git a/site2/tools/build-site.sh b/site2/tools/build-site.sh
index 5a061ea1d6d..b300b86e603 100755
--- a/site2/tools/build-site.sh
+++ b/site2/tools/build-site.sh
@@ -31,13 +31,6 @@ VERSION=next
export NODE_OPTIONS="--max-old-space-size=16000"
-# Generate document for command line tools.
-"$TOOLS_DIR"/pulsar-admin-doc-gen.sh "$WEBSITE_DIR" "$VERSION"
-"$TOOLS_DIR"/pulsar-client-doc-gen.sh "$WEBSITE_DIR" "$VERSION"
-"$TOOLS_DIR"/pulsar-perf-doc-gen.sh "$WEBSITE_DIR" "$VERSION"
-"$TOOLS_DIR"/pulsar-doc-gen.sh "$WEBSITE_DIR" "$VERSION"
-"$TOOLS_DIR"/pulsar-config-doc-gen.sh "$WEBSITE_DIR" "$VERSION"
-
cd "$WEBSITE_DIR"
npm install
diff --git a/site2/tools/pulsar-admin-doc-gen.sh b/site2/tools/pulsar-admin-doc-gen.sh
deleted file mode 100755
index 4acadb0b7d4..00000000000
--- a/site2/tools/pulsar-admin-doc-gen.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -x
-
-ROOT_DIR=$(git rev-parse --show-toplevel)
-WEBSITE=$1
-VERSION=$2
-DOCS_DIR=$WEBSITE/static/reference/$VERSION/pulsar-admin
-
-DOC_GEN="$ROOT_DIR/bin/pulsar-admin documents generate"
-
-COMMANDS="broker-stats brokers bookies clusters functions functions-worker namespaces ns-isolation-policy sources sinks topics topicPolicies proxy-stats resourcegroups transactions tenants resource-quotas schemas packages"
-
-for CMD in $COMMANDS
-do
- $DOC_GEN "$CMD" > "$DOCS_DIR/$CMD".md
-done
diff --git a/site2/tools/pulsar-config-doc-gen.sh b/site2/tools/pulsar-config-doc-gen.sh
deleted file mode 100755
index 0002735bd79..00000000000
--- a/site2/tools/pulsar-config-doc-gen.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# common variables
-ROOT_DIR=$(git rev-parse --show-toplevel)
-WEBSITE=$1
-VERSION=$2
-VERSIONED_DIR=$WEBSITE/static/reference/$VERSION
-JAVA=java
-f=$ROOT_DIR/distribution/server/target/classpath.txt
-
-# config doc gen variables
-DOCS_DIR=$VERSIONED_DIR/config
-GEN_DOCS_BROKER=org.apache.pulsar.utils.CmdGenerateDocumentation
-GEN_DOCS_PROXY=org.apache.pulsar.proxy.util.CmdGenerateDocumentation
-
-# client config doc gen variables
-CLIENT_DIR=$VERSIONED_DIR/client
-CLIENT_CP=$ROOT_DIR/pulsar-client/target/classes
-CLIENT_CONF=org.apache.pulsar.client.impl.conf
-GEN_DOCS_CLIENT=$CLIENT_CONF.CmdGenerateDocumentation
-
-# config doc gen
-$JAVA -cp "$(cat "$f")" $GEN_DOCS_BROKER -c org.apache.pulsar.broker.ServiceConfiguration > "$DOCS_DIR"/reference-configuration-broker.md
-$JAVA -cp "$(cat "$f")" $GEN_DOCS_BROKER -c org.apache.pulsar.client.impl.conf.ClientConfigurationData > "$DOCS_DIR"/reference-configuration-client.md
-$JAVA -cp "$(cat "$f")" $GEN_DOCS_BROKER -c org.apache.pulsar.websocket.service.WebSocketProxyConfiguration > "$DOCS_DIR"/reference-configuration-websocket.md
-$JAVA -cp "$(cat "$f")" $GEN_DOCS_PROXY -c org.apache.pulsar.proxy.server.ProxyConfiguration > "$DOCS_DIR"/reference-configuration-pulsar-proxy.md
-
-cp "$DOCS_DIR"/reference-configuration-broker.md "$DOCS_DIR"/reference-configuration-standalone.md
-
-# client config doc gen
-$JAVA -cp "$CLIENT_CP:$(cat "$f")" $GEN_DOCS_CLIENT -c $CLIENT_CONF.ClientConfigurationData > "$CLIENT_DIR"/client-configuration-client.md
-$JAVA -cp "$CLIENT_CP:$(cat "$f")" $GEN_DOCS_CLIENT -c $CLIENT_CONF.ProducerConfigurationData > "$CLIENT_DIR"/client-configuration-producer.md
-$JAVA -cp "$CLIENT_CP:$(cat "$f")" $GEN_DOCS_CLIENT -c $CLIENT_CONF.ConsumerConfigurationData > "$CLIENT_DIR"/client-configuration-consumer.md
-$JAVA -cp "$CLIENT_CP:$(cat "$f")" $GEN_DOCS_CLIENT -c $CLIENT_CONF.ReaderConfigurationData > "$CLIENT_DIR"/client-configuration-reader.md
diff --git a/site2/tools/pulsar-doc-gen.sh b/site2/tools/pulsar-doc-gen.sh
deleted file mode 100755
index aca31cf6ece..00000000000
--- a/site2/tools/pulsar-doc-gen.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -x
-
-ROOT_DIR=$(git rev-parse --show-toplevel)
-WEBSITE=$1
-VERSION=$2
-DOCS_DIR=$WEBSITE/static/reference/$VERSION/pulsar
-
-PULSAR="$ROOT_DIR/bin/pulsar"
-
-$PULSAR broker -g > "$DOCS_DIR"/broker.md
-$PULSAR broker-tool gen-doc > "$DOCS_DIR"/broker-tool.md
-$PULSAR compact-topic -t tmp -g > "$DOCS_DIR"/compact-topic.md
-$PULSAR tokens gen-doc > "$DOCS_DIR"/tokens.md
-$PULSAR proxy -g > "$DOCS_DIR"/proxy.md
-$PULSAR functions-worker -g > "$DOCS_DIR"/functions-worker.md
-$PULSAR standalone -g > "$DOCS_DIR"/standalone.md
-$PULSAR initialize-cluster-metadata -cs cs -uw uw -zk zk -c c -g > "$DOCS_DIR"/initialize-cluster-metadata.md
-$PULSAR delete-cluster-metadata -zk zk -g > "$DOCS_DIR"/delete-cluster-metadata.md
-$PULSAR initialize-transaction-coordinator-metadata -cs cs -c c -g > "$DOCS_DIR"/initialize-transaction-coordinator-metadata.md
-$PULSAR initialize-namespace -cs cs -c c -g demo > "$DOCS_DIR"/initialize-namespace.md
-$PULSAR version -g > "$DOCS_DIR"/version.md
-$PULSAR websocket -g > "$DOCS_DIR"/websocket.md
-
-
diff --git a/site2/website-next/.gitignore b/site2/website-next/.gitignore
index 7a41a65f3b3..500b913244f 100644
--- a/site2/website-next/.gitignore
+++ b/site2/website-next/.gitignore
@@ -23,6 +23,4 @@ npm-debug.log*
yarn-debug.log*
yarn-error.log*
-bak/
-
-i18n/*
+yarn.lock
diff --git a/tmp/.gitignore b/tmp/.gitignore
new file mode 100644
index 00000000000..72e8ffc0db8
--- /dev/null
+++ b/tmp/.gitignore
@@ -0,0 +1 @@
+*
diff --git a/tools/pytools/README.md b/tools/pytools/README.md
index d4cba192e62..24d1e75316b 100644
--- a/tools/pytools/README.md
+++ b/tools/pytools/README.md
@@ -51,7 +51,7 @@ Pytools provides the following executables:
### [cpp-apidoc-generator](bin/cpp-apidoc-generator.py)
-This executable generate API docs for Pulsar C++ Client using [`doxygen`](https://doxygen.nl/):
+This executable generates API docs for Pulsar C++ Client using [`doxygen`](https://doxygen.nl/):
```bash
poetry run bin/cpp-apidoc-generator.py <VERSION>
@@ -61,7 +61,7 @@ poetry run bin/cpp-apidoc-generator.py <VERSION>
### [py-apidoc-generator](bin/py-apidoc-generator.py)
-This executable generate API docs for Pulsar Python Client:
+This executable generates API docs for Pulsar Python Client:
```bash
poetry run bin/py-apidoc-generator.py <VERSION>
@@ -74,10 +74,38 @@ poetry run bin/py-apidoc-generator.py <VERSION>
### [java-apidoc-generator](bin/java-apidoc-generator.py)
-This executable generate API docs for Pulsar Java Client, Admin and Functions using Maven:
+This executable generates API docs for Pulsar Java Client, Admin and Functions using Maven:
```bash
poetry run bin/java-apidoc-generator.py <VERSION>
```
... where the `VERSION` is released semantic version like `2.10.2`.
+
+### [site-updater](bin/site-updater.py)
+
+This executable synchronizes site content from the main repo:
+
+```bash
+poetry run bin/site-updater.py --master-path=<PATH> [--push={y|n|auto}]
+```
+
+... where:
+
+1. `master-path` is path to the main repo;
+2. `push` specifies whether push to the remote site repo.
+
+### [reference-doc-generator](bin/reference-doc-generator.py)
+
+This executable generates Pulsar references, including command-line documents and configs:
+
+```bash
+poetry run bin/reference-doc-generator.py --master-path=<path> [--version=<VERSION>]
+ [--kind={all|admin|cient|config|perf|pulsar}]
+```
+
+... where:
+
+1. `master-path` is path to the main repo, which must be built to execute the `bin` scripts;
+2. `version` is the version of the main repo, default to `next` a.k.a. the latest master branch;
+3. `kind` is what references to generate, default to `all`.
diff --git a/tools/pytools/bin/reference-doc-generator.py b/tools/pytools/bin/reference-doc-generator.py
new file mode 100755
index 00000000000..b625c2c9c94
--- /dev/null
+++ b/tools/pytools/bin/reference-doc-generator.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import enum
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from pathlib import Path
+
+from execute import config_doc_generator, pulsar_admin_clidoc_generator, pulsar_clidoc_generator
+from execute import pulsar_client_clidoc_generator, pulsar_perf_clidoc_generator
+
+
+class Kind(enum.Enum):
+ all = 'all'
+ config = 'config'
+ admin = 'admin'
+ pulsar = 'pulsar'
+ client = 'client'
+ perf = 'perf'
+
+
+if __name__ == '__main__':
+ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+ parser.add_argument('--master-path', required=True, type=str, metavar='PATH', help='path to pulsar main repo')
+ parser.add_argument('--version', default='next', metavar='VERSION', help='version of Apache Pulsar')
+ parser.add_argument('--kind', default=[Kind.all], type=Kind, choices=list(Kind), nargs='+')
+
+ args = parser.parse_args()
+ master_path = Path(args.master_path)
+ kinds = set(args.kind)
+ if Kind.all in kinds:
+ kinds = {Kind.config, Kind.admin, Kind.pulsar, Kind.client, Kind.perf}
+
+ for kind in kinds:
+ match kind:
+ case Kind.config:
+ config_doc_generator.execute(master_path, args.version)
+ case Kind.admin:
+ pulsar_admin_clidoc_generator.execute(master_path, args.version)
+ case Kind.pulsar:
+ pulsar_clidoc_generator.execute(master_path, args.version)
+ case Kind.client:
+ pulsar_client_clidoc_generator.execute(master_path, args.version)
+ case Kind.perf:
+ pulsar_perf_clidoc_generator.execute(master_path, args.version)
diff --git a/tools/pytools/bin/site-updater.py b/tools/pytools/bin/site-updater.py
index be56eb82bd1..8a6f96583e8 100755
--- a/tools/pytools/bin/site-updater.py
+++ b/tools/pytools/bin/site-updater.py
@@ -50,12 +50,14 @@ def _should_push(mode: Mode) -> bool:
return result
+git = find_command('git', msg="git is required")
+
+
def _do_push(main: Path, site: Path):
- _git = find_command('git', msg="git is required")
- commit = run_pipe(_git, 'rev-parse', '--short', 'HEAD', cwd=main).read().strip()
- run(_git, 'add', '-A', '.', cwd=site)
- run(_git, 'status', cwd=site)
- run(_git, 'remote', '-v', cwd=site)
+ commit = run_pipe(git, 'rev-parse', '--short', 'HEAD', cwd=main).read().strip()
+ run(git, 'add', '-A', '.', cwd=site)
+ run(git, 'status', cwd=site)
+ run(git, 'remote', '-v', cwd=site)
if os.getenv('GITHUB_ACTIONS') is not None:
if os.getenv('GITHUB_EVENT_NAME') != 'schedule':
name = os.getenv('GITHUB_ACTOR')
@@ -63,12 +65,12 @@ def _do_push(main: Path, site: Path):
else:
name = 'github-actions[bot]'
email = f'41898282+{name}@users.noreply.github.com'
- run(_git, 'config', 'user.name', name, cwd=site)
- run(_git, 'config', 'user.email', email, cwd=site)
- changed = run_pipe(_git, 'status', '--porcelain', cwd=site).read().strip()
+ run(git, 'config', 'user.name', name, cwd=site)
+ run(git, 'config', 'user.email', email, cwd=site)
+ changed = run_pipe(git, 'status', '--porcelain', cwd=site).read().strip()
if len(changed) != 0:
- run(_git, 'commit', '-m', f'Docs sync done from apache/pulsar (#{commit})', cwd=site)
- run(_git, 'push', 'origin', 'main', cwd=site)
+ run(git, 'commit', '-m', f'Docs sync done from apache/pulsar (#{commit})', cwd=site)
+ run(git, 'push', 'origin', 'main', cwd=site)
if __name__ == '__main__':
@@ -80,7 +82,6 @@ if __name__ == '__main__':
with tempfile.TemporaryDirectory() as cwd:
if args.master_path is None:
- git = find_command('git', msg="git is required")
run(git, 'clone', '-b', 'master', '--depth', '1', 'https://github.com/apache/pulsar', cwd=cwd)
master = Path(cwd) / 'pulsar'
else:
@@ -89,3 +90,6 @@ if __name__ == '__main__':
site_syncer.execute(master)
if _should_push(args.push):
_do_push(master, root_path())
+ else: # show changes
+ change_files = run_pipe(git, 'status', '--porcelain', cwd=root_path()).read().strip()
+ print(f'\nchange files:\n{change_files}\n')
diff --git a/tools/pytools/lib/command/__init__.py b/tools/pytools/lib/command/__init__.py
index 71ebfb217e6..00dc873b0eb 100644
--- a/tools/pytools/lib/command/__init__.py
+++ b/tools/pytools/lib/command/__init__.py
@@ -17,17 +17,24 @@
import sys
from subprocess import PIPE, Popen
-from typing import Optional, Any, TextIO
+from typing import Optional, Any, TextIO, Set
-def run(*args: str, msg: Optional[str] = None, verbose: bool = False, **kwargs: Any) -> Popen:
+def run(
+ *args: str,
+ msg: Optional[str] = None,
+ verbose: bool = False,
+ codes: Optional[Set[int]] = None,
+ **kwargs: Any
+) -> Popen:
sys.stdout.flush()
if verbose:
print(f"$ {' '.join(args)}")
p = Popen(args, **kwargs)
code = p.wait()
- if code != 0:
+ codes = codes or {0}
+ if code not in codes:
err = f"\nfailed to run: {args}\nexit with code: {code}\n"
if msg:
err += f"error message: {msg}\n"
@@ -36,8 +43,14 @@ def run(*args: str, msg: Optional[str] = None, verbose: bool = False, **kwargs:
return p
-def run_pipe(*args: str, msg: Optional[str] = None, verbose: bool = False, **kwargs: Any) -> TextIO:
- p = run(*args, msg=msg, verbose=verbose, stdout=PIPE, universal_newlines=True, **kwargs)
+def run_pipe(
+ *args: str,
+ msg: Optional[str] = None,
+ verbose: bool = False,
+ codes: Optional[Set[int]] = None,
+ **kwargs: Any
+) -> TextIO:
+ p = run(*args, msg=msg, verbose=verbose, codes=codes, stdout=PIPE, universal_newlines=True, **kwargs)
return p.stdout # type: ignore
diff --git a/tools/pytools/lib/execute/config_doc_generator.py b/tools/pytools/lib/execute/config_doc_generator.py
new file mode 100644
index 00000000000..b5ad27c1d9e
--- /dev/null
+++ b/tools/pytools/lib/execute/config_doc_generator.py
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+from command import find_command, run
+from constant import site_path
+
+
+@dataclass
+class Settings:
+ type: str
+ classname: str
+ generator: str
+ filename: str
+
+
+def execute(master: Path, version: str):
+ java = find_command('java', msg='java is required')
+
+ reference = site_path() / 'static' / 'reference' / version
+ classpath = master / 'distribution' / 'server' / 'target' / 'classpath.txt'
+ classpath = classpath.read_text()
+
+ broker_doc_generator = 'org.apache.pulsar.utils.CmdGenerateDocumentation'
+ client_doc_generator = 'org.apache.pulsar.client.impl.conf.CmdGenerateDocumentation'
+ proxy_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation'
+
+ configs = [
+ Settings(
+ 'config',
+ 'org.apache.pulsar.broker.ServiceConfiguration',
+ broker_doc_generator,
+ 'reference-configuration-broker.md'
+ ),
+ Settings(
+ 'config',
+ 'org.apache.pulsar.client.impl.conf.ClientConfigurationData',
+ broker_doc_generator,
+ 'reference-configuration-client.md'
+ ),
+ Settings(
+ 'config',
+ 'org.apache.pulsar.websocket.service.WebSocketProxyConfiguration',
+ broker_doc_generator,
+ 'reference-configuration-websocket.md'
+ ),
+ Settings(
+ 'config',
+ 'org.apache.pulsar.proxy.server.ProxyConfiguration',
+ proxy_doc_generator,
+ 'reference-configuration-pulsar-proxy.md'
+ ),
+ Settings(
+ 'client',
+ 'org.apache.pulsar.client.impl.conf.ClientConfigurationData',
+ client_doc_generator,
+ 'client-configuration-client.md'
+ ),
+ Settings(
+ 'client',
+ 'org.apache.pulsar.client.impl.conf.ProducerConfigurationData',
+ client_doc_generator,
+ 'client-configuration-producer.md'
+ ),
+ Settings(
+ 'client',
+ 'org.apache.pulsar.client.impl.conf.ConsumerConfigurationData',
+ client_doc_generator,
+ 'client-configuration-consumer.md'
+ ),
+ Settings(
+ 'client',
+ 'org.apache.pulsar.client.impl.conf.ReaderConfigurationData',
+ client_doc_generator,
+ 'client-configuration-reader.md'
+ ),
+ ]
+
+ for config in configs:
+ with (reference / config.type / config.filename).open('w') as f:
+ run(java, '-cp', classpath, config.generator, '-c', config.classname, stdout=f)
+
+ shutil.copy2(
+ reference / 'config' / 'reference-configuration-broker.md',
+ reference / 'config' / 'reference-configuration-standalone.md')
diff --git a/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py
new file mode 100644
index 00000000000..088312b5319
--- /dev/null
+++ b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from pathlib import Path
+
+from command import run
+from constant import site_path
+
+
+def execute(basedir: Path, version: str):
+ admin = basedir / 'bin' / 'pulsar-admin'
+ reference = site_path() / 'static' / 'reference' / version / 'pulsar-admin'
+
+ commands = [
+ 'broker-stats',
+ 'brokers',
+ 'bookies',
+ 'clusters',
+ 'functions',
+ 'functions-worker',
+ 'namespaces',
+ 'ns-isolation-policy',
+ 'sources',
+ 'sinks',
+ 'topics',
+ 'topicPolicies',
+ 'proxy-stats',
+ 'resourcegroups',
+ 'transactions',
+ 'tenants',
+ 'resource-quotas',
+ 'schemas',
+ 'packages',
+ ]
+
+ for command in commands:
+ with (reference / f'{command}.md').open('w') as f:
+ run(str(admin.absolute()), 'documents', 'generate', command, stdout=f)
diff --git a/tools/pytools/lib/execute/pulsar_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_clidoc_generator.py
new file mode 100644
index 00000000000..e6907923107
--- /dev/null
+++ b/tools/pytools/lib/execute/pulsar_clidoc_generator.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from pathlib import Path
+
+from command import run
+from constant import site_path
+
+
+def execute(basedir: Path, version: str):
+ pulsar = basedir / 'bin' / 'pulsar'
+ reference = site_path() / 'static' / 'reference' / version / 'pulsar'
+
+ commands = [
+ ('broker', '-g'),
+ ('broker-tool', 'gen-doc'),
+ ('compact-topic', '-t', 'tmp', '-g'),
+ ('tokens', 'gen-doc'),
+ ('proxy', '-g'),
+ ('functions-worker', '-g'),
+ ('standalone', '-g'),
+ ('initialize-cluster-metadata', '-cs', 'cs', '-uw', 'uw', '-zk', 'zk', '-c', 'c', '-g'),
+ ('delete-cluster-metadata', '-zk', 'zk', '-g'),
+ ('initialize-transaction-coordinator-metadata', '-cs', 'cs', '-c', 'c', '-g'),
+ ('initialize-namespace', '-cs', 'cs', '-c', 'c', '-g', 'demo'),
+ ('version', '-g'),
+ ('websocket', '-g'),
+ ]
+
+ for command in commands:
+ with (reference / f'{command[0]}.md').open('w') as f:
+ run(str(pulsar.absolute()), *command, codes={0, 255}, stdout=f)
diff --git a/site2/tools/pulsar-client-doc-gen.sh b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py
old mode 100755
new mode 100644
similarity index 67%
rename from site2/tools/pulsar-client-doc-gen.sh
rename to tools/pytools/lib/execute/pulsar_client_clidoc_generator.py
index 298fd8ee4df..c3c1616e0b6
--- a/site2/tools/pulsar-client-doc-gen.sh
+++ b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,13 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-set -x
+from pathlib import Path
+
+from command import run
+from constant import site_path
+
-ROOT_DIR=$(git rev-parse --show-toplevel)
-WEBSITE=$1
-VERSION=$2
-DOCS_DIR=$WEBSITE/static/reference/$VERSION/pulsar-client
+def execute(basedir: Path, version: str):
+ client = basedir / 'bin' / 'pulsar-client'
+ reference = site_path() / 'static' / 'reference' / version / 'pulsar-client'
-"$ROOT_DIR"/bin/pulsar-client generate_documentation > "$DOCS_DIR"/pulsar-client.md
+ with (reference / 'pulsar-client.md').open('w') as f:
+ run(str(client.absolute()), 'generate_documentation', stdout=f)
diff --git a/site2/tools/pulsar-perf-doc-gen.sh b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py
old mode 100755
new mode 100644
similarity index 68%
rename from site2/tools/pulsar-perf-doc-gen.sh
rename to tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py
index db5eb8aa384..e737eed00f0
--- a/site2/tools/pulsar-perf-doc-gen.sh
+++ b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,13 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-set -x
+from pathlib import Path
+
+from command import run
+from constant import site_path
+
-ROOT_DIR=$(git rev-parse --show-toplevel)
-WEBSITE=$1
-VERSION=$2
-DOCS_DIR=$WEBSITE/static/reference/$VERSION/pulsar-perf
+def execute(basedir: Path, version: str):
+ perf = basedir / 'bin' / 'pulsar-perf'
+ reference = site_path() / 'static' / 'reference' / version / 'pulsar-perf'
-"$ROOT_DIR"/bin/pulsar-perf gen-doc > "$DOCS_DIR"/pulsar-perf.md
+ with (reference / 'pulsar-perf.md').open('w') as f:
+ run(str(perf.absolute()), 'gen-doc', stdout=f)