You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2021/02/11 02:00:16 UTC
[skywalking-satellite] branch main updated: polish docs (#23)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-satellite.git
The following commit(s) were added to refs/heads/main by this push:
new 95481ae polish docs (#23)
95481ae is described below
commit 95481aec1538d0fda27de5d45b5bc3071630b942
Author: Evan <31...@users.noreply.github.com>
AuthorDate: Thu Feb 11 10:00:09 2021 +0800
polish docs (#23)
---
.github/workflows/build-and-test.yaml | 60 ++++++--
Makefile | 38 ++++-
README.md | 29 +++-
cmd/command.go | 11 +-
docs/README.md | 19 +++
docs/compile/compile.md | 10 --
docs/configuration/queue.md | 11 --
docs/design/module_structure.md | 25 ----
docs/en/FAQ/README.md | 2 +
docs/en/concepts-and-designs/README.md | 13 ++
.../concepts-and-designs}/module_design.md | 10 +-
docs/en/concepts-and-designs/overview.md | 22 +++
.../concepts-and-designs/plugin_mechanism.md} | 2 +-
docs/en/concepts-and-designs/project-goals.md | 15 ++
docs/en/concepts-and-designs/project_structue.md | 33 +++++
docs/en/guides/README.md | 6 +
docs/en/guides/compile/compile.md | 14 ++
docs/en/guides/contribuation/plugin.md | 42 ++++++
docs/{ => en/guides}/test/test.md | 0
docs/en/setup/README.md | 19 +++
docs/en/setup/configuration/common.md | 18 +++
docs/en/setup/configuration/override-settings.md | 24 +++
docs/en/setup/configuration/pipe-plugins.md | 67 +++++++++
docs/en/setup/configuration/sharing-plugins.md | 29 ++++
docs/en/setup/plugins/client_kafka-client.md | 53 +++++++
.../en/setup/plugins/fallbacker_none-fallbacker.md | 5 +
.../setup/plugins/fallbacker_timer-fallbacker.md | 8 +
.../plugins/forwarder_nativelog-kafka-forwarder.md | 8 +
docs/en/setup/plugins/plugin-list.md | 21 +++
docs/en/setup/plugins/queue_memory-queue.md | 8 +
docs/en/setup/plugins/queue_mmap-queue.md | 20 +++
.../plugins/receiver_grpc-nativelog-receiver.md | 5 +
.../en/setup/plugins/receiver_http-log-receiver.md | 10 ++
docs/en/setup/plugins/server_grpc-server.md | 18 +++
docs/en/setup/plugins/server_http-server.md | 8 +
docs/en/setup/plugins/server_prometheus-server.md | 10 ++
docs/plugin-description.md | 164 ---------------------
docs/plugins/queue/mmap/README.md | 86 -----------
docs/project_structue.md | 82 -----------
internal/satellite/tools/generate_plugin_doc.go | 88 +++++++----
plugins/client/kafka/client.go | 2 +-
plugins/fallbacker/none/none_fallbacker.go | 2 +-
plugins/fallbacker/timer/timer_fallbacker.go | 2 +-
.../forwarder/kafka/nativelog/sync_forwarder.go | 2 +-
plugins/queue/memory/queue.go | 2 +-
plugins/queue/mmap/branchmark_test.go | 2 +
plugins/queue/mmap/meta/meta.go | 2 +
plugins/queue/mmap/meta/meta_test.go | 2 +
plugins/queue/mmap/queue.go | 5 +-
plugins/queue/mmap/queue_opreation.go | 2 +
plugins/queue/mmap/queue_test.go | 2 +
plugins/queue/mmap/segment/segment.go | 2 +
plugins/queue/mmap/segment/segment_test.go | 2 +
plugins/queue/mmap/segment_operation.go | 2 +
plugins/server/grpc/server.go | 2 +-
plugins/server/http/server.go | 2 +-
plugins/server/prometheus/prometheus.go | 2 +-
57 files changed, 710 insertions(+), 440 deletions(-)
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
index 114525a..7eb20c1 100644
--- a/.github/workflows/build-and-test.yaml
+++ b/.github/workflows/build-and-test.yaml
@@ -23,49 +23,85 @@ on:
- main
jobs:
- build:
- runs-on: ubuntu-18.04
+ CI-Windows:
+ runs-on: windows-latest
timeout-minutes: 90
strategy:
matrix:
- go-version: [1.14, 1.15]
+ go-version: [ 1.14, 1.15 ]
fail-fast: true
steps:
- name: Set up Go ${{ matrix.go-version }}
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
-
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
submodules: true
+ - name: Get dependencies
+ run: make deps
+ - name: Test
+ run: make test
+ - name: Build
+ run: make build
+ CI-MacOS:
+ runs-on: macos-latest
+ timeout-minutes: 90
+ strategy:
+ matrix:
+ go-version: [ 1.14, 1.15 ]
+ fail-fast: true
+ steps:
+ - name: Set up Go ${{ matrix.go-version }}
+ uses: actions/setup-go@v2
+ with:
+ go-version: ${{ matrix.go-version }}
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v2
+ with:
+ submodules: true
+ - name: Get dependencies
+ run: make deps
+ - name: Test
+ run: make test
+ - name: Build
+ run: make build
+ CI:
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ strategy:
+ matrix:
+ go-version: [ 1.14, 1.15 ]
+ fail-fast: true
+ steps:
+ - name: Set up Go ${{ matrix.go-version }}
+ uses: actions/setup-go@v2
+ with:
+ go-version: ${{ matrix.go-version }}
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v2
+ with:
+ submodules: true
- name: Check License
run: make license
-
- name: Get dependencies
run: make deps
-
- name: Check generate
run: make gen && make build
-
- name: Check CI Consistency
if: matrix.go-version == '1.15'
run: make check
-
- name: Lint
run: make lint
-
- name: Test
run: make test
-
-
result:
runs-on: ubuntu-latest
timeout-minutes: 90
- needs: [build]
+ needs: [ CI ]
steps:
- name: Build Result
run: echo "Just to make the GitHub merge button green"
diff --git a/Makefile b/Makefile
index 1749558..a1251cd 100644
--- a/Makefile
+++ b/Makefile
@@ -44,19 +44,19 @@ ARCH = amd64
SHELL = /bin/bash
-all: clean license deps lint test build
+all: deps verify build check
.PHONY: tools
tools:
$(GO_LINT) version || curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_PATH)/bin v1.33.0
$(GO_LICENSER) -version || GO111MODULE=off $(GO_GET) -u github.com/elastic/go-licenser
- $(PROTOC) --version || sh tools/install_protoc.sh
deps: tools
$(GO_GET) -v -t -d ./...
.PHONY: gen
gen:
+ $(PROTOC) --version || sh tools/install_protoc.sh
/bin/sh tools/protocol_gen.sh
.PHONY: lint
@@ -64,7 +64,7 @@ lint: tools
$(GO_LINT) run -v ./...
.PHONY: test
-test: clean lint
+test: clean
$(GO_TEST) ./... -coverprofile=coverage.txt -covermode=atomic
.PHONY: license
@@ -83,7 +83,7 @@ build: deps linux darwin windows
.PHONY: check
check: clean
- $(OUT_DIR)/$(BINARY)-$(VERSION)-$(OSNAME)-$(ARCH) docs
+ $(OUT_DIR)/$(BINARY)-$(VERSION)-$(OSNAME)-$(ARCH) docs --output=docs/en/setup/plugins
$(GO) mod tidy > /dev/null
@if [ ! -z "`git status -s`" ]; then \
echo "Following files are not consistent with CI:"; \
@@ -92,6 +92,36 @@ check: clean
exit 1; \
fi
+release-src: clean
+ -tar -zcvf $(RELEASE_SRC).tgz \
+ --exclude bin \
+ --exclude .git \
+ --exclude .idea \
+ --exclude .DS_Store \
+ --exclude .github \
+ --exclude $(RELEASE_SRC).tgz \
+ --exclude protocol/skywalking-data-collect-protocol \
+ .
+
+release-bin: build
+ -mkdir $(RELEASE_BIN)
+ -cp -R bin $(RELEASE_BIN)
+ -cp -R dist/* $(RELEASE_BIN)
+ -cp -R CHANGES.md $(RELEASE_BIN)
+ -cp -R README.md $(RELEASE_BIN)
+ -tar -zcvf $(RELEASE_BIN).tgz $(RELEASE_BIN)
+ -rm -rf $(RELEASE_BIN)
+
+
+
+.PHONY: release
+release: verify release-src release-bin
+ gpg --batch --yes --armor --detach-sig $(RELEASE_SRC).tgz
+ shasum -a 512 $(RELEASE_SRC).tgz > $(RELEASE_SRC).tgz.sha512
+ gpg --batch --yes --armor --detach-sig $(RELEASE_BIN).tgz
+ shasum -a 512 $(RELEASE_BIN).tgz > $(RELEASE_BIN).tgz.sha512
+
+
.PHONY: $(PLATFORMS)
$(PLATFORMS):
mkdir -p $(OUT_DIR)
diff --git a/README.md b/README.md
index da4ac82..67d2417 100644
--- a/README.md
+++ b/README.md
@@ -8,16 +8,34 @@ Apache SkyWalking Satellite
[](https://github.com/apache/skywalking)
[](https://twitter.com/AsfSkyWalking)
-# Quick Start
-
# Documentation
-- [The first design of Satellite 0.1.0](https://skywalking.apache.org/blog/2020-11-25-skywalking-satellite-0.1.0-design/)
-- [The project structure](./docs/project_structue.md)
+- [Official documentation](https://skywalking.apache.org/docs/)
+- [Blog](https://skywalking.apache.org/blog/2020-11-25-skywalking-satellite-0.1.0-design/) about the design of Satellite 0.1.0.
+
+NOTICE, SkyWalking Satellite uses [v3 protocols](https://github.com/apache/skywalking/blob/master/docs/en/protocols/README.md). They are incompatible with previous SkyWalking releases before SkyWalking 8.0.
# Download
+Go to the [download page](https://skywalking.apache.org/downloads/) to download all available binaries, including MacOS, Linux and Windows. Due to system compatibility problems, some plugins of SkyWalking Satellite cannot be used in Windows system. Check [the corresponding documentation](./docs/en/guides/compile/compile.md) to see whether the plugin is available on Windows.
# Compile
-[How to compile the Satellite.](./docs/compile/compile.md)
+As SkyWalking Satellite is using `Makefile`, compiling the project is as easy as executing a command in the root directory of the project.
+```shell script
+git clone https://github.com/apache/skywalking-satellite
+cd skywalking-satellite
+git submodule init
+git submodule update
+make build
+```
+If you want to know more details about compiling, please read [the doc](./docs/en/guides/compile/compile.md).
+
+
+# Commands
+| Commands| Flags | Description |
+| ---- | ---- |---- |
+| start | --config FILE | Start Satellite with the configuration FILE. (default: "configs/satellite_config.yaml" or read value from *SATELLITE_CONFIG* env).|
+| docs | --output value | Generate Satellite plugin documentations to the output path. (default: "docs" or read value from *SATELLITE_DOC_PATH* env) |
+
+
# Contact Us
* Mail list: **dev@skywalking.apache.org**. Mail to `dev-subscribe@skywalking.apache.org`, follow the reply to subscribe the mail list.
* Join `skywalking` channel at [Apache Slack](http://s.apache.org/slack-invite). If the link is not working, find the latest one at [Apache INFRA WIKI](https://cwiki.apache.org/confluence/display/INFRA/Slack+Guest+Invites).
@@ -27,4 +45,3 @@ Apache SkyWalking Satellite
# License
[Apache 2.0 License.](/LICENSE)
-
diff --git a/cmd/command.go b/cmd/command.go
index e8c3f0b..2a96179 100644
--- a/cmd/command.go
+++ b/cmd/command.go
@@ -47,8 +47,17 @@ var (
cmdDocs = cli.Command{
Name: "docs",
Usage: "generate satellite plugin docs",
+ Flags: []cli.Flag{
+ &cli.StringFlag{
+ Name: "output, o",
+ Usage: "The output path for the plugin documentation",
+ EnvVars: []string{"SATELLITE_DOC_PATH"},
+ Value: "docs",
+ },
+ },
Action: func(c *cli.Context) error {
- return tools.GeneratePluginDoc()
+ outputPath := c.String("output")
+ return tools.GeneratePluginDoc(outputPath)
},
}
)
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000..309f795
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,19 @@
+# Welcome
+**Here are SkyWalking Satellite official documentations. You're welcome to join us.**
+
+From here you can learn all about **SkyWalking Satellite's** architecture, how to deploy and use SkyWalking Satellite.
+
+- [Concepts and Designs](en/concepts-and-designs/README.md). The most important core ideas about SkyWalking Satellite. You can learn from here if you want to understand what is going on under our cool features.
+- [Setup](en/setup/README.md). Introduce how to set up the SkyWalking Satellite.
+
+- [Guides](en/guides/README.md). Guide users to develop or debug SkyWalking Satellite.
+
+- [Protocols](https://github.com/apache/skywalking/tree/master/docs/en/protocols/README.md). Protocols show the communication ways between agents/probes, Satellite and SkyWalking. Anyone interested in uplink telemetry data should definitely read this.
+
+- [Change logs](../CHANGES.md). The feature records of the different versions.
+
+- [FAQs](en/FAQ/README.md). A manifest of already known setup problems, secondary developments experiments. When you are facing a problem, check here first.
+
+We're always looking for help improving our documentation and codes, so please don’t hesitate to [file an issue](https://github.com/apache/skywalking/issues/new)
+if you see any problem.
+Or better yet, submit your own contributions through pull request to help make them better.
diff --git a/docs/compile/compile.md b/docs/compile/compile.md
deleted file mode 100644
index 1d0d115..0000000
--- a/docs/compile/compile.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Compile
-## command
-```
-make build
-```
-## platform
-Linux and MacOs is supported in SkyWalking Satellite. Windows is not good supported beacuse some features is not adaptive on the Windows, such as the mmap feature.
-
-The Windows platform does not support plugins list:
-1. mmap-queue
\ No newline at end of file
diff --git a/docs/configuration/queue.md b/docs/configuration/queue.md
deleted file mode 100644
index 3781b3b..0000000
--- a/docs/configuration/queue.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# queue configuration
-
-| Type | Param | DefaultValue| Meaning|
-| ---- | ---- |---- | ---- |
-| mmap-queue | segment_size | 131072 | The size of each segment(Unit:Byte). The minimum value is the system memory page size.
-| mmap-queue | max_in_mem_segments | 10 | The max num of segments in memory. The minimum value is 4.
-| mmap-queue | queue_capacity_segments | 4000 | The capacity of Queue = segment_size * queue_capacity_segments.
-| mmap-queue | flush_period | 1000 | The period flush time. The unit is ms.
-| mmap-queue | flush_ceiling_num | 10000 | The max number in one flush time.
-| mmap-queue | queue_dir | satellite-mmap-queue |Contains all files in the queue.
-| mmap-queue | max_event_size | 20480 |The max size of the input event(Unit:Byte).
diff --git a/docs/design/module_structure.md b/docs/design/module_structure.md
deleted file mode 100644
index 54899aa..0000000
--- a/docs/design/module_structure.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# Module structure
-
-## Overview
-Module is the core workers in Satellite. Module is constituted by the specific extension plugins.
-There are 3 modules in one namespace, which are Gatherer, Processor, and Sender.
-
-- The Gatherer module is responsible for fetching or receiving data and pushing the data to Queue. So there are 2 kinds of Gatherer, which are ReceiverGatherer and FetcherGatherer.
-- The Processor module is responsible for reading data from the queue and processing data by a series of filter chains.
-- The Sender module is responsible for async processing and forwarding the data to the external services in the batch mode. After sending success, Sender would also acknowledge the offset of Queue in Gatherer.
-
-```
- Namespace
- --------------------------------------------------------------------
-| ---------- ----------- -------- |
-| | Gatherer | => | Processor | => | Sender | |
-| ---------- ----------- -------- |
- --------------------------------------------------------------------
-```
-
-## LifeCycle
-
-- Prepare: Prepare phase is to do some preparation works, such as register the client status listener to the client in ReceiverGatherer.
-- Boot: Boot phase is to start the current module until receives a close signal.
-- ShutDown: ShutDown phase is to close the used resources.
-
diff --git a/docs/en/FAQ/README.md b/docs/en/FAQ/README.md
new file mode 100644
index 0000000..d3be759
--- /dev/null
+++ b/docs/en/FAQ/README.md
@@ -0,0 +1,2 @@
+## FAQ
+
diff --git a/docs/en/concepts-and-designs/README.md b/docs/en/concepts-and-designs/README.md
new file mode 100644
index 0000000..89155c0
--- /dev/null
+++ b/docs/en/concepts-and-designs/README.md
@@ -0,0 +1,13 @@
+# Concepts and Designs
+Concepts and Designs help you to learn and understand the SkyWalking Satellite and the landscape.
+
+- What is SkyWalking Satellite?
+ - [Overview and Core concepts](overview.md). Provides a high-level description and introduction, including the problems the project solves.
+ - [Project Goals](project-goals.md). Provides the goals, which SkyWalking Satellite is trying to focus and provide features about them.
+
+After you read the above documents, you should understand basic goals of the SkyWalking Satellite. Now, you can choose which following parts
+you are interested, then dive in.
+
+- [Module Design](./module_design.md)
+- [Plugin mechanism](./plugin_mechanism.md)
+- [Project Structure](./project_structue.md)
diff --git a/docs/design/module_design.md b/docs/en/concepts-and-designs/module_design.md
similarity index 94%
rename from docs/design/module_design.md
rename to docs/en/concepts-and-designs/module_design.md
index 7217f94..f378d28 100644
--- a/docs/design/module_design.md
+++ b/docs/en/concepts-and-designs/module_design.md
@@ -18,7 +18,8 @@ Each pipe has one pipeline to process the telemetry data(metrics/traces/logs). T
---------------------------------------------------------------------
```
## Modules
-There are 3 modules in one pipe, which are Gatherer, Processor, and Sender.
+Module is the core workers in Satellite. Module is constituted by the specific extension plugins.
+There are 3 modules in one namespace, which are Gatherer, Processor, and Sender.
- The Gatherer module is responsible for fetching or receiving data and pushing the data to Queue. So there are 2 kinds of Gatherer, which are ReceiverGatherer and FetcherGatherer.
- The Processor module is responsible for reading data from the queue and processing data by a series of filter chains.
@@ -33,6 +34,13 @@ There are 3 modules in one pipe, which are Gatherer, Processor, and Sender.
--------------------------------------------------------------------
```
+LifeCycle
+
+- Prepare: Prepare phase is to do some preparation works, such as register the client status listener to the client in ReceiverGatherer.
+- Boot: Boot phase is to start the current module until receives a close signal.
+- ShutDown: ShutDown phase is to close the used resources.
+
+
## Plugins
Plugin is the minimal components in the module. Sateliite has 2 plugin catalogs, which are sharing plugins and normal plugins.
diff --git a/docs/en/concepts-and-designs/overview.md b/docs/en/concepts-and-designs/overview.md
new file mode 100644
index 0000000..7fee34b
--- /dev/null
+++ b/docs/en/concepts-and-designs/overview.md
@@ -0,0 +1,22 @@
+# Overview
+SkyWalking Satellite: an open-source agent designed for the cloud-native infrastructures, which provides a low-cost, high-efficient, and more secure way to collect telemetry data, such that Trace Segments, Logs, or Metrics.
+
+
+## Why use SkyWalking Satellite?
+Observability is the solution to the complex scenario of cloud-native services. However, we may encounter different telemetry data scenarios, different language services, big data analysis, etc. Satellite provides a unified data collection layer for cloud-native services.
+You can easily use it to connect to the SkyWalking ecosystem and enhance the capacity of SkyWalking.
+There are some enhance features on the following when using Satellite.
+
+1. Provide a unified data collection layer to collect logs, traces, and metrics.
+2. Provide a safer local cache to reduce the memory cost of the service.
+3. Provide the unified transfer way shields the functional differences in the different language libs, such as MQ.
+4. Provides the preprocessing functions to ensure accuracy of the metrics, such as sampling.
+
+## Architecture
+SkyWalking Satellite is logically split into three parts: Gatherer, Processor, and Sender.
+
+<img src="https://skywalking.apache.org/blog/2020-11-25-skywalking-satellite-0.1.0-design/Satellite.png"/>
+
+- Gatherer collect data and reformat them for SkyWalking requirements.
+- Processor processes the input data to generate the new data for Observability.
+- Sender would transfer the downstream data to the SkyWalking OAP with different protocols.
diff --git a/docs/design/plugin_structure.md b/docs/en/concepts-and-designs/plugin_mechanism.md
similarity index 94%
rename from docs/design/plugin_structure.md
rename to docs/en/concepts-and-designs/plugin_mechanism.md
index 060a1a8..3c5f460 100644
--- a/docs/design/plugin_structure.md
+++ b/docs/en/concepts-and-designs/plugin_mechanism.md
@@ -28,7 +28,7 @@ Initialize processing is like the following.
## Plugin usage in Satellite
-Nowadays, the numbers of the Plugin categories is 2. One is the [sharing Plugin](module_design.md), and another is the other [normal Plugin](module_design.md).
+Nowadays, the numbers of the Plugin categories is 2. One is the sharing Plugin, and another is the other normal Plugin.
- Extension Plugins:
- sharing plugins
diff --git a/docs/en/concepts-and-designs/project-goals.md b/docs/en/concepts-and-designs/project-goals.md
new file mode 100644
index 0000000..c620f4c
--- /dev/null
+++ b/docs/en/concepts-and-designs/project-goals.md
@@ -0,0 +1,15 @@
+# Design Goals
+The document outlines the core design goals for SkyWalking Satellite project.
+
+- **Light Weight**. SkyWalking Satellite has a limited cost for resources and high-performance because of the requirements of the sidecar deployment model.
+
+- **Pluggable**. SkyWalking Satellite core team provides many default implementations, but definitely it is not enough,
+and also don't fit every scenario. So, we provide a lot of features for being pluggable.
+
+- **Portability**. SkyWalking Satellite can run in multiple environments, including:
+ - Use traditional deployment as a demon process to collect data.
+ - Use cloud services as a sidecar, such as in the kubernetes platform.
+
+- **Interop**. Observability is a big landscape, SkyWalking is impossible to support all, even by its community. So SkyWalking Satellite is compatible with many protocols, including:
+ - SkyWalking protocol
+ - (WIP) Prometheus protocol.
diff --git a/docs/en/concepts-and-designs/project_structue.md b/docs/en/concepts-and-designs/project_structue.md
new file mode 100644
index 0000000..9693729
--- /dev/null
+++ b/docs/en/concepts-and-designs/project_structue.md
@@ -0,0 +1,33 @@
+# Project Structure
+- cmd: The starter of Satellite.
+- configs: Satellite configs.
+- internal: Core, API, and common utils.
+ - internal/pkg: Sharing with Core and Plugins, such as api and utils.
+ - internal/satellite: The core of Satellite.
+- plugins: Contains all plugins.
+ - plugins/{type}: Contains the plugins of this {type}. Satellite has 9 plugin types.
+ - plugins/{type}/api: Contains the plugin definition and initializer.
+ - plugins/{type}/{plugin-name}: Contains the specific plugin.
+ - init.go: Register the plugins to the plugin registry.
+```
+.
+├── CHANGES.md
+├── cmd
+├── configs
+├── docs
+├── go.sum
+├── internal
+│ ├── pkg
+│ └── satellite
+├── plugins
+│ ├── client
+│ ├── fallbacker
+│ ├── fetcher
+│ ├── filter
+│ ├── forwarder
+│ ├── init.go
+│ ├── parser
+│ ├── queue
+│ ├── receiver
+│ └── server
+```
diff --git a/docs/en/guides/README.md b/docs/en/guides/README.md
new file mode 100644
index 0000000..100b2d8
--- /dev/null
+++ b/docs/en/guides/README.md
@@ -0,0 +1,6 @@
+# Guides
+If you want to debug or develop SkyWalking Satellite, The following documentations would guide you.
+
+- [How to contribute a plugin?](./contribuation/plugin.md)
+- [How to add unit test for a plugin?](./test/test.md)
+- [How to compile SkyWalking Satellite?](./compile/compile.md)
diff --git a/docs/en/guides/compile/compile.md b/docs/en/guides/compile/compile.md
new file mode 100644
index 0000000..5519a4c
--- /dev/null
+++ b/docs/en/guides/compile/compile.md
@@ -0,0 +1,14 @@
+# Compiling
+
+## Platform
+Linux, MacOS and Windows are supported in SkyWalking Satellite. However, some components don't fit the Windows platform, including:
+1. mmap-queue
+
+## Command
+```shell script
+git clone https://github.com/apache/skywalking-satellite
+cd skywalking-satellite
+git submodule init
+git submodule update
+make build
+```
\ No newline at end of file
diff --git a/docs/en/guides/contribuation/plugin.md b/docs/en/guides/contribuation/plugin.md
new file mode 100644
index 0000000..ffbd624
--- /dev/null
+++ b/docs/en/guides/contribuation/plugin.md
@@ -0,0 +1,42 @@
+# How to write a new plugin?
+If you want to add a custom plugin in SkyWalking Satellite, the following contents would guide you.
+Let's use memory-queue as an example of how to write a plugin.
+
+1. Choose the plugin category. According to the memory-queue is a queue, the plugin should be write in the **skywalking-satellite/plugins/queue** directory. So we create a new directory called memory as the plugin codes space.
+
+2. Implement the interface in the **skywalking-satellite/plugins/queue/api**. Each plugin has 3 common methods, which are Name(), Description(), DefaultConfig().
+ - Name() returns the unique name in the plugin category.
+ - Description() returns the description of the plugin, which would be used to generate the plugin documentation.
+ - DefaultConfig() returns the default plugin config with yaml pattern, which would be used as the default value in the plugin struct and to generate the plugin documentation.
+ ```go
+ type Queue struct {
+ config.CommonFields
+ // config
+ EventBufferSize int `mapstructure:"event_buffer_size"` // The maximum buffer event size.
+
+ // components
+ buffer *goconcurrentqueue.FixedFIFO
+ }
+
+ func (q *Queue) Name() string {
+ return Name
+ }
+
+ func (q *Queue) Description() string {
+ return "this is a memory queue to buffer the input event."
+ }
+
+ func (q *Queue) DefaultConfig() string {
+ return `
+ # The maximum buffer event size.
+ event_buffer_size: 5000
+ ```
+
+3. Add [unit test](../test/test.md).
+4. Generate the plugin docs.
+```shell script
+make check
+```
+
+
+
diff --git a/docs/test/test.md b/docs/en/guides/test/test.md
similarity index 100%
rename from docs/test/test.md
rename to docs/en/guides/test/test.md
diff --git a/docs/en/setup/README.md b/docs/en/setup/README.md
new file mode 100644
index 0000000..77a2d2c
--- /dev/null
+++ b/docs/en/setup/README.md
@@ -0,0 +1,19 @@
+# Setup
+First and most important thing is, SkyWalking Satellite startup behaviours are driven by configs/satellite_config.yaml. Understood the setting file will help you to read this document.
+
+## Startup script
+The startup script is /bin/skywalking-satellite-{version}-{plateform}-amd64.
+1. Start SkyWalking Satellite.
+```shell script
+./bin/skywalking-satellite-{version}-{plateform}-amd64 start --config=./configs/satellite_config.yaml
+```
+## satellite_config.yaml
+The core concept behind this setting file is, SkyWalking Satellite is based on pure modularization design. End user can switch or assemble the collector features by their own requirements.
+
+So, in satellite_config.yaml, there are three parts.
+1. [The common configurations](./configuration/common.md).
+2. [The sharing plugin configurations](./configuration/sharing-plugins.md).
+3. [The pipe plugin configurations](./configuration/pipe-plugins.md).
+
+## Advanced feature document link list
+1. [Overriding settings](./configuration/override-settings.md) in satellite_config.yaml is supported
\ No newline at end of file
diff --git a/docs/en/setup/configuration/common.md b/docs/en/setup/configuration/common.md
new file mode 100644
index 0000000..e4581e2
--- /dev/null
+++ b/docs/en/setup/configuration/common.md
@@ -0,0 +1,18 @@
+# Common configuration
+The common configuration has 2 parts, which are logger
+configuration and the telemetry configuration.
+
+## Logger
+| Config |Default| Description |
+| ---- | ---- | ---- |
+| log_pattern | %time [%level][%field] - %msg | The log format pattern configuration.|
+| time_pattern | 2006-01-02 15:04:05.000 |The time format pattern configuration.|
+| level | info |The lowest level of printing allowed.|
+
+## Self Telemetry
+| Config |Default| Description |
+| ---- | ---- | ---- |
+| cluster | default-cluster | The space concept for the deployment, such as the namespace concept in the kubernetes.|
+| service | default-service | The group concept for the deployment, such as the service resource concept in the kubernetes.|
+| instance | default-instance |The minimum running unit, such as the pod concept in the kubernetes.|
+
diff --git a/docs/en/setup/configuration/override-settings.md b/docs/en/setup/configuration/override-settings.md
new file mode 100644
index 0000000..3bab93f
--- /dev/null
+++ b/docs/en/setup/configuration/override-settings.md
@@ -0,0 +1,24 @@
+# Setting Override
+SkyWalking Satellite supports setting overrides by system environment variables.
+You could override the settings in `satellite_config.yaml`
+
+
+## System environment variables
+- Example
+
+ Override `log_pattern` in this setting segment through environment variables
+
+```yaml
+logger:
+ log_pattern: ${SATELLITE_LOGGER_LOG_PATTERN:%time [%level][%field] - %msg}
+ time_pattern: ${SATELLITE_LOGGER_TIME_PATTERN:2006-01-02 15:04:05.000}
+ level: ${SATELLITE_LOGGER_LEVEL:info}
+```
+
+If the `SATELLITE_LOGGER_LOG_PATTERN ` environment variable exists in your operating system and its value is `%msg`,
+then the value of `log_pattern` here will be overwritten to `%msg`, otherwise, it will be set to `%time [%level][%field] - %msg`.
+
+
+
+
+
diff --git a/docs/en/setup/configuration/pipe-plugins.md b/docs/en/setup/configuration/pipe-plugins.md
new file mode 100644
index 0000000..eb47cb7
--- /dev/null
+++ b/docs/en/setup/configuration/pipe-plugins.md
@@ -0,0 +1,67 @@
+# Pipe Plugins
+The pipe plugin configurations contain a series of pipe configuration. Each pipe configuration has 5 parts, which are common_config, gatherer, processor and the sender.
+
+## common_config
+| Config | Description |
+| ---- | ---- |
+| pipe_name | The unique collect space name. |
+
+## Gatherer
+The gatherer has 2 roles, which are the receiver and fetcher.
+
+### Receiver Role
+| Config | Description |
+| ---- | ---- |
+| server_name | The server name in the sharing pipe, which would be used in the receiver plugin.|
+| receiver | The receiver configuration. Please read [the doc](../plugins/plugin-list.md) to find all receiver plugins.|
+| queue | The queue buffers the input telemetry data. Please read [the doc](../plugins/plugin-list.md) to find all queue plugins.|
+
+
+
+### Fetcher Role
+| Config | Description |
+| ---- | ---- |
+| fetch_interval | The time interval between two fetch operations. The time unit is millisecond.|
+| fetcher | The fetcher configuration. Please read [the doc](../plugins/plugin-list.md) to find all fetcher plugins.|
+| queue | The queue buffers the input telemetry data. Please read [the doc](../plugins/plugin-list.md) to find all queue plugins.|
+
+## processor
+The filter configuration. Please read [the doc](../plugins/plugin-list.md) to find all filter plugins.
+## sender
+| Config | Description |
+| ---- | ---- |
+| flush_time | The time interval between two flush operations. And the time unit is millisecond.|
+| max_buffer_size | The maximum buffer elements.|
+| min_flush_events | The minimum flush elements.|
+| client_name | The client name used in the forwarders of the sharing pipe.|
+| forwarders |The forwarder plugin list. Please read [the doc](../plugins/plugin-list.md) to find all forwarders plugins.|
+| fallbacker |The fallbacker plugin. Please read [the doc](../plugins/plugin-list.md) to find all fallbacker plugins.|
+
+
+## Example
+```yaml
+pipes:
+ - common_config:
+ pipe_name: pipe1
+ gatherer:
+ server_name: "grpc-server"
+ receiver:
+ plugin_name: "grpc-nativelog-receiver"
+ queue:
+ plugin_name: "mmap-queue"
+ segment_size: ${SATELLITE_MMAP_QUEUE_SIZE:524288}
+ max_in_mem_segments: ${SATELLITE_MMAP_QUEUE_MAX_IN_MEM_SEGMENTS:6}
+ queue_dir: "pipe1-log-grpc-receiver-queue"
+ processor:
+ filters:
+ sender:
+ fallbacker:
+ plugin_name: none-fallbacker
+ flush_time: ${SATELLITE_PIPE1_SENDER_FLUSH_TIME:1000}
+ max_buffer_size: ${SATELLITE_PIPE1_SENDER_MAX_BUFFER_SIZE:200}
+ min_flush_events: ${SATELLITE_PIPE1_SENDER_MIN_FLUSH_EVENTS:100}
+ client_name: kafka-client
+ forwarders:
+ - plugin_name: nativelog-kafka-forwarder
+ topic: ${SATELLITE_NATIVELOG-TOPIC:log-topic}
+```
\ No newline at end of file
diff --git a/docs/en/setup/configuration/sharing-plugins.md b/docs/en/setup/configuration/sharing-plugins.md
new file mode 100644
index 0000000..8621f78
--- /dev/null
+++ b/docs/en/setup/configuration/sharing-plugins.md
@@ -0,0 +1,29 @@
+# Sharing Plugins
+Sharing plugin configurations has three 3 parts, which are common_config, clients and servers.
+
+
+## Common Configuration
+| Config |Default| Description |
+| ---- | ---- | ---- |
+| pipe_name | sharing| The group name of sharing plugins |
+
+## Clients
+Clients have a series of client plugins, which would be sharing with the plugins of the other pipes. Please read [the doc](../plugins/plugin-list.md) to find all client plugin configurations.
+## Servers
+Servers have a series of server plugins, which would be sharing with the plugins of the other pipes. Please read [the doc](../plugins/plugin-list.md) to find all server plugin configurations.
+
+## Example
+```yaml
+# The sharing plugins referenced by the specific plugins in the different pipes.
+sharing:
+ common_config:
+ pipe_name: sharing
+ clients:
+ - plugin_name: "kafka-client"
+ brokers: ${SATELLITE_KAFKA_CLIENT_BROKERS:127.0.0.1:9092}
+ version: ${SATELLITE_KAFKA_VERSION:"2.1.1"}
+ servers:
+ - plugin_name: "grpc-server"
+ - plugin_name: "prometheus-server"
+ address: ${SATELLITE_PROMETHEUS_ADDRESS:":8090"}
+```
\ No newline at end of file
diff --git a/docs/en/setup/plugins/client_kafka-client.md b/docs/en/setup/plugins/client_kafka-client.md
new file mode 100755
index 0000000..39150b3
--- /dev/null
+++ b/docs/en/setup/plugins/client_kafka-client.md
@@ -0,0 +1,53 @@
+# Client/kafka-client
+## Description
+The Kafka client is a sharing plugin to keep connection with the Kafka brokers and delivery the data to it.
+## DefaultConfig
+```yaml
+# The Kafka broker addresses (default localhost:9092). Multiple values are separated by commas.
+brokers: localhost:9092
+
+# The Kakfa version should follow this pattern, which is major_minor_veryMinor_patch (default 1.0.0.0).
+version: 1.0.0.0
+
+# The TLS switch (default false).
+enable_TLS: false
+
+# The file path of client.pem. The config only works when opening the TLS switch.
+client_pem_path: ""
+
+# The file path of client.key. The config only works when opening the TLS switch.
+client_key_path: ""
+
+# The file path oca.pem. The config only works when opening the TLS switch.
+ca_pem_path: ""
+
+# 0 means NoResponse, 1 means WaitForLocal and -1 means WaitForAll (default 1).
+required_acks: 1
+
+# The producer max retry times (default 3).
+producer_max_retry: 3
+
+# The meta max retry times (default 3).
+meta_max_retry: 3
+
+# How long to wait for the cluster to settle between retries (default 100ms). Time unit is ms.
+retry_backoff: 100
+
+# The max message bytes.
+max_message_bytes: 1000000
+
+# If enabled, the producer will ensure that exactly one copy of each message is written (default false).
+idempotent_writes: false
+
+# A user-provided string sent with every request to the brokers for logging, debugging, and auditing purposes (default Satellite).
+client_id: Satellite
+
+# Compression codec represents the various compression codecs recognized by Kafka in messages. 0 : None, 1 : Gzip, 2 : Snappy, 3 : LZ4, 4 : ZSTD
+compression_codec: 0
+
+# How frequently to refresh the cluster metadata in the background. Defaults to 10 minutes. The unit is minute.
+refresh_period: 10
+
+# InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name.
+insecure_skip_verify: true
+```
diff --git a/docs/en/setup/plugins/fallbacker_none-fallbacker.md b/docs/en/setup/plugins/fallbacker_none-fallbacker.md
new file mode 100755
index 0000000..4b8fa4d
--- /dev/null
+++ b/docs/en/setup/plugins/fallbacker_none-fallbacker.md
@@ -0,0 +1,5 @@
+# Fallbacker/none-fallbacker
+## Description
+The fallbacker would nothing to do when facing failure data.
+## DefaultConfig
+```yaml```
diff --git a/docs/en/setup/plugins/fallbacker_timer-fallbacker.md b/docs/en/setup/plugins/fallbacker_timer-fallbacker.md
new file mode 100755
index 0000000..4329348
--- /dev/null
+++ b/docs/en/setup/plugins/fallbacker_timer-fallbacker.md
@@ -0,0 +1,8 @@
+# Fallbacker/timer-fallbacker
+## Description
+This is a timer fallback trigger to process the forward failure data.
+## DefaultConfig
+```yaml
+max_times: 3
+latency_factor: 2000
+```
diff --git a/docs/en/setup/plugins/forwarder_nativelog-kafka-forwarder.md b/docs/en/setup/plugins/forwarder_nativelog-kafka-forwarder.md
new file mode 100755
index 0000000..a7cf6fe
--- /dev/null
+++ b/docs/en/setup/plugins/forwarder_nativelog-kafka-forwarder.md
@@ -0,0 +1,8 @@
+# Forwarder/nativelog-kafka-forwarder
+## Description
+This is a synchronization Kafka forwarder with the SkyWalking native log protocol.
+## DefaultConfig
+```yaml
+# The remote topic.
+topic: "log-topic"
+```
diff --git a/docs/en/setup/plugins/plugin-list.md b/docs/en/setup/plugins/plugin-list.md
new file mode 100755
index 0000000..19a7023
--- /dev/null
+++ b/docs/en/setup/plugins/plugin-list.md
@@ -0,0 +1,21 @@
+# Plugin List
+- Client
+ - [kafka-client](./client_kafka-client.md)
+- Fallbacker
+ - [none-fallbacker](./fallbacker_none-fallbacker.md)
+ - [timer-fallbacker](./fallbacker_timer-fallbacker.md)
+- Fetcher
+- Filter
+- Forwarder
+ - [nativelog-kafka-forwarder](./forwarder_nativelog-kafka-forwarder.md)
+- Parser
+- Queue
+ - [memory-queue](./queue_memory-queue.md)
+ - [mmap-queue](./queue_mmap-queue.md)
+- Receiver
+ - [grpc-nativelog-receiver](./receiver_grpc-nativelog-receiver.md)
+ - [http-log-receiver](./receiver_http-log-receiver.md)
+- Server
+ - [grpc-server](./server_grpc-server.md)
+ - [http-server](./server_http-server.md)
+ - [prometheus-server](./server_prometheus-server.md)
diff --git a/docs/en/setup/plugins/queue_memory-queue.md b/docs/en/setup/plugins/queue_memory-queue.md
new file mode 100755
index 0000000..2d77288
--- /dev/null
+++ b/docs/en/setup/plugins/queue_memory-queue.md
@@ -0,0 +1,8 @@
+# Queue/memory-queue
+## Description
+This is a memory queue to buffer the input event.
+## DefaultConfig
+```yaml
+# The maximum buffer event size.
+event_buffer_size: 5000
+```
diff --git a/docs/en/setup/plugins/queue_mmap-queue.md b/docs/en/setup/plugins/queue_mmap-queue.md
new file mode 100755
index 0000000..2fe4602
--- /dev/null
+++ b/docs/en/setup/plugins/queue_mmap-queue.md
@@ -0,0 +1,20 @@
+# Queue/mmap-queue
+## Description
+This is a memory mapped queue to provide the persistent storage for the input event. Please note that this plugin does not support Windows platform.
+## DefaultConfig
+```yaml
+# The size of each segment. Default value is 128K. The unit is Byte.
+segment_size: 131072
+# The max num of segments in memory. Default value is 10.
+max_in_mem_segments: 10
+# The capacity of Queue = segment_size * queue_capacity_segments.
+queue_capacity_segments: 4000
+# The period flush time. The unit is ms. Default value is 1 second.
+flush_period: 1000
+# The max number in one flush time. Default value is 10000.
+flush_ceiling_num: 10000
+# Contains all files in the queue.
+queue_dir: satellite-mmap-queue
+# The max size of the input event. Default value is 20k.
+max_event_size: 20480
+```
diff --git a/docs/en/setup/plugins/receiver_grpc-nativelog-receiver.md b/docs/en/setup/plugins/receiver_grpc-nativelog-receiver.md
new file mode 100755
index 0000000..0790aa3
--- /dev/null
+++ b/docs/en/setup/plugins/receiver_grpc-nativelog-receiver.md
@@ -0,0 +1,5 @@
+# Receiver/grpc-nativelog-receiver
+## Description
+This is a receiver for SkyWalking native logging format, which is defined at https://github.com/apache/skywalking-data-collect-protocol/blob/master/logging/Logging.proto.
+## DefaultConfig
+```yaml```
diff --git a/docs/en/setup/plugins/receiver_http-log-receiver.md b/docs/en/setup/plugins/receiver_http-log-receiver.md
new file mode 100755
index 0000000..4e9496a
--- /dev/null
+++ b/docs/en/setup/plugins/receiver_http-log-receiver.md
@@ -0,0 +1,10 @@
+# Receiver/http-log-receiver
+## Description
+This is a receiver for SkyWalking http logging format, which is defined at https://github.com/apache/skywalking-data-collect-protocol/blob/master/logging/Logging.proto.
+## DefaultConfig
+```yaml
+# The native log request URI.
+uri: "/logging"
+# The request timeout seconds.
+timeout: 5
+```
diff --git a/docs/en/setup/plugins/server_grpc-server.md b/docs/en/setup/plugins/server_grpc-server.md
new file mode 100755
index 0000000..5a43558
--- /dev/null
+++ b/docs/en/setup/plugins/server_grpc-server.md
@@ -0,0 +1,18 @@
+# Server/grpc-server
+## Description
+This is a sharing plugin, which would start a gRPC server.
+## DefaultConfig
+```yaml
+# The address of grpc server. Default value is :11800
+address: :11800
+# The network of grpc. Default value is :tcp
+network: tcp
+# The max size of receiving log. Default value is 2M. The unit is Byte.
+max_recv_msg_size: 2097152
+# The max concurrent stream channels.
+max_concurrent_streams: 32
+# The TLS cert file path.
+tls_cert_file:
+# The TLS key file path.
+tls_key_file:
+```
diff --git a/docs/en/setup/plugins/server_http-server.md b/docs/en/setup/plugins/server_http-server.md
new file mode 100755
index 0000000..e2b3ab0
--- /dev/null
+++ b/docs/en/setup/plugins/server_http-server.md
@@ -0,0 +1,8 @@
+# Server/http-server
+## Description
+This is a sharing plugin, which would start a http server.
+## DefaultConfig
+```yaml
+# The http server address.
+address: ":12800"
+```
diff --git a/docs/en/setup/plugins/server_prometheus-server.md b/docs/en/setup/plugins/server_prometheus-server.md
new file mode 100755
index 0000000..fa0c2f4
--- /dev/null
+++ b/docs/en/setup/plugins/server_prometheus-server.md
@@ -0,0 +1,10 @@
+# Server/prometheus-server
+## Description
+This is a prometheus server to export the metrics in Satellite.
+## DefaultConfig
+```yaml
+# The prometheus server address.
+address: ":9299"
+# The prometheus server metrics endpoint.
+endpoint: "/metrics"
+```
diff --git a/docs/plugin-description.md b/docs/plugin-description.md
deleted file mode 100755
index 8b4d973..0000000
--- a/docs/plugin-description.md
+++ /dev/null
@@ -1,164 +0,0 @@
-# api.Client
-## kafka-client
-### description
-this is a sharing client to delivery the data to Kafka.
-### defaultConfig
-```yaml
-# The Kafka broker addresses (default localhost:9092). Multiple values are separated by commas.
-brokers: localhost:9092
-
-# The Kakfa version should follow this pattern, which is major_minor_veryMinor_patch (default 1.0.0.0).
-version: 1.0.0.0
-
-# The TLS switch (default false).
-enable_TLS: false
-
-# The file path of client.pem. The config only works when opening the TLS switch.
-client_pem_path: ""
-
-# The file path of client.key. The config only works when opening the TLS switch.
-client_key_path: ""
-
-# The file path oca.pem. The config only works when opening the TLS switch.
-ca_pem_path: ""
-
-# 0 means NoResponse, 1 means WaitForLocal and -1 means WaitForAll (default 1).
-required_acks: 1
-
-# The producer max retry times (default 3).
-producer_max_retry: 3
-
-# The meta max retry times (default 3).
-meta_max_retry: 3
-
-# How long to wait for the cluster to settle between retries (default 100ms). Time unit is ms.
-retry_backoff: 100
-
-# The max message bytes.
-max_message_bytes: 1000000
-
-# If enabled, the producer will ensure that exactly one copy of each message is written (default false).
-idempotent_writes: false
-
-# A user-provided string sent with every request to the brokers for logging, debugging, and auditing purposes (default Satellite).
-client_id: Satellite
-
-# Compression codec represents the various compression codecs recognized by Kafka in messages. 0 : None, 1 : Gzip, 2 : Snappy, 3 : LZ4, 4 : ZSTD
-compression_codec: 0
-
-# How frequently to refresh the cluster metadata in the background. Defaults to 10 minutes. The unit is minute.
-refresh_period: 10
-
-# InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name.
-insecure_skip_verify: true
-```
-# api.Fallbacker
-## none-fallbacker
-### description
-this is a nothing to do fallbacker.
-### defaultConfig
-```yaml```
-# api.Fallbacker
-## timer-fallbacker
-### description
-this is a timer fallback trigger when forward fails.
-### defaultConfig
-```yaml
-max_times: 3
-latency_factor: 2000
-```
-# api.Forwarder
-## nativelog-kafka-forwarder
-### description
-this is a synchronization Kafka log forwarder.
-### defaultConfig
-```yaml
-# The remote topic.
-topic: "log-topic"
-```
-# api.Queue
-## memory-queue
-### description
-this is a memory queue to buffer the input event.
-### defaultConfig
-```yaml
-# The maximum buffer event size.
-event_buffer_size: 5000
-```
-# api.Queue
-## mmap-queue
-### description
-this is a memory mapped queue to provide the persistent storage.
-### defaultConfig
-```yaml
-# The size of each segment. Default value is 128K. The unit is Byte.
-segment_size: 131072
-# The max num of segments in memory. Default value is 10.
-max_in_mem_segments: 10
-# The capacity of Queue = segment_size * queue_capacity_segments.
-queue_capacity_segments: 4000
-# The period flush time. The unit is ms. Default value is 1 second.
-flush_period: 1000
-# The max number in one flush time. Default value is 10000.
-flush_ceiling_num: 10000
-# Contains all files in the queue.
-queue_dir: satellite-mmap-queue
-# The max size of the input event. Default value is 20k.
-max_event_size: 20480
-```
-# api.Receiver
-## grpc-nativelog-receiver
-### description
-This is a receiver for SkyWalking native logging format, which is defined at https://github.com/apache/skywalking-data-collect-protocol/blob/master/logging/Logging.proto.
-### defaultConfig
-```yaml```
-# api.Receiver
-## http-log-receiver
-### description
-This is a receiver for SkyWalking http logging format, which is defined at https://github.com/apache/skywalking-data-collect-protocol/blob/master/logging/Logging.proto.
-### defaultConfig
-```yaml
-# The native log request URI.
-uri: "/logging"
-# The request timeout seconds.
-timeout: 5
-```
-# api.Server
-## grpc-server
-### description
-this is a grpc server
-### defaultConfig
-```yaml
-# The address of grpc server. Default value is :11800
-address: :11800
-# The network of grpc. Default value is :tcp
-network: tcp
-# The max size of receiving log. Default value is 2M. The unit is Byte.
-max_recv_msg_size: 2097152
-# The max concurrent stream channels.
-max_concurrent_streams: 32
-# The TLS cert file path.
-tls_cert_file:
-# The TLS key file path.
-tls_key_file:
-```
-# api.Server
-## http-server
-### description
-this is a http server.
-### defaultConfig
-```yaml
-# The http server address.
-address: ":12800"
-```
-# api.Server
-## prometheus-server
-### description
-this is a prometheus server to export the metrics in Satellite.
-### defaultConfig
-```yaml
-# The prometheus server address.
-address: ":9299"
-# The prometheus server metrics endpoint.
-endpoint: "/metrics"
-```
diff --git a/docs/plugins/queue/mmap/README.md b/docs/plugins/queue/mmap/README.md
deleted file mode 100644
index 1b76412..0000000
--- a/docs/plugins/queue/mmap/README.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# Design
-The mmap-queue is a big, fast, and persistent queue based on the memory-mapped files. One mmap-queue has a directory to store the whole data. The queue directory is made up of many segments and 1 metafile. This is originally implemented by [bigqueue](https://github.com/grandecola/bigqueue) project, we changed it a little for fitting the Satellite project requirements.
-
-- Segment: Segment is the real data store center, that provides large-space storage and does not reduce read and write performance as much as possible by using mmap. And we will avoid deleting files by reusing them.
-- Meta: The purpose of meta is to find the data that the consumer needs.
-
-## Meta
-Metadata only needs 80B to store the Metadata for the pipe. But for memory alignment, it takes at least one memory page size, which is generally 4K.
-```
-[ 8Bit ][ 8Bit ][ 8Bit ][ 8Bit ][ 8Bit ][ 8Bit ][ 8Bit ][ 8Bit ][ 8Bit ][ 8Bit ]
-[metaVersion][ ID ][ offset][ ID ][ offset][ ID ][ offset][ ID ][ offset][capacity]
-[metaVersion][writing offset][watermark offset][committed offset][reading offset][capacity]
-
-```
-### Transforming
-
-
-
-## Configuration
-[Configuration Params](../../../configuration/queue.md)
-
-## Segment
-Segments are a series of files of the same size. Each input data would cost `8Bit+Len(data)Bit` to store the raw bytes. The first 8Bit is equal to `Len(data)` for finding the ending position.
-### Swapper
-For the performance and resources thinking, we define a page replacement policy.
-
-- Keep the reading and writing segments on the memory.
-- When the mmapcount is greater or equals to the max_in_mem_segments, we first scan the read scope and then scan the written scope to swap the segments to promise the reading or writing segments are always in memory.
- - Read scope: [reading_offset - max_in_mem_segments,reading_offset - 1]
- - Written scope: [writing_offset - max_in_mem_segments,writing_offset - 1]
- - Each displacement operation guarantees at least `max_in_mem_segments/2-1` capacity available. Subtract operation to subtract the amount of memory that must always exist.
-
-## BenchmarkTest
-Test machine: macbook pro 2018
-
-```
-Model Name: MacBook Pro
-Model Identifier: MacBookPro15,1
-Processor Name: 6-Core Intel Core i7
-Processor Speed: 2.2 GHz
-Number of Processors: 1
-Total Number of Cores: 6
-L2 Cache (per Core): 256 KB
-L3 Cache: 9 MB
-Hyper-Threading Technology: Enabled
-Memory: 16 GB
-System Firmware Version: 1554.60.15.0.0 (iBridge: 18.16.13030.0.0,0
-```
-
-### enqueue operation
-
-```
-goos: darwin
-goarch: amd64
-pkg: github.com/apache/skywalking-satellite/plugins/queue/mmap
-BenchmarkEnqueue
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:10000
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:10000-12 45946 28185 ns/op 9884 B/op 9 allocs/op
-BenchmarkEnqueue/segmentSize:_256KB_maxInMemSegments:10_message:8KB_queueCapacity:10000
-BenchmarkEnqueue/segmentSize:_256KB_maxInMemSegments:10_message:8KB_queueCapacity:10000-12 68137 19142 ns/op 9838 B/op 9 allocs/op
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:20_message:8KB_queueCapacity:10000
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:20_message:8KB_queueCapacity:10000-12 47361 22318 ns/op 9884 B/op 9 allocs/op
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:10_message:16KB_queueCapacity:10000
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:10_message:16KB_queueCapacity:10000-12 24478 45501 ns/op 18934 B/op 10 allocs/op
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:100000
-BenchmarkEnqueue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:100000-12 45691 29413 ns/op 9884 B/op 9 allocs/op
-PASS
-```
-### enqueue and dequeue operation
-```
-goos: darwin
-goarch: amd64
-pkg: github.com/apache/skywalking-satellite/plugins/queue/mmap
-BenchmarkEnqueueAndDequeue
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:10000
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:10000-12 30657 34182 ns/op 28725 B/op 41 allocs/op
-BenchmarkEnqueueAndDequeue/segmentSize:_256KB_maxInMemSegments:10_message:8KB_queueCapacity:10000
-BenchmarkEnqueueAndDequeue/segmentSize:_256KB_maxInMemSegments:10_message:8KB_queueCapacity:10000-12 34617 31619 ns/op 28677 B/op 41 allocs/op
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:20_message:8KB_queueCapacity:10000
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:20_message:8KB_queueCapacity:10000-12 32440 38439 ns/op 28726 B/op 41 allocs/op
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:10_message:16KB_queueCapacity:10000
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:10_message:16KB_queueCapacity:10000-12 18554 56840 ns/op 54931 B/op 42 allocs/op
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:100000
-BenchmarkEnqueueAndDequeue/segmentSize:_128KB_maxInMemSegments:10_message:8KB_queueCapacity:100000-12 27303 40016 ns/op 28725 B/op 41 allocs/op
-PASS
-```
diff --git a/docs/project_structue.md b/docs/project_structue.md
deleted file mode 100644
index 8aefe8a..0000000
--- a/docs/project_structue.md
+++ /dev/null
@@ -1,82 +0,0 @@
-# Project Structure
-- configs: Satellite configs.
-- internal: Core, Api, and common utils.
-- internal/pkg: Sharing with Core and Plugins, such as api and utils.
-- internal/satellite: The core of Satellite.
-- plugins: Contains all plugins.
-- plugins/{type}: Contains the plugins of this {type}. Satellite has 9 plugin types.
-- plugins/api: Contains the plugin definition and initlizer.
-- plugins/{type}/{plugin-name}: Contains the specific plugin, and {plugin-name}-{type} would be registered as the plugin unique name in the registry.
-```
-.
-├── cmd
-│ ├── command.go
-│ └── main.go
-├── configs
-│ └── satellite_config.yaml
-├── docs
-│ ├── design
-│ │ ├── module_design.md
-│ │ ├── module_structure.md
-│ │ └── plugin_structure.md
-│ └── project_structue.md
-├── internal
-│ ├── container
-│ ├── pkg
-│ │ ├── event
-│ │ ├── log
-│ │ ├── plugin
-│ │ │ ├── definition.go
-│ │ │ ├── plugin_test.go
-│ │ │ └── registry.go
-│ └── satellite
-│ ├── boot
-│ │ └── boot.go
-│ ├── config
-│ ├── event
-│ ├── module
-│ │ ├── api
-│ │ ├── buffer
-│ │ ├── gatherer
-│ │ ├── processor
-│ │ └── sender
-│ └── sharing
-└── plugins
- ├── client
- │ ├── api
- │ │ ├── client.go
- │ │ └── client_repository.go
- ├── fallbacker
- │ ├── api
- │ │ ├── fallbacker.go
- │ │ └── fallbacker_repository.go
- ├── fetcher
- │ └── api
- │ ├── fetcher.go
- │ └── fetcher_repository.go
- ├── filter
- │ ├── api
- │ │ ├── filter.go
- │ │ └── filter_repository.go
- ├── forwarder
- │ ├── api
- │ │ ├── forwarder.go
- │ │ └── forwarder_repository.go
- ├── init.go
- ├── parser
- │ ├── api
- │ │ ├── parser.go
- │ │ └── parser_repository.go
- ├── queue
- │ ├── api
- │ │ ├── queue.go
- │ │ └── queue_repository.go
- ├── receiver
- │ ├── api
- │ │ ├── receiver.go
- │ │ └── receiver_repository.go
- └── server
- └── api
- ├── server.go
- └── server_repository.go
-```
diff --git a/internal/satellite/tools/generate_plugin_doc.go b/internal/satellite/tools/generate_plugin_doc.go
index 3dd3fda..7b0d32c 100644
--- a/internal/satellite/tools/generate_plugin_doc.go
+++ b/internal/satellite/tools/generate_plugin_doc.go
@@ -31,24 +31,30 @@ import (
)
const (
- docDir = "docs"
- docPath = docDir + "/plugin-description.md"
topLevel = "# "
- SecondLevel = "## "
- thirdLevel = "### "
- LF = "\n"
+ secondLevel = "## "
+ lf = "\n"
yamlQuoteStart = "```yaml"
yamlQuoteEnd = "```"
- descStr = "description"
- confStr = "defaultConfig"
+ markdownSuffix = ".md"
)
-func GeneratePluginDoc() error {
+func GeneratePluginDoc(docDir string) error {
log.Init(&log.LoggerConfig{})
plugins.RegisterPlugins()
- // the generated doc content
- var doc string
- // sort categories by dictionary sequence
+
+ if err := createDir(docDir); err != nil {
+ return fmt.Errorf("create docs dir error: %v", err)
+ }
+ if err := generatePluginListDoc(docDir, getSortedCategories()); err != nil {
+ return err
+ }
+ log.Logger.Info("Successfully generate documentation!")
+ return nil
+}
+
+// sort categories by dictionary sequence
+func getSortedCategories() []reflect.Type {
var categories []reflect.Type
for c := range plugin.Reg {
categories = append(categories, c)
@@ -56,35 +62,61 @@ func GeneratePluginDoc() error {
sort.Slice(categories, func(i, j int) bool {
return strings.Compare(categories[i].String(), categories[j].String()) <= 0
})
+ return categories
+}
+
+func generatePluginListDoc(docDir string, categories []reflect.Type) error {
+ fileName := docDir + "/" + "plugin-list" + markdownSuffix
+ doc := topLevel + "Plugin List" + lf
for _, category := range categories {
- mapping := plugin.Reg[category]
- var keys []string
- for k := range mapping {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- for _, key := range keys {
- p := plugin.Get(category, plugin.Config{plugin.NameField: key})
- doc += topLevel + category.String() + LF
- doc += SecondLevel + key + LF
- doc += thirdLevel + descStr + LF + p.Description() + LF
- doc += thirdLevel + confStr + LF + yamlQuoteStart + p.DefaultConfig() + yamlQuoteEnd + LF
+ doc += "- " + category.Name() + lf
+ pluginList := getPluginsByCategory(category)
+ for _, pluginName := range pluginList {
+ doc += " - [" + pluginName + "](./" + getPluginDocFileName(category, pluginName) + ")" + lf
+ if err := generatePluginDoc(docDir, category, pluginName); err != nil {
+ return err
+ }
}
}
- if err := createDir(docDir); err != nil {
- return fmt.Errorf("create docs dir error: %v", err)
+ return writeDoc([]byte(doc), fileName)
+}
+
+func generatePluginDoc(docDir string, category reflect.Type, pluginName string) error {
+ docFileName := docDir + "/" + getPluginDocFileName(category, pluginName)
+ p := plugin.Get(category, plugin.Config{plugin.NameField: pluginName})
+ doc := topLevel + category.Name() + "/" + pluginName + lf
+ doc += secondLevel + "Description" + lf
+ doc += p.Description() + lf
+ doc += secondLevel + "DefaultConfig" + lf
+ doc += yamlQuoteStart + p.DefaultConfig() + yamlQuoteEnd + lf
+ return writeDoc([]byte(doc), docFileName)
+}
+
+func getPluginsByCategory(category reflect.Type) []string {
+ mapping := plugin.Reg[category]
+ var keys []string
+ for k := range mapping {
+ keys = append(keys, k)
}
- if err := ioutil.WriteFile(docPath, []byte(doc), os.ModePerm); err != nil {
+ sort.Strings(keys)
+ return keys
+}
+
+func getPluginDocFileName(category reflect.Type, pluginName string) string {
+ return strings.ToLower(category.Name() + "_" + pluginName + markdownSuffix)
+}
+
+func writeDoc(doc []byte, docFileName string) error {
+ if err := ioutil.WriteFile(docFileName, doc, os.ModePerm); err != nil {
return fmt.Errorf("cannot init the plugin doc: %v", err)
}
- log.Logger.Info("Successfully generate documentation!")
return nil
}
func createDir(path string) error {
fileInfo, err := os.Stat(path)
if os.IsNotExist(err) || fileInfo.Size() == 0 {
- return os.Mkdir(docDir, os.ModePerm)
+ return os.Mkdir(path, os.ModePerm)
}
return err
}
diff --git a/plugins/client/kafka/client.go b/plugins/client/kafka/client.go
index c898442..7f94432 100644
--- a/plugins/client/kafka/client.go
+++ b/plugins/client/kafka/client.go
@@ -64,7 +64,7 @@ func (c *Client) Name() string {
}
func (c *Client) Description() string {
- return "this is a sharing client to delivery the data to Kafka."
+ return "The Kafka client is a sharing plugin to keep connection with the Kafka brokers and delivery the data to it."
}
func (c *Client) DefaultConfig() string {
diff --git a/plugins/fallbacker/none/none_fallbacker.go b/plugins/fallbacker/none/none_fallbacker.go
index fdfd444..ffb4822 100644
--- a/plugins/fallbacker/none/none_fallbacker.go
+++ b/plugins/fallbacker/none/none_fallbacker.go
@@ -34,7 +34,7 @@ func (f *Fallbacker) Name() string {
}
func (f *Fallbacker) Description() string {
- return "this is a nothing to do fallbacker."
+ return "The fallbacker would nothing to do when facing failure data."
}
func (f *Fallbacker) DefaultConfig() string {
diff --git a/plugins/fallbacker/timer/timer_fallbacker.go b/plugins/fallbacker/timer/timer_fallbacker.go
index 324f4b0..49f4cf5 100644
--- a/plugins/fallbacker/timer/timer_fallbacker.go
+++ b/plugins/fallbacker/timer/timer_fallbacker.go
@@ -38,7 +38,7 @@ func (t *Fallbacker) Name() string {
}
func (t *Fallbacker) Description() string {
- return "this is a timer fallback trigger when forward fails."
+ return "This is a timer fallback trigger to process the forward failure data."
}
func (t *Fallbacker) DefaultConfig() string {
diff --git a/plugins/forwarder/kafka/nativelog/sync_forwarder.go b/plugins/forwarder/kafka/nativelog/sync_forwarder.go
index 98979cd..942d9e6 100644
--- a/plugins/forwarder/kafka/nativelog/sync_forwarder.go
+++ b/plugins/forwarder/kafka/nativelog/sync_forwarder.go
@@ -44,7 +44,7 @@ func (f *Forwarder) Name() string {
}
func (f *Forwarder) Description() string {
- return "this is a synchronization Kafka log forwarder."
+ return "This is a synchronization Kafka forwarder with the SkyWalking native log protocol."
}
func (f *Forwarder) DefaultConfig() string {
diff --git a/plugins/queue/memory/queue.go b/plugins/queue/memory/queue.go
index 3a34e00..15fe84a 100644
--- a/plugins/queue/memory/queue.go
+++ b/plugins/queue/memory/queue.go
@@ -45,7 +45,7 @@ func (q *Queue) Name() string {
}
func (q *Queue) Description() string {
- return "this is a memory queue to buffer the input event."
+ return "This is a memory queue to buffer the input event."
}
func (q *Queue) DefaultConfig() string {
diff --git a/plugins/queue/mmap/branchmark_test.go b/plugins/queue/mmap/branchmark_test.go
index 8b8e42d..4a385d2 100644
--- a/plugins/queue/mmap/branchmark_test.go
+++ b/plugins/queue/mmap/branchmark_test.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package mmap
import (
diff --git a/plugins/queue/mmap/meta/meta.go b/plugins/queue/mmap/meta/meta.go
index ea36595..11f75fa 100644
--- a/plugins/queue/mmap/meta/meta.go
+++ b/plugins/queue/mmap/meta/meta.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package meta
import (
diff --git a/plugins/queue/mmap/meta/meta_test.go b/plugins/queue/mmap/meta/meta_test.go
index fcc6b57..29f7126 100644
--- a/plugins/queue/mmap/meta/meta_test.go
+++ b/plugins/queue/mmap/meta/meta_test.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package meta
import (
diff --git a/plugins/queue/mmap/queue.go b/plugins/queue/mmap/queue.go
index 5a5edda..c717a35 100644
--- a/plugins/queue/mmap/queue.go
+++ b/plugins/queue/mmap/queue.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package mmap
import (
@@ -78,7 +80,8 @@ func (q *Queue) Name() string {
}
func (q *Queue) Description() string {
- return "this is a memory mapped queue to provide the persistent storage."
+ return "This is a memory mapped queue to provide the persistent storage for the input event." +
+ " Please note that this plugin does not support Windows platform."
}
func (q *Queue) DefaultConfig() string {
diff --git a/plugins/queue/mmap/queue_opreation.go b/plugins/queue/mmap/queue_opreation.go
index 056069c..64c5bba 100644
--- a/plugins/queue/mmap/queue_opreation.go
+++ b/plugins/queue/mmap/queue_opreation.go
@@ -20,6 +20,8 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
+// +build !windows
+
package mmap
import (
diff --git a/plugins/queue/mmap/queue_test.go b/plugins/queue/mmap/queue_test.go
index 8d29562..b172ea0 100644
--- a/plugins/queue/mmap/queue_test.go
+++ b/plugins/queue/mmap/queue_test.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package mmap
import (
diff --git a/plugins/queue/mmap/segment/segment.go b/plugins/queue/mmap/segment/segment.go
index cf738d1..720769a 100644
--- a/plugins/queue/mmap/segment/segment.go
+++ b/plugins/queue/mmap/segment/segment.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package segment
import (
diff --git a/plugins/queue/mmap/segment/segment_test.go b/plugins/queue/mmap/segment/segment_test.go
index 98c38e6..07915ef 100644
--- a/plugins/queue/mmap/segment/segment_test.go
+++ b/plugins/queue/mmap/segment/segment_test.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package segment
import (
diff --git a/plugins/queue/mmap/segment_operation.go b/plugins/queue/mmap/segment_operation.go
index d586057..e80e120 100644
--- a/plugins/queue/mmap/segment_operation.go
+++ b/plugins/queue/mmap/segment_operation.go
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+// +build !windows
+
package mmap
import (
diff --git a/plugins/server/grpc/server.go b/plugins/server/grpc/server.go
index 993472f..c640399 100644
--- a/plugins/server/grpc/server.go
+++ b/plugins/server/grpc/server.go
@@ -47,7 +47,7 @@ func (s *Server) Name() string {
}
func (s *Server) Description() string {
- return "this is a grpc server"
+ return "This is a sharing plugin, which would start a gRPC server."
}
func (s *Server) DefaultConfig() string {
diff --git a/plugins/server/http/server.go b/plugins/server/http/server.go
index e63d16b..d23de7f 100644
--- a/plugins/server/http/server.go
+++ b/plugins/server/http/server.go
@@ -37,7 +37,7 @@ func (s *Server) Name() string {
}
func (s *Server) Description() string {
- return "this is a http server."
+ return "This is a sharing plugin, which would start a http server."
}
func (s *Server) DefaultConfig() string {
diff --git a/plugins/server/prometheus/prometheus.go b/plugins/server/prometheus/prometheus.go
index 768fd1b..ec32098 100644
--- a/plugins/server/prometheus/prometheus.go
+++ b/plugins/server/prometheus/prometheus.go
@@ -43,7 +43,7 @@ func (s *Server) Name() string {
}
func (s *Server) Description() string {
- return "this is a prometheus server to export the metrics in Satellite."
+ return "This is a prometheus server to export the metrics in Satellite."
}
func (s *Server) DefaultConfig() string {