You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by ti...@apache.org on 2021/02/23 07:45:55 UTC
[servicecomb-service-center] branch master updated: [SCB-2094]add
data-source.md (#866)
This is an automated email from the ASF dual-hosted git repository.
tianxiaoliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-service-center.git
The following commit(s) were added to refs/heads/master by this push:
new 81f6acb [SCB-2094]add data-source.md (#866)
81f6acb is described below
commit 81f6acb2393f5e2b7e3bcaa4e72a65e5d87ab9ce
Author: robotLJW <79...@qq.com>
AuthorDate: Tue Feb 23 15:45:47 2021 +0800
[SCB-2094]add data-source.md (#866)
---
datasource/etcd/client/config.go | 18 +--
datasource/etcd/config.go | 2 +
datasource/etcd/etcd.go | 16 +--
datasource/options.go | 5 -
docs/README.md | 5 +-
docs/user-guides.rst | 1 +
docs/user-guides/data-source.md | 128 +++++++++++++++++++++
etc/conf/app.yaml | 74 ++++++++----
.../start_scripts/linux/start-service-center.sh | 2 -
server/config/config.go | 3 -
server/config/server.go | 4 +-
server/server.go | 10 +-
12 files changed, 206 insertions(+), 62 deletions(-)
diff --git a/datasource/etcd/client/config.go b/datasource/etcd/client/config.go
index 438acae..70894cb 100644
--- a/datasource/etcd/client/config.go
+++ b/datasource/etcd/client/config.go
@@ -25,14 +25,16 @@ import (
)
type Config struct {
- SslEnabled bool `json:"-"`
- ManagerAddress string `json:"manageAddress,omitempty"`
- ClusterName string `json:"manageName,omitempty"`
- ClusterAddresses string `json:"manageClusters,omitempty"` // the raw string of cluster configuration
- Clusters cluster.Clusters `json:"-"` // parsed from ClusterAddresses
- DialTimeout time.Duration `json:"connectTimeout"`
- RequestTimeOut time.Duration `json:"registryTimeout"`
- AutoSyncInterval time.Duration `json:"autoSyncInterval"`
+ SslEnabled bool `json:"-"`
+ ManagerAddress string `json:"manageAddress,omitempty"`
+ ClusterName string `json:"manageName,omitempty"`
+ ClusterAddresses string `json:"manageClusters,omitempty"` // the raw string of cluster configuration
+ Clusters cluster.Clusters `json:"-"` // parsed from ClusterAddresses
+ DialTimeout time.Duration `json:"connectTimeout"`
+ RequestTimeOut time.Duration `json:"registryTimeout"`
+ AutoSyncInterval time.Duration `json:"autoSyncInterval"`
+ CompactIndexDelta int64 `json:"-"`
+ CompactInterval time.Duration `json:"-"`
}
//InitClusterInfo re-org address info with node name
diff --git a/datasource/etcd/config.go b/datasource/etcd/config.go
index ebcc773..5d45957 100644
--- a/datasource/etcd/config.go
+++ b/datasource/etcd/config.go
@@ -41,6 +41,8 @@ func Configuration() *client.Config {
defaultRegistryConfig.DialTimeout = config.GetDuration("registry.etcd.connect.timeout", client.DefaultDialTimeout, config.WithStandby("connect_timeout"))
defaultRegistryConfig.RequestTimeOut = config.GetDuration("registry.etcd.request.timeout", client.DefaultRequestTimeout, config.WithStandby("registry_timeout"))
defaultRegistryConfig.AutoSyncInterval = config.GetDuration("registry.etcd.autoSyncInterval", 30*time.Second, config.WithStandby("auto_sync_interval"))
+ defaultRegistryConfig.CompactIndexDelta = config.GetInt64("registry.etcd.compact.indexDelta", 100, config.WithStandby("compact_index_delta"))
+ defaultRegistryConfig.CompactInterval = config.GetDuration("registry.etcd.compact.interval", 12*time.Hour, config.WithStandby("compact_interval"))
})
return &defaultRegistryConfig
}
diff --git a/datasource/etcd/etcd.go b/datasource/etcd/etcd.go
index 5d16f84..096eff7 100644
--- a/datasource/etcd/etcd.go
+++ b/datasource/etcd/etcd.go
@@ -47,9 +47,6 @@ type DataSource struct {
SchemaEditable bool
// InstanceTTL options
InstanceTTL int64
- // Compact options
- CompactIndexDelta int64
- CompactInterval time.Duration
lockMux sync.Mutex
locks map[string]*etcdsync.DLock
@@ -60,12 +57,9 @@ func NewDataSource(opts datasource.Options) (datasource.DataSource, error) {
log.Warnf("data source enable etcd mode")
inst := &DataSource{
- SchemaEditable: opts.SchemaEditable,
- InstanceTTL: opts.InstanceTTL,
- CompactInterval: opts.CompactInterval,
- CompactIndexDelta: opts.CompactIndexDelta,
-
- locks: make(map[string]*etcdsync.DLock),
+ SchemaEditable: opts.SchemaEditable,
+ InstanceTTL: opts.InstanceTTL,
+ locks: make(map[string]*etcdsync.DLock),
}
registryAddresses := strings.Join(Configuration().RegistryAddresses(), ",")
@@ -122,8 +116,8 @@ func (ds *DataSource) initKvStore() {
}
func (ds *DataSource) autoCompact() {
- delta := ds.CompactIndexDelta
- interval := ds.CompactInterval
+ delta := Configuration().CompactIndexDelta
+ interval := Configuration().CompactInterval
if delta <= 0 || interval == 0 {
return
}
diff --git a/datasource/options.go b/datasource/options.go
index dd7eca2..6ee2356 100644
--- a/datasource/options.go
+++ b/datasource/options.go
@@ -17,8 +17,6 @@
package datasource
-import "time"
-
//Options contains configuration for plugins
type Options struct {
PluginImplName ImplName
@@ -26,8 +24,5 @@ type Options struct {
SchemaEditable bool
// InstanceTTL: the default ttl of instance lease
InstanceTTL int64
- // Compact options
- CompactIndexDelta int64
- CompactInterval time.Duration
// TODO: pay attention to more net config like TLSConfig when coding
}
diff --git a/docs/README.md b/docs/README.md
index bffe96a..21af891 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -27,4 +27,7 @@ make html
## Check the result
1. See html pages in _build folder
-2. You can start a http server using `python -m http.server` which will serve at http://0.0.0.0:8000/.
+```shell
+cd _build/html
+```
+2. You can start a http server using `python -m http.server` which will serve at http://127.0.0.1:8000/.
diff --git a/docs/user-guides.rst b/docs/user-guides.rst
index a06bafc..fad3bcd 100644
--- a/docs/user-guides.rst
+++ b/docs/user-guides.rst
@@ -7,6 +7,7 @@ User Guides
user-guides/pr-raising-guide.md
user-guides/security-tls.md
+ user-guides/data-source.md
user-guides/sc-cluster.rst
user-guides/integration-grafana.rst
user-guides/rbac.md
diff --git a/docs/user-guides/data-source.md b/docs/user-guides/data-source.md
new file mode 100644
index 0000000..6b8b5a8
--- /dev/null
+++ b/docs/user-guides/data-source.md
@@ -0,0 +1,128 @@
+# Data Source
+
+## etcd
+
+Download the etcd according to your own
+environment. [Installation package address](https://github.com/etcd-io/etcd/releases).
+
+Configure app.yaml according to your needs.
+
+```YAML
+registry:
+ # buildin, etcd, embeded_etcd, mongo
+ kind: etcd
+ # registry cache, if this option value set 0, service center can run
+ # in lower memory but no longer push the events to client.
+ cache:
+ mode: 1
+ # the cache will be clear after X, if not set cache will be never clear
+ ttl:
+ # enabled if registry.kind equal to etcd or embeded_etcd
+ etcd:
+ # the interval of etcd health check, aggregation conflict check and sync loop
+ autoSyncInterval: 30s
+ compact:
+ # indicate how many revision you want to keep in etcd
+ indexDelta: 100
+ interval: 12h
+ cluster:
+ # if registry_plugin equals to 'embeded_etcd', then
+ # name: sc-0
+ # managerEndpoints: http://127.0.0.1:2380"
+ # endpoints: sc-0=http://127.0.0.1:2380
+ # if registry_plugin equals to 'etcd', then
+ # endpoints: 127.0.0.1:2379
+ endpoints: 127.0.0.1:2379
+ # the timeout for failing to establish a connection
+ connect:
+ timeout: 10s
+ # the timeout for failing to read response of registry
+ request:
+ timeout: 30s
+```
+
+| field | description | required | value |
+| :---- | :---- | :---- | :---- |
+| registry.kind | database type (etcd or mongo) | yes | etcd / mongo |
+| registry.cache.mode | open cache (1 is on, 0 is off)| yes | 1 / 0 |
+| registry.cache.ttl | cache timeout (if not set cache will be never clear)| no | an integer time, like 30s/20m/10h |
+| registry.etcd.autoSyncInterval | synchronization interval | yes | an integer time, like 30s/20m/10h |
+| registry.etcd.compact.indexDelta | version retained in etcd | yes | a 64 bit integer, like 100 |
+| registry.etcd.compact.interval | compression interval | yes | an integer time, like 30s/20m/10h |
+| registry.etcd.cluster.endpoints | endpoints address | yes | string, like 127.0.0.1:2379 |
+| registry.etcd.connect.timeout | the timeout for establishing a connection | yes | an integer time, like 30s/20m/10h |
+| registry.etcd.request.timeout | request timeout | yes | an integer time, like 30s/20m/10h |
+
+**Download the installation package according to the environment information**
+
+1. Download etcd package.
+2. Unzip, modify the configuration and start etcd.
+3. Download the latest release from [ServiceComb Website](http://servicecomb.apache.org/release/).
+4. Decompress, modify /conf/app.yaml.
+5. Execute the start script to run service center
+
+## mongo
+
+Download the mongodb according to your own
+environment.[Installation package address](https://www.mongodb.com/try/download/community).
+
+Configure app.yaml according to your needs.
+
+```YAML
+registry:
+ # buildin, etcd, embeded_etcd, mongo
+ kind: mongo
+ # registry cache, if this option value set 0, service center can run
+ # in lower memory but no longer push the events to client.
+ cache:
+ mode: 1
+ # the cache will be clear after X, if not set cache will be never clear
+ ttl:
+ mongo:
+ heartbeat:
+ # Mongo's heartbeat plugin
+ # heartbeat.kind="checker or cache"
+ # if heartbeat.kind equals to 'cache', should set cacheCapacity,workerNum and taskTimeout
+ # capacity = 10000
+ # workerNum = 10
+ # timeout = 10
+ kind: cache
+ cacheCapacity: 10000
+ workerNum: 10
+ timeout: 10
+ cluster:
+ uri: mongodb://localhost:27017
+ sslEnabled: false
+ rootCAFile: /opt/ssl/ca.crt
+ verifyPeer: false
+ certFile: /opt/ssl/client.crt
+ keyFile: /opt/ssl/client.key
+```
+
+| field | description | required | value |
+| :---- | :---- | :---- | :---- |
+| registry.kind | database type (etcd or mongo) | yes | mongo / etcd |
+| registry.cache.mode | open cache (1 is on, 0 is off)| yes | 1 / 0 |
+| registry.cache.ttl | cache timeout (if not set cache will be never clear)| no | an integer time, like 30s/20m/10h |
+| registry.mongo.heartbeat.kind | there are two types of heartbeat plug-ins. With cache and without cache. | yes | cache/checker |
+| registry.mongo.heartbeat.cacheCapacity | cache capacity | yes | a integer, like 10000 |
+| registry.mongo.heartbeat.workerNum | the number of working cooperations | yes | a integer, like 10 |
+| registry.mongo.heartbeat.timeout | processing task timeout (default unit: s)| yes | a integer, like 10 |
+| registry.mongo.cluster.uri | mongodb server address | yes | string, like mongodb://localhost:27017 |
+| registry.mongo.cluster.sslEnabled | ssl enabled / not enabled | yes | false / true |
+| registry.mongo.cluster.rootCAFile | if sslEnabled equal true, should set CA file path | yes | string, like /opt/ssl/ca.crt |
+| registry.mongo.cluster.verifyPeer | insecure skip verify | yes | false / true |
+| registry.mongo.cluster.certFile | the cert file path need to be set according to the configuration of mongodb server | no | string, like /opt/ssl/client.crt |
+| registry.mongo.cluster.keyFile | the key file path need to be set according to the configuration of mongodb server | no | string, like /opt/ssl/client.key |
+
+**For example, I want to run the service-center in linux environment, the database is Mongo, and the SSL/TLS
+authentication is enabled for the database.**
+
+1. Download mongodb package.
+2. Unzip, modify the configuration and start mongodb.
+
+> [configure-ssl](https://docs.mongodb.com/v4.0/tutorial/configure-ssl/)
+
+3. Download the latest release from [ServiceComb Website](http://servicecomb.apache.org/release/).
+4. Decompress, modify /conf/app.yaml.
+5. Execute the start script to run service center
\ No newline at end of file
diff --git a/etc/conf/app.yaml b/etc/conf/app.yaml
index 9a21d6e..4f21a9d 100644
--- a/etc/conf/app.yaml
+++ b/etc/conf/app.yaml
@@ -14,19 +14,26 @@
# limitations under the License.
server:
- host:
- port:
+ host: 127.0.0.1
+ port: 30100
request:
- maxHeaderBytes:
- maxBodyBytes:
- headerTimeout:
- timeout:
+ maxHeaderBytes: 32768
+ maxBodyBytes: 2097152
+ headerTimeout: 60s
+ timeout: 60s
response:
- timeout:
+ timeout: 60s
idle:
- timeout:
+ timeout: 60s
pprof:
mode: 0
+ limit:
+ #ttl=m, s, ms
+ unit: s
+ #set 0 to disable rate limit
+ connections: 0
+ #list of places to look for IP address
+ ipLookups: RemoteAddr,X-Forwarded-For,X-Real-IP
gov:
plugins:
@@ -35,23 +42,35 @@ gov:
endpoint: http://127.0.0.1:30110
log:
+ # DEBUG, INFO, WARN, ERROR, FATAL
level: DEBUG
- file:
+ file: ./service-center.log
accessEnable: false
- accessFile:
- format:
+ # access log format: remoteIp requestReceiveTime "method requestUri proto" statusCode requestBodySize delay(ms)
+ # example: 127.0.0.1 2006-01-02T15:04:05.000Z07:00 "GET /v4/default/registry/microservices HTTP/1.1" 200 0 0
+ # access log inherits log's rotate and backup configuration
+ # whether enable access log
+ # enable_access_log = true
+ # access log file
+ accessFile: ./access.log
+ # log format(text or json type)
+ format: text
+ # whether enable record syslog
system: false
- rotateSize:
- backupCount:
+ # MaxSize of a log file before rotate. By M Bytes.
+ rotateSize: 20
+ # Max counts to keep of a log's backup files.
+ backupCount: 50
ssl:
dir:
# ssl.mode enable ssl or not, set 1 if enable
mode: 0
- minVersion:
+ # minimal tls protocol, [TLSv1.0, TLSv1.1, TLSv1.2]
+ minVersion: TLSv1.2
# ssl.verifyClient enable verify client certification CN
verifyClient: 1
- ciphers:
+ ciphers: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256
plugin:
# plugin.dir is the directory of the *.so files
@@ -60,16 +79,20 @@ plugin:
registry:
# buildin, etcd, embeded_etcd, mongo
kind: etcd
- compact:
- indexDelta:
- interval:
+ # registry cache, if this option value set 0, service center can run
+ # in lower memory but no longer push the events to client.
cache:
mode: 1
+ # the cache will be clear after X, if not set cache will be never clear
ttl:
# enabled if registry.kind equal to etcd or embeded_etcd
etcd:
# the interval of etcd health check, aggregation conflict check and sync loop
- autoSyncInterval:
+ autoSyncInterval: 30s
+ compact:
+ # indicate how many revision you want to keep in etcd
+ indexDelta: 100
+ interval: 12h
cluster:
# if registry_plugin equals to 'embeded_etcd', then
# name: sc-0
@@ -78,10 +101,12 @@ registry:
# if registry_plugin equals to 'etcd', then
# endpoints: 127.0.0.1:2379
endpoints: 127.0.0.1:2379
+ # the timeout for failing to establish a connection
connect:
- timeout:
+ timeout: 10s
+ # the timeout for failing to read response of registry
request:
- timeout:
+ timeout: 30s
mongo:
heartbeat:
# Mongo's heartbeat plugin
@@ -97,7 +122,7 @@ registry:
cluster:
uri: mongodb://localhost:27017
sslEnabled: false
- rootCAFile: /opt/ssl/ca.crt
+ rootCAFile: /opt/ssl/ca.pem
verifyPeer: false
certFile: /opt/ssl/client.crt
keyFile: /opt/ssl/client.key
@@ -121,8 +146,11 @@ registry:
# enable to register sc itself when startup
selfRegister: 1
+# pluggable discovery service
discovery:
- kind:
+ kind: etcd
+ aggregate:
+ mode:
rbac:
enable: false
diff --git a/scripts/release/start_scripts/linux/start-service-center.sh b/scripts/release/start_scripts/linux/start-service-center.sh
index 336bed8..23f187f 100644
--- a/scripts/release/start_scripts/linux/start-service-center.sh
+++ b/scripts/release/start_scripts/linux/start-service-center.sh
@@ -21,6 +21,4 @@ root_path=$(cd "$(dirname "$0")"; pwd)
cd ${root_path}
-sed -i "s|^runmode.*=.*$|runmode = prod|g" conf/app.conf
-
./service-center > start-sc.log 2>&1 &
diff --git a/server/config/config.go b/server/config/config.go
index facab7a..a4573f3 100644
--- a/server/config/config.go
+++ b/server/config/config.go
@@ -151,9 +151,6 @@ func newInfo() ServerInformation {
SslEnabled: GetInt("ssl.mode", 1, WithStandby("ssl_mode")) != 0,
- CompactIndexDelta: GetInt64("registry.compact.indexDelta", 100, WithStandby("compact_index_delta")),
- CompactInterval: GetDuration("registry.compact.interval", 12*time.Hour, WithStandby("compact_interval")),
-
LogRotateSize: maxLogFileSize,
LogBackupCount: maxLogBackupCount,
LogFilePath: GetString("log.file", "", WithStandby("logfile")),
diff --git a/server/config/server.go b/server/config/server.go
index c0138af..ddba080 100644
--- a/server/config/server.go
+++ b/server/config/server.go
@@ -38,9 +38,7 @@ type ServerConfig struct {
SslEnabled bool `json:"sslEnabled,string"`
- AutoSyncInterval time.Duration `json:"-"`
- CompactIndexDelta int64 `json:"-"`
- CompactInterval time.Duration `json:"-"`
+ AutoSyncInterval time.Duration `json:"-"`
EnablePProf bool `json:"enablePProf"`
EnableCache bool `json:"enableCache"`
diff --git a/server/server.go b/server/server.go
index 174b9b2..dd22263 100644
--- a/server/server.go
+++ b/server/server.go
@@ -110,12 +110,10 @@ func (s *ServiceCenterServer) initDatasource() {
// init datasource
kind := datasource.ImplName(config.GetString("registry.kind", "", config.WithStandby("registry_plugin")))
if err := datasource.Init(datasource.Options{
- PluginImplName: kind,
- SslEnabled: config.GetSSL().SslEnabled,
- InstanceTTL: config.GetRegistry().InstanceTTL,
- SchemaEditable: config.GetRegistry().SchemaEditable,
- CompactInterval: config.GetRegistry().CompactInterval,
- CompactIndexDelta: config.GetRegistry().CompactIndexDelta,
+ PluginImplName: kind,
+ SslEnabled: config.GetSSL().SslEnabled,
+ InstanceTTL: config.GetRegistry().InstanceTTL,
+ SchemaEditable: config.GetRegistry().SchemaEditable,
}); err != nil {
log.Fatal("init datasource failed", err)
}