You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/08/12 10:09:08 UTC
[kylin] branch doc5.0 updated (765c73be12 -> e840d1fb2a)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch doc5.0
in repository https://gitbox.apache.org/repos/asf/kylin.git
from 765c73be12 KYLIN-5221, add rdbms metastore
new 8ab848ed25 KYLIN-5221, add deploy mode
new 60c5380905 KYLIN-5221, add rest api
new e840d1fb2a KYLIN-5221, add monitoring in operations
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../deployment/deploy_mode/cluster_deployment.md | 86 +
.../deploy_mode/images/cluster_20191231.png | Bin 0 -> 199821 bytes
website/docs/deployment/deploy_mode/intro.md | 18 +
.../docs/deployment/deploy_mode/rw_separation.md | 90 +
.../deployment/deploy_mode/service_discovery.md | 36 +
.../rdbms_metastore/{ => mysql}/install_mysql.md | 6 +-
.../docs/deployment/rdbms_metastore/mysql/intro.md | 20 +
.../rdbms_metastore/mysql/mysql_metastore.md | 118 ++
.../{ => postgresql}/default_metastore.md | 0
.../{ => postgresql}/install_postgresql.md | 4 +-
.../deployment/rdbms_metastore/postgresql/intro.md | 20 +
website/docs/development/intro.md | 25 +
website/docs/intro.md | 14 +-
website/docs/modeling/intro.md | 25 +-
website/docs/operations/intro.md | 26 +-
.../operations/monitoring/images/dashboard.jpg | Bin 0 -> 235619 bytes
.../docs/operations/monitoring/images/interval.png | Bin 0 -> 162846 bytes
.../operations/monitoring/influxdb/influxdb.md | 211 +++
.../monitoring/influxdb/influxdb_maintenance.md | 124 ++
.../docs/operations/monitoring/influxdb/intro.md | 22 +
website/docs/operations/monitoring/intro.md | 24 +
.../docs/operations/monitoring/metrics_intro.md | 209 +++
website/docs/operations/monitoring/service.md | 181 ++
website/docs/restapi/acl_api/acl_api.md | 609 +++++++
website/docs/restapi/acl_api/intro.md | 18 +
website/docs/restapi/acl_api/project_acl_api.md | 217 +++
website/docs/restapi/acl_api/user_api.md | 343 ++++
website/docs/restapi/acl_api/user_group_api.md | 276 +++
website/docs/restapi/async_query_api.md | 414 +++++
website/docs/restapi/authentication.md | 111 ++
website/docs/restapi/callback_api.md | 153 ++
website/docs/restapi/data_source_api.md | 541 ++++++
website/docs/restapi/error_code.md | 156 ++
website/docs/restapi/image/job_api_en.png | Bin 0 -> 21482 bytes
website/docs/restapi/image/metrics_store_en.png | Bin 0 -> 245660 bytes
website/docs/restapi/image/work_follow_en.png | Bin 0 -> 50463 bytes
website/docs/restapi/intro.md | 25 +-
website/docs/restapi/job_api.md | 168 ++
website/docs/restapi/model_api/intro.md | 20 +
website/docs/restapi/model_api/model_build_api.md | 80 +
.../model_api/model_import_and_export_api.md | 310 ++++
.../docs/restapi/model_api/model_management_api.md | 1835 ++++++++++++++++++++
.../model_api/model_multilevel_partitioning_api.md | 366 ++++
website/docs/restapi/project_api.md | 700 ++++++++
website/docs/restapi/query_api.md | 543 ++++++
website/docs/restapi/segment_managment_api.md | 393 +++++
website/docs/restapi/snapshot_management_api.md | 304 ++++
website/docs/restapi/streaming_job_api.md | 282 +++
website/sidebars.js | 184 +-
49 files changed, 9262 insertions(+), 45 deletions(-)
create mode 100644 website/docs/deployment/deploy_mode/cluster_deployment.md
create mode 100644 website/docs/deployment/deploy_mode/images/cluster_20191231.png
create mode 100644 website/docs/deployment/deploy_mode/intro.md
create mode 100644 website/docs/deployment/deploy_mode/rw_separation.md
create mode 100644 website/docs/deployment/deploy_mode/service_discovery.md
rename website/docs/deployment/rdbms_metastore/{ => mysql}/install_mysql.md (97%)
create mode 100644 website/docs/deployment/rdbms_metastore/mysql/intro.md
create mode 100644 website/docs/deployment/rdbms_metastore/mysql/mysql_metastore.md
rename website/docs/deployment/rdbms_metastore/{ => postgresql}/default_metastore.md (100%)
rename website/docs/deployment/rdbms_metastore/{ => postgresql}/install_postgresql.md (98%)
create mode 100644 website/docs/deployment/rdbms_metastore/postgresql/intro.md
create mode 100644 website/docs/development/intro.md
create mode 100644 website/docs/operations/monitoring/images/dashboard.jpg
create mode 100644 website/docs/operations/monitoring/images/interval.png
create mode 100644 website/docs/operations/monitoring/influxdb/influxdb.md
create mode 100644 website/docs/operations/monitoring/influxdb/influxdb_maintenance.md
create mode 100644 website/docs/operations/monitoring/influxdb/intro.md
create mode 100644 website/docs/operations/monitoring/intro.md
create mode 100644 website/docs/operations/monitoring/metrics_intro.md
create mode 100644 website/docs/operations/monitoring/service.md
create mode 100644 website/docs/restapi/acl_api/acl_api.md
create mode 100644 website/docs/restapi/acl_api/intro.md
create mode 100644 website/docs/restapi/acl_api/project_acl_api.md
create mode 100644 website/docs/restapi/acl_api/user_api.md
create mode 100644 website/docs/restapi/acl_api/user_group_api.md
create mode 100644 website/docs/restapi/async_query_api.md
create mode 100644 website/docs/restapi/authentication.md
create mode 100644 website/docs/restapi/callback_api.md
create mode 100644 website/docs/restapi/data_source_api.md
create mode 100644 website/docs/restapi/error_code.md
create mode 100644 website/docs/restapi/image/job_api_en.png
create mode 100644 website/docs/restapi/image/metrics_store_en.png
create mode 100644 website/docs/restapi/image/work_follow_en.png
create mode 100644 website/docs/restapi/job_api.md
create mode 100644 website/docs/restapi/model_api/intro.md
create mode 100644 website/docs/restapi/model_api/model_build_api.md
create mode 100644 website/docs/restapi/model_api/model_import_and_export_api.md
create mode 100644 website/docs/restapi/model_api/model_management_api.md
create mode 100644 website/docs/restapi/model_api/model_multilevel_partitioning_api.md
create mode 100644 website/docs/restapi/project_api.md
create mode 100755 website/docs/restapi/query_api.md
create mode 100644 website/docs/restapi/segment_managment_api.md
create mode 100644 website/docs/restapi/snapshot_management_api.md
create mode 100644 website/docs/restapi/streaming_job_api.md
[kylin] 03/03: KYLIN-5221, add monitoring in operations
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch doc5.0
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e840d1fb2af5a143f8b43514a8fa699227da43da
Author: Mukvin <bo...@163.com>
AuthorDate: Fri Aug 12 18:04:29 2022 +0800
KYLIN-5221, add monitoring in operations
---
.../operations/monitoring/images/dashboard.jpg | Bin 0 -> 235619 bytes
.../docs/operations/monitoring/images/interval.png | Bin 0 -> 162846 bytes
.../operations/monitoring/influxdb/influxdb.md | 211 +++++++++++++++++++++
.../monitoring/influxdb/influxdb_maintenance.md | 124 ++++++++++++
.../docs/operations/monitoring/influxdb/intro.md | 22 +++
website/docs/operations/monitoring/intro.md | 24 +++
.../docs/operations/monitoring/metrics_intro.md | 209 ++++++++++++++++++++
website/docs/operations/monitoring/service.md | 181 ++++++++++++++++++
8 files changed, 771 insertions(+)
diff --git a/website/docs/operations/monitoring/images/dashboard.jpg b/website/docs/operations/monitoring/images/dashboard.jpg
new file mode 100644
index 0000000000..9d893aaa28
Binary files /dev/null and b/website/docs/operations/monitoring/images/dashboard.jpg differ
diff --git a/website/docs/operations/monitoring/images/interval.png b/website/docs/operations/monitoring/images/interval.png
new file mode 100644
index 0000000000..f89954398b
Binary files /dev/null and b/website/docs/operations/monitoring/images/interval.png differ
diff --git a/website/docs/operations/monitoring/influxdb/influxdb.md b/website/docs/operations/monitoring/influxdb/influxdb.md
new file mode 100644
index 0000000000..d0636f0f64
--- /dev/null
+++ b/website/docs/operations/monitoring/influxdb/influxdb.md
@@ -0,0 +1,211 @@
+---
+title: Use InfluxDB as Time-Series Database
+language: en
+sidebar_label: Use InfluxDB as Time-Series Database
+pagination_label: Use InfluxDB as Time-Series Database
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - influxdb
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+### <span id="preparation">Preparation</span>
+
+Starting with Kylin 5.0, the system uses RDBMS to store query history, which only use InfluxDB to record the monitoring information of the system.
+
+If you need this information, you need to configure the time series database InfluxDB in advance to store data such as the monitoring information of the system.
+
+We recommend you to use InfluxDB v1.6.4, which is provided in the Kylin installation package.
+
+The InfluxDB installation package, `influxdb-1.6.4.x86_64.rpm` is under the `influxdb` directory in the installation directory of Kylin.
+
+If you need to use the existed InfluxDB database in your environment, please use the versions below:
+
+- InfluxDB 1.6.4 or above
+
+You can use the following command to check the version of InfluxDB in your current environment.
+
+```shell
+service influxdb version
+```
+
+### <span id="root">Installation and Configuration for `root` User</span>
+
+The following steps are using InfluxDB 1.6.4 as an example.
+
+1. Run command to check if InfluxDB is installed already.
+
+ ```shell
+ service influxdb status
+ ```
+
+ If not, you can go to the directory where the InfluxDB installation package is located and install InfluxDB.
+
+ ```shell
+ cd $KYLIN_HOME/influxdb
+ rpm -ivh influxdb-1.6.4.x86_64.rpm
+ ```
+
+2. Launch InfluxDB.
+
+ ```sh
+ service influxdb start
+ ```
+
+ By default, you can find InfluxDB's log at `/var/log/influxdb`.
+
+3. If your InfluxDB server port is in use, you can modify the InfluxDB configuration file to change the server port.
+
+ ```sh
+ vi /etc/influxdb/influxdb.conf
+ ```
+
+ Please note the following three points:
+
+ - Modify RPC port: The initial property is `bind-address = "127.0.0.1:8088"`, you can change `8088` to an available port, for instance, `18087`.
+ - Modify HTTP port: The initial property is `bind-address = ":8086"`, you can change `8086` to available port, for instance, `18086`.
+ - Set `reporting-disabled = true`, which means that the InfluxDB will not send reports to [influxdata.com](https://www.influxdata.com/) regularly.
+
+4. InfluxDB is accessible without a user name and password by default. If you want to strengthen the security level, you can set a password with the following steps:
+
+ 1. log in InfluxDB.
+
+ ```sh
+ influx -port 18086
+ ```
+
+ **Tips:** Please replace `18086` with an actually available port number.
+
+ 2. Manage admin user and password.
+
+ ```mariadb
+ CREATE USER admin WITH PASSWORD 'admin' WITH ALL PRIVILEGES
+ ```
+
+ 3. Open the configuration file and modify ` [http] auth-enabled = true` to enable authorization.
+
+ ```sh
+ vi /etc/influxdb/influxdb.conf
+ ```
+
+ 4. Restart InfluxDB to take effect and login InfluxDB.
+
+ ```sh
+ service influxdb restart
+ influx -port 18086 -username admin -password admin
+ ```
+
+5. Open the property file `kylin.properties` and modify the InfluxDB configurations. Please replace `ip:http_port`, `user`, `pwd`, `ip:rpc_port` with real values.
+
+ ```properties
+ vi $KYLIN_HOME/conf/kylin.properties
+
+ ### Modify the following properties
+
+ kylin.influxdb.address=ip:http_port
+ kylin.influxdb.username=user
+ kylin.influxdb.password=pwd
+ kylin.metrics.influx-rpc-service-bind-address=ip:rpc_port
+ ```
+
+
+ **Note**: If more than one Kylin instances are deployed, you should configure the above configurations in `kylin.properties` for each Kylin node and let them point to the same Influxdb instance.
+
+6. Encrypt influxdb password
+
+ If you need to encrypt influxdb's password, you can do it like this：
+
+ **i.** run following commands in ${KYLIN_HOME}, it will print encrypted password
+ ```shell
+ ./bin/kylin.sh org.apache.kylin.tool.general.CryptTool -e AES -s <password>
+ ```
+
+ **ii.** config kylin.influxdb.password like this
+ ```properties
+ kylin.influxdb.password=ENC('${encrypted_password}')
+ ```
+
+ **iii.** Here is an example, assuming influxdb's password is kylin
+
+ First, we need to encrypt kylin using the following command
+ ```shell
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.general.CryptTool -e AES -s kylin
+ AES encrypted password is:
+ YeqVr9MakSFbgxEec9sBwg==
+ ```
+ Then, config kylin.metadata.url like this：
+ ```properties
+ kylin.influxdb.password=ENC('YeqVr9MakSFbgxEec9sBwg==')
+ ```
+
+7. start Kylin.
+
+### <span id="not_root">Installation and Configuration for Non `root` User </span>
+
+The following steps are using InfluxDB 1.6.4 as an example.
+
+
+1. Suppose you install as user `abc` . Then create a directory `home/abc/influx` to copy the InfluxDB installation package, `influxdb-1.6.4.x86_64.rpm` ,from `$KYLIN_HOME/influxdb` to this directory.
+
+ ```sh
+ mkdir /home/abc/influx
+ cp $KYLIN_HOME/influxdb/influxdb-1.6.4.x86_64.rpm /home/abc/influx
+ cd /home/abc/influx
+ rpm2cpio influxdb-1.6.4.x86_64.rpm | cpio -idmv
+ ```
+
+2. Edit the InfluxDB configuration file and replace `/var/lib` with `/home/abc/influx` globally. Also, you can modify `bind-address` property according to your case.
+
+ ```sh
+ vi /home/abc/influx/etc/influxdb/influxdb.conf
+ ```
+
+3. Run following command to launch InfluxDB.
+
+ ```sh
+ nohup ./usr/bin/influxd run -config /home/abc/influx/etc/influxdb/influxdb.conf &
+ ```
+ By default, you can find InfluxDB's log at `/home/abc/influx/var/log/influxdb`.
+
+4. As for other configurations, please refer to the second part in this section [`root` User Installation And Configuration](#root). Note that if you want to restart influxdb, you need to execute the following commands. Using `service influxdb restart` will not work since it requires root permission.
+
+ ```sh
+ ps -ef | grep influxdb
+ kill {pid}
+ ```
+
+5. Launch Kylin.
+
+
+### <span id="service">InfluxDB Connectivity</span>
+To ensure the connectivity of InfluxDB service, it is recommended that you perform some tests after starting InfluxDB.
+- Log in to InfluxDB by entering the command line in the terminal:
+ ```sh
+ /home/abc/influx/usr/bin/influx -port 18086 -username ${username} -password ${pwd}
+ ```
+ If the login fails, you can set `auth-enabled = true` in the configuration file `influxdb.conf` and try to login again.
+- After successful login , you can execute some simple queries to check if InfluxDB is configured correctly:
+ ```sql
+ show databases;
+ ```
+ If the query fails and the message `authorization failed` is displayed, please confirm whether the user has sufficient permissions.
+
+For more information about InfluxDB connectivity, please refer to the [InfluxDB Maintenance](../../Operation-and-Maintenance-Guide/influxdb/influxdb_maintenance.en.md) section.
+
+
+
+### <span id="https">（optional）Configure HTTPS connection to InfluxDB </span>
+
+Before using HTTPS to connect to InfluxDB, you need to enable InfluxDB's HTTPS connection. To enable HTTPS for InfluxDB please refer to the official documentation: [Enabling HTTPS with InfluxDB](https://docs.influxdata.com/influxdb/v1.6/administration/https_setup/)。
+
+If the InfluxDB you are using has enabled HTTPS connection, please set the following parameters in the `$KYLIN_HOME/conf/kylin.properties` configuration file:
+
+```
+kylin.influxdb.https.enabled=true
+```
diff --git a/website/docs/operations/monitoring/influxdb/influxdb_maintenance.md b/website/docs/operations/monitoring/influxdb/influxdb_maintenance.md
new file mode 100644
index 0000000000..632a6c675d
--- /dev/null
+++ b/website/docs/operations/monitoring/influxdb/influxdb_maintenance.md
@@ -0,0 +1,124 @@
+---
+title: InfluxDB Maintenance
+language: en
+sidebar_label: InfluxDB Maintenance
+pagination_label: InfluxDB Maintenance
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - influxdb
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+## InfluxDB Maintenance
+
+This chapter introduces the basic maintenance of InfluxDB.
+
+### Connectivity
+
+When InfluxDB is not accessible, you can locate the problem from the following aspects:
+
+1. Check if InfluxDB is running normally by executing `service influxdb status`. If it is not running, please check log files of `/var/log/influxdb/influxd.log` or `/var/log/messages` to find out the reason, at the same time, run `service influxdb restart` to restart InfluxDB service and make sure the service can be launched normally by observing the logs. (You should be able to login InfluxDB via `influx -host ? -port ?` command)
+
+2. If you find the port has been taken in the starting process, run `netstat -anp | grep influxdb_port` to get the process id, and execute `ps -ef | grep pid` to get the specific process. You can choose to kill the process if you do not need it or to change InfluxDB's server port to another.
+
+3. If you are having your Kylin and InfluxDB installed in different nodes, please execute `telnet influxdb_ip influxdb_port` on Kylin node to check if two nodes can communicate normally, if not, please make sure the Firewall service is not turned on on InfluxDB node via `service iptables status` command or contact the system admin to check the network condition.
+
+### Log Management
+
+- **Log Configuration**
+
+ - By default, InfluxDB writes standard error to log. InfluxDB redirects stderr to `/var/log/influxdb/influxd.log` file when it is started. If you would like to change the log path, please modify the property in the configuration file `/etc/default/influxdb` to `STDERR=/path/to/influxdb.log`, and restart the service via `service influxdb restart` command.
+ - InfluxDB enables HTTP access log by default. Generally, HTTP access log is quite large, you can modify the property `[http] log-enabled=false` to disable the log output.
+
+- **Log Clean**
+
+ InfluxDB itself does not clean its log regularly, it uses **logrotate** to manage log, which is installed on Linux system by default. The configuration file of **logrotate** is located at `/etc/logrotate.d/influxdb`, the log rotates by day, and the retention is 7 days.
+
+### Backup and Restore
+
+InfluxDB provides the availability to do backup and restore.
+
+- **Backup**
+
+ ```sh
+ influxd backup -portable -database KE_METRIC -host 127.0.0.1:8089 /path/to/backup
+ ```
+
+- **Restore**
+
+ Please make sure that the database exists, otherwise the restore will be failed.
+
+ ```sh
+ influxd restore -portable -database KE_METRIC -host 127.0.0.1:8089 /path/to/backup
+ ```
+
+> **note:** Please replace KE_METRIC with the actual database name, replace 127.0.0.1:8089 with the actual IP and port, replace `/path/to/backup` with the path you would like to set.
+
+### Monitoring and Diagnosis
+
+- **Memory Monitoring**
+
+ - Check runtime
+
+ Run following command to check GC, memory usage, etc.
+ `influx -database KE_METRIC -execute "show stats for 'runtime'"`
+
+ Please focus on these important arguments:
+ - *HeapAlloc* -> Heap allocation size
+ - *Sys* -> The total number of bytes of memory obtained from the system
+ - *NumGC* -> GC times
+ - *PauseTotalNs* -> The total GC pause time
+
+ - Check the memory usage of InfluxDB index
+
+ `show stats for 'indexes'`
+
+ - Monitor InfluxDB memory usage
+
+ Run following command:
+
+ `pidstat -rh -p PID 5`
+
+ If the memory usage is too high or GC is too frequent, please increase memory.
+
+ > **tips:** It is recommended to install InfluxDB on a separate machine with high memory allocation, because data read and write speed are dependent on the indexes, and the indexes are stored in memory.
+
+- **Disk Monitoring**
+
+ Run following command to check disk situation:
+
+ ```sh
+ pidstat -d -p PID 5
+ ```
+
+ When the disk read/write load is found to be too high, you can consider mapping the WAL directory and the data directory to different disks to reduce the interaction between read and write operations.
+
+ 1. Run `vi /etc/default/influxdb` to edit the configuration file.
+ 2. Modify the properties `[data] dir = "/var/lib/influxdb/data"` and `wal-dir = "/var/lib/influxdb/wal"` to point WAL directory and data directory to different disk.
+
+- **Read/Write Response Time**
+
+ 1. Write:
+
+ ```sql
+ SELECT non_negative_derivative(percentile("writeReqDurationNs", 99)) / non_negative_derivative(max("writeReq")) / (1000 * 1000) AS "Write Request"
+ FROM "_internal".."httpd"
+ WHERE time > now() - 10d
+ GROUP BY time(1h) fill(0)
+ ```
+
+ 2. Read:
+
+ ```sql
+ SELECT non_negative_derivative(percentile("queryReqDurationNs", 99)) / non_negative_derivative(max("queryReq")) / (1000 * 1000) AS "Query Request"
+ FROM "_internal".."httpd"
+ WHERE time > now() - 10d
+ GROUP BY time(1h)
+ ```
+
diff --git a/website/docs/operations/monitoring/influxdb/intro.md b/website/docs/operations/monitoring/influxdb/intro.md
new file mode 100644
index 0000000000..96ea606197
--- /dev/null
+++ b/website/docs/operations/monitoring/influxdb/intro.md
@@ -0,0 +1,22 @@
+---
+title: InfluxDB
+language: en
+sidebar_label: InfluxDB
+pagination_label: InfluxDB
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - monitoring
+ - maintenance
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+Kylin supports to use InfluxDB as its time series database, this chapter will cover:
+
+* [Use InfluxDB as Time-Series Database](influxdb.md)
+* [InfluxDB Maintenance](influxdb_maintenance.md)
+
diff --git a/website/docs/operations/monitoring/intro.md b/website/docs/operations/monitoring/intro.md
new file mode 100644
index 0000000000..2d2d302d57
--- /dev/null
+++ b/website/docs/operations/monitoring/intro.md
@@ -0,0 +1,24 @@
+---
+title: Monitoring
+language: en
+sidebar_label: Monitoring
+pagination_label: Monitoring
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - monitoring
+ - maintenance
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+This chapter will discuss how to do system monitoring, we will cover:
+
+* [InfluxDB](influxdb/intro.md)
+ * [Use InfluxDB as Time-Series Database](influxdb/influxdb.md)
+ * [InfluxDB Maintenance](influxdb/influxdb_maintenance.md)
+* [Metrics Monitoring](metrics_intro.md)
+* [Service Monitoring](service.md)
diff --git a/website/docs/operations/monitoring/metrics_intro.md b/website/docs/operations/monitoring/metrics_intro.md
new file mode 100644
index 0000000000..cee36b3265
--- /dev/null
+++ b/website/docs/operations/monitoring/metrics_intro.md
@@ -0,0 +1,209 @@
+---
+title: Metrics Monitoring
+language: en
+sidebar_label: Metrics Monitoring
+pagination_label: Metrics Monitoring
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - metrics monitoring
+draft: true
+last_update:
+ date: 12/08/2022
+---
+
+
+By default, the system collects metric data every minute, including storage, query, job, metadata, and cleanup mechanism. The monitoring data is stored in the specified [InfluxDB](https://www.influxdata.com/time-series-platform/) and displayed through [Grafana](https://grafana.com/grafana). It can help administrators to understand the health of the system in order to take necessary actions.
+
+> **Note**: Since Grafana depends on InfluxDB, please make sure that InfluxDB is correctly configured and started according to [Use InfluxDB as Time-Series Database](influxdb/influxdb.md) before you use Grafana.
+
+### <span id="grafana_startup">Grafana</span>
+
+1. **Working Directory**: ``` $KYLIN_HOME/grafana```
+2. **Configuration Directory**: ```$KYLIN_HOME/grafana/conf```
+3. **Start Grafana Command**: ```$ KYLIN_HOME/bin/grafana.sh start```
+4. **Stop Grafana Command**: ``` $ KYLIN_HOME/bin/grafana.sh stop```
+
+> Changing grafana configuration please refer to [Configuration](https://grafana.com/docs/installation/configuration/).
+
+After the startup is successful, you may access Grafana through web browser with default port: 3000, username: admin, password: admin
+
+[comment]: <#TODO> ()
+
+### <span id="dashboard">Dashboard</span>
+
+Default Dashboard: ```Kylin```
+
+The dashboard consists of 10 modules: Cluster, Summaries, Models, Queries, Favorites, Jobs, Cleanings, Metadata Operations, Transactions, among which Summaries module is automatically displayed in detail. Read more details about the modules, please refer to [Metrics Explanation](#explanation). If you want to make some changes for the dashboard, please refer to Grafana official website manual [Provisioning Grafana](https://grafana.com/docs/administration/provisioning/).
+
+### <span id="panel">Panel</span>
+
+Each indicator monitor corresponds to a specific panel.
+
+### <span id="__interval">Time Range</span>
+
+In the upper right corner of the dashboard, choose the time range. Time range: the time interval in which the indicator was observed.
+
+
+### <span id="granularity">Data Granularity</span>
+
+Located in the upper left corner of the dashboard, the data granularity: auto, 1m, 5m, 10m, 30m, 1h, 6h, 12h, 1d, 7d, 14d, 30d ('auto' is automatically adjusted according to the time range, such as the time range '30min' corresponding granularity 5min, and the granularity corresponding to the time range of 24h is 4h).
+
+### <span id="explanation">Metrics Explanation</span>
+
+- [**Cluster**: Cluster overiew](#cluster)
+- [**Summaries**: Global overview](#summaries)
+- [**Models**: Model related metrics](#models)
+- [**Queries**: Query related metrics](#queries)
+- [**Favorites**: Favorite Query related metrics](#favorites)
+- [**Jobs**: Job related metrics](#jobs)
+- [**Cleanings**: Cleanup mechanisms related metrics](#cleanings)
+- [**Metadata Operations**: Metadata operations related metrics](#metadata)
+- [**Transactions**: Transaction mechanisms related metrics](#transactions)
+
+> **Tip **: “Project related” in the following table indicates whether the metric is related to the project, “Y” indicates that the metric is related to the project, and “N” indicates that the metric is not related to the project. "Host related" in the following table indicates whether the metric is related to Kylin nodes, "Y" indicates that the metric is related to the Kylin nodes, "N" indicates that the metric is not related to the host. "all", "job", "query" is Kylin nodes' server mode.
+
+<span id="cluster">**Cluster**：Cluster overview</span>
+
+| Name | Meaning | Project related |
+| :------------- | :---------- | :----------- |
+| build_unavailable_duration | the unavailable time of building | N |
+| query_unavailable_duration | the unavailable time of query | N |
+
+<span id="summaries">**Summaries**: Global overview</span>
+
+| Name | Meaning | Project related | Host related | Remark |
+| :------------- | :---------- | :----------- | :----------- | :----------- |
+| summary_exec_total_times | Times of all indicators collected | N | Y(all, job, query) | The cost of collecting indicators |
+| summary_exec_total_duration | Duration of all indicators collected | N | Y(all, job, query) | The cost of collecting indicators |
+| num_of_projects | Total project number | N | N | - |
+| storage_size_gauge | Storage used of the system | Y | N | - |
+| num_of_users | Total user number | N | N | - |
+| num_of_hive_tables | Total data table number | Y | N | - |
+| num_of_hive_databases | Total database number | Y | N | - |
+| summary_of_heap | The heap size of Kylin | N | Y(all, job, query) | - |
+| usage_of_heap | The ratio of heap of Kylin | N | Y(all, job, query) | - |
+| count_of_garbage_collection | The count of garbage collection | N | Y(all, job, query) | - |
+| time_of_garbage_collection | The total time of garbage collection | N | Y(all, job, query) | - |
+| garbage_size_gauge | Storage used of garbage | Y | N | Refer to the definition of "Garbage" |
+| sparder_restart_total_times | "Sparder" restart times | N | Y(all, job, query) | "Sparder" is the internal query engine |
+| query_load | spark sql load | N | Y(all, query) | - |
+| cpu_cores | The number of cup cores for query configured in kylin.properties | N | Y(all, query) | Refer "Spark-related Configuration" |
+
+<span id="models">**Models**：Model related metrics</span>
+
+| Name | Meaning | Project related | Host related |
+| :------------- | :---------- | :----------- | :----------- |
+| model_num_gauge | "Model number: curve with time | Y | N |
+| non_broken_model_num_gauge | "Healthy model number" curve with time | Y | N |
+| last_query_time_of_models | The last query time of models | Y | N |
+| hit_count_of_models | The query hit count of models | Y | N |
+| storage_of_models | The storage of models | Y | N |
+| segments_num_of_models | The num of segments of models | Y | N |
+| model_build_duration | Total build time of models | Y | N |
+| model_wait_duration | Total wait time of models | Y | N |
+| number_of_indexes | indexes number of models | Y | N |
+| expansion_rate_of_models | Expansion rate of models | Y | N |
+| model_build_duration (avg) | Avg build time of models | Y | N |
+
+<span id="queries">**Queries**：Query related metrics</span>
+
+| Name | Meaning | Project related | Host related | Remark |
+| :------------- | :---------- | :----------- | :----------- | :----------- |
+| count_of_queries | Total count of queries | Y | Y(all, query) | - |
+| num_of_query_per_host | The num of query per host | N | Y(all, query) | - |
+| count_of_queries_hitting_agg_index | The count of queries hitting agg index | Y | Y(all, query) | - |
+| ratio_of_queries_hitting_agg_index | The ratio of queries hitting agg index | Y | Y(all, query) | - |
+| count_of_queries_hitting_table_index | The count of queries hitting table index | Y | Y(all, query) | - |
+| ratio_of_queries_hitting_table_index | The ratio of queries hitting table index | Y | Y(all, query) | - |
+| count_of_pushdown_queries | The count of pushdown queries | Y | Y(all, query) | - |
+| ratio_of_pushdown_queries | The ratio of pushdown queries | Y | Y(all, query) | - |
+| count_of_queries_hitting_cache | The count of queries hitting cache | Y | Y(all, query) | - |
+| ratio_of_queries_hitting_cache | The ratio of queries hitting cache | Y | Y(all, query) | - |
+| count_of_queries_less_than_1s | Total count of queries when duration is less than 1 second | Y | Y(all, query) | - |
+| ratio_of_queries_less_than_1s | The ratio of queries when duration is less than 1 second | Y | Y(all, query) | - |
+| count_of_queries_between_1s_and_3s | Total count of queries when duration is between 1 second and 3 seconds | Y | Y(all, query) | - |
+| ration_of_queries_between_1s_and_3s | The ratio of queries when duration is between 1 second and 3 seconds | Y | Y(all, query) | - |
+| count_of_queries_between_3s_and_5s | Total count of queries when duration is between 3 seconds and 5 seconds | Y | Y(all, query) | - |
+| ratio_of_queries_between_3s_and_5s | The ratio of queries when duration is between 3 seconds and 5 seconds | Y | Y(all, query) | - |
+| count_of_queries_between_5s_and_10s | Total count of queries when duration is between 5 seconds and 10 seconds | Y | Y(all, query) | - |
+| ratio_of_queries_between_5s_and_10s | The ratio of queries when duration is between 5 seconds and 10 seconds | Y | Y(all, query) | - |
+| count_of_queries_greater_than_10s | Total count of queries when duration exceeding 10 seconds | Y | Y(all, query) | - |
+| ratio_of_queries_greater_than_10s | The ratio of queries when duration exceeding 10 seconds | Y | Y(all, query) | - |
+| count_of_timeout_queries | The count of timeout queries | Y | Y(all, query) | - |
+| count_of_failed_queries | The count of failed queries | Y | Y(all, query) | - |
+| mean_time_of_query_per_host | The mean time of queries per host | N | Y(all, query) | - |
+| 99%_of_query_latency | Query duration 99-percentile | Y | Y(all, query) | - |
+| gt10s_query_rate_5-minute | Query duration exceeding 10s per second over 5 minutes | Y | Y(all, query) | - |
+| failed_query_rate_5-minute | Failed queries per second over 5 minutes | Y | Y(all, query) | - |
+| pushdown_query_rate_5-minute | Pushdown queries per second over 5 minutes | Y | Y(all, query) | - |
+| scan_bytes_of_99%_queries | Query scan bytes 99-percentile | Y | Y(all, query) | - |
+| query_scan_bytes_of_host | Query scan bytes per host | N | Y(all, query) |-|
+| mean_scan_bytes_of_queries | The mean scan bytes of queries | Y | Y(all, query) | - |
+
+<span id="favorites">**Favorites**：Favorite Query related metrics</span>
+
+| Name | Meaning | Project related | Host related | Remark |
+| :------------- | :---------- | :----------- | :----------- | :----------- |
+| fq_accepted_total_times | Favorite Query user submitted total times | Y | Y(all, job, query) | - |
+| fq_proposed_total_times | Favorite Query system triggered total times | Y | N | - |
+| fq_proposed_total_duration | Favorite Query system triggered total duration | Y | N |-|
+| failed_fq_proposed_total_times | Favorite Query system triggered failed total times | Y | N | Refer to the definition of "pushdown" |
+| fq_adjusted_total_times | Favorite Query system adjusted total times | Y | Y(all, job, query) | - |
+| fq_adjusted_total_duration | Favorite Query system adjusted total duration | Y | Y(all, job, query) | - |
+| fq_update_usage_total_times | Favorite Query usage updated total times | Y | N | - |
+| fq_update_usage_total_duration | Favorite Query usage updated total duration | Y | N | - |
+| failed_fq_update_usage_total_times | Favorite Query usage updated failed total times | Y | N | - |
+| fq_tobeaccelerated_num_gauge | Favorite Query to be accelerated | Y | N | - |
+| fq_accelerated_num_gauge | Favorite Query accelerated | Y | N | - |
+| fq_failed_num_gauge | Favorite Query accelerated failed times | Y | N | - |
+| fq_accelerating_num_gauge | Favorite Query accelerating | Y | N | - |
+| fq_pending_num_gauge | Favorite Query pending | Y | N | Favorite Query lacks of necessary conditions, such as missing column names, requiring user intervention |
+| fq_blacklist_num_gauge | Favorite Query in blacklist | Y | N | Refer to the definition of "Blacklist" |
+
+<span id="jobs">**Jobs**：Job related metrics</span>
+
+| Name | Meaning | Project related | Host related |
+| :------------- | :---------- | :----------- | :----------- |
+| num_of_jobs_created | Jobs created total number | Y | Y(all, job) |
+| num_of_jobs_finished | Jobs finished total number | Y | Y(all, job) |
+| num_of_running_jobs | The num of running jobs currently | Y | N |
+| num_of_pending_jobs | The num of pending jobs currently | Y | N |
+| num_of_error_jobs | The num of error jobs currently | Y | N |
+| count_of_error_jobs | The total count of error | Y | Y(all, job) |
+| finished_jobs_total_duration | Jobs finished total duration | Y | Y(all, job) |
+| job_duration_99p | Jobs duration 99-percentile | Y | Y(all, job) |
+| job_step_attempted_total_times | Jobs step attempted total times | Y | Y(all, job) |
+| failed_job_step_attempted_total_times | Jobs step attempted failed total times | Y | Y(all, job) |
+| job_resumed_total_times | Jobs resumed total times | Y | Y(all, job) |
+| job_discarded_total_times | Jobs discarded total times | Y | Y(all, job) |
+| job_duration | The build duration of job | Y | Y(all, job) |
+| job_wait_duration | The wait duration of job | Y | Y(all, job) |
+
+<span id="cleanings">**Cleanings**：Cleanup mechanisms related metrics</span>
+
+| Name | Meaning | Project related | Host related |
+| :------------- | :---------- | :----------- | :----------- |
+| storage_clean_total_times | Storage cleanup total times | N | Y(all, job, query) |
+| storage_clean_total_duration | Storage cleanup total duration | N | Y(all, job, query) |
+| failed_storage_clean_total_times | Storage cleanup failed total times | N | Y(all, job, query) |
+
+<span id="metadata">**Metadata Operations**：Metadata operations related metrics</span>
+
+| Name | Meaning | Project related | Host related | Remark |
+| :------------- | :---------- | :----------- | :----------- | :----------- |
+| metadata_clean_total_times | Metadata cleanup total times | Y | Y(all, job, query) | - |
+| metadata_backup_total_times | Metadata backup total times | Y | Y(all, job, query) | Differentiate projects and global |
+| metadata_backup_total_duration | Metadata backup total duration | Y | Y(all, job, query) | Differentiate projects and global |
+| failed_metadata_backup_total_times | Metadata backup failed total times | Y | Y(all, job, query) | Differentiate projects and global |
+| metadata_ops_total_times | Metadata daily operations total times | N | Y(all, job, query) | Fixed time per day (configurable): automatically backup metadata; rotate audit_log; cleanup metadata and storage space; adjust FQ; cleanup query histories. |
+| metadata_success_ops_total_times | Metadata daily operations failed total times | N | Y(all, job, query) |-|
+
+<span id="transactions">**Transactions**：Transaction mechanisms related metrics</span>
+
+| Name | Meaning | Project related | Host related | Remark |
+| :------------- | :---------- | :----------- | :----------- |:----------- |
+| transaction_retry_total_times | Transactions retried total times | Y | Y(all, job, query) | Differentiate projects, and, global |
+| transaction_latency_99p | Transactions duration 99-percentile | Y | Y(all, job, query) | Differentiate projects, and, global |
diff --git a/website/docs/operations/monitoring/service.md b/website/docs/operations/monitoring/service.md
new file mode 100644
index 0000000000..d1194457cf
--- /dev/null
+++ b/website/docs/operations/monitoring/service.md
@@ -0,0 +1,181 @@
+---
+title: Service Monitoring
+language: en
+sidebar_label: Service Monitoring
+pagination_label: Service Monitoring
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - service monitoring
+draft: true
+last_update:
+ date: 12/08/2022
+---
+
+## Service Monitoring
+
+Kylin provides the service monitoring for main components to help administrators obtain the service status and maintain instances.
+
+Currently, we provide the following methods to monitor the core components in Kylin:
+
+1. Query: each Query node will records its service status in InfluxDB
+2. Build: each All node will records the service status and job status in InfluxDB
+
+Two Rest APIs are provided to monitor and obtain the service status so that customers can integrate it with their own monitor platform.
+
+- Get the Kylin cluster status by monitor query and building services. If the status is `WARNING` or `CRASH`, it means the cluster is unstable.
+- Get the service unavailable time with the specified time range and some detailed monitor data to help admins to track and retrospect.
+
+### How to Use
+
+**Get Cluster Status**
+
+`GET http://host:port/kylin/api/monitor/status`
+
+- HTTP Header
+
+ - Accept: application/vnd.apache.kylin-v4-public+json
+ - Accept-Language: en
+ - Content-Type: application/json;charset=utf-8
+
+- Curl Request Example
+
+ ```
+ curl -X GET \
+ 'http://host:port/kylin/api/monitor/status' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Details
+
+ - `active_instances` number of active instances in current cluster.
+ - `query_status` query service status. It could be GOOD / WARNING / CRASH
+ - `job_status` building service status. It could be GOOD / WARNING / CRASH.
+ - `Job` job instance status. It will show the instance details and status.
+ - `query` query instance status. It will show the instance details and status.
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "active_instances": 1,
+ "query_status": "GOOD",
+ "job_status": "GOOD",
+ "job": [
+ {
+ "instance": "sandbox.hortonworks.com:7070",
+ "status": "GOOD"
+ }
+ ],
+ "query": [
+ {
+ "instance": "sandbox.hortonworks.com:7070",
+ "status": "GOOD"
+ }
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+**Get Cluster Status with Specific Time Range**
+
+`GET http://host:port/kylin/api/monitor/status/statistics`
+
+- HTTP Header
+
+ - Accept: application/vnd.apache.kylin-v4-public+json
+ - Accept-Language: en
+ - Content-Type: application/json;charset=utf-8
+
+- URL Parameters
+
+ - `start` - `required` `Long` timestamp. Get the monitor data greater than or equal to the timestamp.
+ - `end` - `reuquired` `Long` timestamp. Get the monitor data smaller than the timestamp.
+
+- Curl Example
+
+ ```
+ curl -X GET \
+ 'http://host:port/kylin/api/monitor/status/statistics?start=1583562358466&end=1583562358466' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Details
+
+ - `Start` start time of monitoring. It will be rounded down based on the interval of monitoring data. If the interval is 1 minute, it will only record data in minute level. For example, if the argument is `1587353550000`, it will be recognized as `1587353520000`. Therefore, the data might be inaccurate.
+ - `end` end time of monitoring. It will be rounded down based on the interval of monitoring data. If the interval is 1 minute, it will only record data in minute level. For example, if the argument is `1587353550000`, it will be recognized as `1587353520000`. Therefore, the data might be inaccurate.
+ - `interval` interval of monitor data, default value is 60000 ms (1 min)
+ - `job` job instance status. It will show the instance details and status, which includes unavailable time and counts. The time unit of unavailable time is ms.
+ - `query` query instance status. It will show the instance details and status, which includes unavailable time and counts. The time unit of unavailable time is ms.
+
+- Response Example
+
+ ```
+ {
+ "code":"000",
+ "data":{
+ "start":1584151560000,
+ "end":1584151680000,
+ "interval":60000,
+ "job":[
+ {
+ "instance":"sandbox.hortonworks.com:7070",
+ "details":[
+ {
+ "time":1584151572650,
+ "status":"GOOD"
+ },
+ {
+ "time":1584151632770,
+ "status":"GOOD"
+ }
+ ],
+ "unavailable_time":0,
+ "unavailable_count":0
+ }
+ ],
+ "query":[
+ {
+ "instance":"sandbox.hortonworks.com:7070",
+ "details":[
+ {
+ "time":1584151609142,
+ "status":"GOOD"
+ },
+ {
+ "time":1584151669142,
+ "status":"GOOD"
+ }
+ ],
+ "unavailable_time":0,
+ "unavailable_count":0
+ }
+ ]
+ },
+ "msg":""
+ }
+ ```
+
+
+
+
+### Know Limitation
+
+1. The detected query is constant query which will not scan HDFS files.
+2. InfluxDB is not high available now. Hence, some monitor data will be lost if the InfluxDB service is down.
+3. The job status will be inaccurate if deleting or discarding plenty of jobs.
+4. Since system monitoring depends on InfluxDB, if the system monitoring is still enabled (enabled by default) when InfluxDB is not configured, some useless errors may appear in the log. So when InfluxDB is not configured, it is recommended to configure `kylin.monitor.enabled = false` in `kylin.properties` to turn off the system monitoring function.
+
[kylin] 02/03: KYLIN-5221, add rest api
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch doc5.0
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 60c53809051ed1d3d820944437e2295f342ba2fc
Author: Mukvin <bo...@163.com>
AuthorDate: Fri Aug 12 18:03:50 2022 +0800
KYLIN-5221, add rest api
---
website/docs/development/intro.md | 25 +
website/docs/restapi/acl_api/acl_api.md | 609 +++++++
website/docs/restapi/acl_api/intro.md | 18 +
website/docs/restapi/acl_api/project_acl_api.md | 217 +++
website/docs/restapi/acl_api/user_api.md | 343 ++++
website/docs/restapi/acl_api/user_group_api.md | 276 +++
website/docs/restapi/async_query_api.md | 414 +++++
website/docs/restapi/authentication.md | 111 ++
website/docs/restapi/callback_api.md | 153 ++
website/docs/restapi/data_source_api.md | 541 ++++++
website/docs/restapi/error_code.md | 156 ++
website/docs/restapi/image/job_api_en.png | Bin 0 -> 21482 bytes
website/docs/restapi/image/metrics_store_en.png | Bin 0 -> 245660 bytes
website/docs/restapi/image/work_follow_en.png | Bin 0 -> 50463 bytes
website/docs/restapi/job_api.md | 168 ++
website/docs/restapi/model_api/intro.md | 20 +
website/docs/restapi/model_api/model_build_api.md | 80 +
.../model_api/model_import_and_export_api.md | 310 ++++
.../docs/restapi/model_api/model_management_api.md | 1835 ++++++++++++++++++++
.../model_api/model_multilevel_partitioning_api.md | 366 ++++
website/docs/restapi/project_api.md | 700 ++++++++
website/docs/restapi/query_api.md | 543 ++++++
website/docs/restapi/segment_managment_api.md | 393 +++++
website/docs/restapi/snapshot_management_api.md | 304 ++++
website/docs/restapi/streaming_job_api.md | 282 +++
website/sidebars.js | 98 +-
26 files changed, 7961 insertions(+), 1 deletion(-)
diff --git a/website/docs/development/intro.md b/website/docs/development/intro.md
new file mode 100644
index 0000000000..3d61612265
--- /dev/null
+++ b/website/docs/development/intro.md
@@ -0,0 +1,25 @@
+---
+title: Deployment Mode
+language: en
+sidebar_label: Deployment Mode
+pagination_label: Deployment Mode
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - deployment mode
+draft: false
+last_update:
+ date: 12/08/2022
+---
+# Tutorial Intro
+
+Let's discover ** Kylin 5.0 in than 15 minutes **.
+
+## Getting Started
+
+```shell
+
+docker pull apachekylin/apache-kylin-standalone:5.0.0
+```
diff --git a/website/docs/restapi/acl_api/acl_api.md b/website/docs/restapi/acl_api/acl_api.md
new file mode 100644
index 0000000000..12e444baf4
--- /dev/null
+++ b/website/docs/restapi/acl_api/acl_api.md
@@ -0,0 +1,609 @@
+---
+title: Data ACL API
+language: en
+sidebar_label: Data ACL API
+pagination_label: Data ACL API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - data acl api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+### Get ACL
+
+- `GET http://localhost:port/kylin/api/acl/{type}/{name}?authorized_only=true&project=m`
+
+- URL Parameters
+
+ - `type` - `required` `string`, user type, optional values: user, group (case-insensitive)
+ - `name` - `required` `string`, user name or user group name.
+ - `project` - `required` `string`, project name.
+ - `authorized_only` - `optional` `boolean`, whether to return only authorized table rows and columns，default value is `false`。
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/acl/User/bb?authorized_only=true&project=m' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": [
+ {
+ "tables": [
+ {
+ "authorized": true,
+ "columns": [
+ {
+ "authorized": true,
+ "column_name": "C_ADDRESS",
+ "data_mask_type": "AS_NULL",
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": true,
+ "column_name": "C_CITY",
+ "data_mask_type": "DEFAULT",
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": false,
+ "column_name": "C_CUSTKEY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "integer"
+ },
+ {
+ "authorized": true,
+ "column_name": "C_MKTSEGMENT",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": true,
+ "column_name": "C_NAME",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": true,
+ "column_name": "C_NATION",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": true,
+ "column_name": "C_PHONE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": true,
+ "column_name": "C_REGION",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ }
+ ],
+ "row_filter": {
+ "type": "AND",
+ "filter_groups": []
+ },
+ "table_name":"ANALYSIS_PACKAGE_TABLE",
+ "authorized_column_num":10,
+ "total_column_num":10
+ },
+ {
+ "authorized": true,
+ "columns": [
+ {
+ "authorized": true,
+ "column_name": "LO_COMMITDATE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "date"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_CUSTKEY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "integer"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_DISCOUNT",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_EXTENDEDPRICE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_LINENUMBER",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_ORDERDATE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "date"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_ORDERKEY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_ORDERPRIOTITY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_ORDTOTALPRICE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_PARTKEY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "integer"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_QUANTITY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_REVENUE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_SHIPMODE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "varchar(4096)"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_SHIPPRIOTITY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "integer"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_SUPPKEY",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "integer"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_SUPPLYCOST",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_TAX",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ },
+ {
+ "authorized": true,
+ "column_name": "V_REVENUE",
+ "data_mask_type": null,
+ "dependent_columns": null,
+ "datatype": "bigint"
+ }
+ ],
+ "row_filter": {
+ "type": "AND",
+ "filter_groups": []
+ },
+ "table_name":"QUERY_FACT_TABLE",
+ "authorized_column_num":11,
+ "total_column_num":11
+ }
+ ],
+ "authorized_table_num": 2,
+ "total_table_num": 2,
+ "database_name": "SSB"
+ }
+ ],
+ "msg": ""
+ }
+ ```
+
+
+### Update Acl
+
+- `PUT http://localhost:port/kylin/api/acl/{type}/{name}?project=m`
+
+- URL Parameters
+
+ - `type` - `required` `string`, User type, optional values: user, group. case insensitive.
+ - `name` - `required` `string`, User name or user group name.
+ - `project` - `required` `string`, Project name.
+
+- HTTP Body
+
+ **Note:** You can grant or revoke tables, columns or rows's acls incrementally.
+
+ - `database_name` - `required` `string`, database name. case insensitive.
+ - `tables` - `required` `array[string]`, table information.
+
+ - `table_name` - `required` `string`, name of the table. case insensitive.
+ - `authorized` - `optional` `boolean`, whether to authorize the permissions of this table. `true` represents authorization and the default value is `false`.
+ - `columns` - `optional` `array[string]`, column level permissions to grant or revoke, if no change, set this value to null.
+
+ - `column_name` - `required` `string`, the name of the column. case insensitive.
+ - `authorized` - `optional` `boolean`, whether to authorize the permissions in this column. `true` indicates authorization and the default value is `false`.
+ - `data_mask_type`- `optional` `string`, sensitive data column masking type, optional values: DEFAULT, AS_NULL. Example
+ ```json
+ {
+ "authorized":true,
+ "columns":[
+ {
+ "authorized":true,
+ "column_name":"STORE_AND_FWD_FLAG",
+ "data_mask_type":null,
+ "dependent_columns":null
+ },
+ {
+ "authorized":true,
+ "column_name":"TOTAL_AMOUNT",
+ "data_mask_type":"DEFAULT",
+ "dependent_columns":null
+ },
+ {
+ "authorized":true,
+ "column_name":"TRIP_DISTANCE",
+ "data_mask_type":"AS_NULL",
+ "dependent_columns":null
+ }
+ ]
+ }
+ ```
+
+ DEFAULT: use column datatype's default value as mask value. INT's default mask value is 0, VARCHAR's default mask value is ****.
+ AS_NULL: use null as mask value
+
+ - `dependent_columns` - `optional` `array`. Column-level permission control of associated row values parameter. Example
+
+ ```json
+ {
+ "authorized":true,
+ "column_name":"PASSENGER_COUNT",
+ "data_mask_type":null,
+ "dependent_columns":[
+ {
+ "column_identity":"DEFAULT.GREEN_TRIP_DATA.DO_LOCATION_ID",
+ "values":[
+ "1",
+ "2"
+ ]
+ }
+ ]
+ }
+ ```
+
+ column PASSENGER_COUNT will depend on DEFAULT.GREEN_TRIP_DATA.DO_LOCATION_ID's values.
+ - `row_filter` - `optional` set row level access control. Set to null if you do not want to modify it.
+
+ - `type` - `optional` `string` set logical operator type to `AND` or `OR` between filters/filter groups. Default value is `AND`.
+ - `filter_groups` - `optional` `array[string]` set filters or filter groups. Default value is an empty list.
+
+ - `type` - `optional` `string` set logical operator type to `AND` or `OR` between filters within one filter group. Default value is `AND`.
+ - `is_group` - `required` `boolean` set type, indicating whether it is a filter or filter group.
+ - `filters` - `optional` `array[string]` set filters. Default value is an empty list.
+
+ - `column_name` - `required` `string` set the name of the column where the row level access control applies on.
+ - `in_items` - `optional` `array[string]` the values of filter condition. Default is an empty list.
+ - `like_items` - `optional` `array[string]` the patterns of filter condition. Default is an empty list.
+
+ For example, the request below sets one filter group with two filters, and another standalone filter.
+ The logical operator between the filter group and the standalone filter is `OR`. The logical operator between filters in the filter group is `AND`.
+ ```json
+ {
+ "row_filter": {
+ "type": "OR",
+ "filter_groups": [{
+ "type": "AND",
+ "filters": [
+ {
+ "column_name": "LSTG_FORMAT_NAME",
+ "in_items": ["ABIN", "Others"],
+ "like_items": ["B%"]
+ },
+ {
+ "column_name": "TRANS_ID",
+ "in_items": ["0", "1"],
+ "like_items": []
+ }
+ ],
+ "is_group": true
+ }, {
+ "type": "AND",
+ "filters": [
+ {
+ "column_name": "TRANS_ID",
+ "in_items": ["0"],
+ "like_items": []
+ }
+ ],
+ "is_group": false
+ }]
+ }
+ }
+ ```
+ After authorized successfully, you can only see the results of
+ ```mysql-sql
+ (
+ (LSTG_FORMAT_NAME in ('ABIN', 'Others') OR LSTG_FORMAT_NAME like 'B%')
+ AND
+ (TRANS_ID in (0, 1))
+ ) OR (
+ (TRANS_ID in (0))
+ )
+ ```
+ If you need to set row level access control, all three `column_name`，`in_items` and `like_items` need to be filled in. This field uses a full update.
+ Thus in order to disable the row level access control, you need to send an empty `filter_groups` as shown below:
+ ```json
+ {
+ "row_filter":
+ {
+ "type": "AND",
+ "filter_groups": []
+ }
+ }
+ ```
+
+ **Note:** If you are still using the old row level access control API before Kylin version 4.3.3, you may encounter an update failure. It is recommended to use the latest API to manage row level access control.
+
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ - Grant / Revoke access on the specific table. If revoke table access , all columns' / rows' access in the specific table will be revoked.
+
+ ```sh
+ curl --location --request PUT 'http://localhost:8080/kylin/api/acl/User/user_1?project=project_1' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: cn' \
+ -H 'Authorization: Basic QURNSU46a3lsaW5AMjAyMA==' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ --data-raw '[
+ {
+ "tables": [
+ {
+ "authorized": false,
+ "table_name": "CUSTOMER"
+ }
+ ],
+ "database_name": "SSB"
+ }
+ ]'
+ ```
+
+ - Grant / Revoke access on the specific columns. Only update the specific column in the specific table, not update others.
+
+ ```sh
+ curl --location --request PUT 'http://localhost:8080/kylin/api/acl/User/user_1?project=project_1' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: cn' \
+ -H 'Authorization: Basic QURNSU46a3lsaW5AMjAyMA==' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ --data-raw '[
+ {
+ "tables": [
+ {
+ "columns": [
+ {
+ "authorized": false,
+ "column_name": "C_CITY"
+ }
+ ],
+ "authorized": true,
+ "table_name": "CUSTOMER"
+ },
+ {
+ "columns": [
+ {
+ "authorized": true,
+ "column_name": "LO_REVENUE",
+ "data_mask_type": "AS_NULL"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_TAX",
+ "data_mask_type": "DEFAULT"
+ },
+ {
+ "authorized": true,
+ "column_name": "LO_QUANTITY",
+ "data_mask_type": null,
+ "dependent_columns": [
+ {
+ "column_identity": "SSB.CUSTOMER.C_CUSTKEY",
+ "values": [
+ "1",
+ "2"
+ ]
+ }
+ ]
+ }
+ ],
+ "authorized": true,
+ "table_name": "P_LINEORDER"
+ }
+ ],
+ "database_name": "SSB"
+ }
+ ]'
+ ```
+
+ - Grant / Revoke access on the specific rows. Fully update the row access of the specified table, not update others.
+
+ ```sh
+ curl --location --request PUT 'http://localhost:8080/kylin/api/acl/User/user_1?project=project_1' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: cn' \
+ -H 'Authorization: Basic QURNSU46a3lsaW5AMjAyMA==' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ --data-raw '[
+ {
+ "tables": [
+ {
+ "authorized": true,
+ "row_filter":
+ {
+ "type": "AND",
+ "filter_groups": [
+ {
+ "type": "AND",
+ "filters": [
+ {
+ "column_name": "C_NATION",
+ "in_items": ["CHINA", "UNITED KINGDOM"],
+ "like_items": []
+ }
+ ],
+ "is_group": false
+ },
+ {
+ "type": "AND",
+ "filters": [
+ {
+ "column_name": "C_CUSTKEY",
+ "in_items": ["15", "16", "19"],
+ "like_items": []
+ }
+ ],
+ "is_group": false
+ }
+ ]
+ },
+ "table_name": "CUSTOMER"
+ },
+ {
+ "row_filter":
+ {
+ "type": "AND",
+ "filter_groups": [
+ {
+ "type": "AND",
+ "filters": [
+ {
+ "column_name": "LO_CUSTKEY",
+ "in_items": ["15", "16", "20"],
+ "like_items": []
+ }
+ ],
+ "is_group": false
+ },
+ {
+ "type": "AND",
+ "filters": [
+ {
+ "column_name": "LO_QUANTITY",
+ "in_items": ["31", "33", "23"],
+ "like_items": []
+ }
+ ],
+ "is_group": false
+ },
+ {
+ "type": "AND",
+ "filters": [
+ {
+ "column_name": "LO_ORDERDATE",
+ "in_items": ["1995-02-01", "1996-01-26", "1992-11-21"],
+ "like_items": []
+ }
+ ],
+ "is_group": false
+ }
+ ]
+ },
+ "authorized": true,
+ "table_name": "P_LINEORDER"
+ }
+ ],
+ "database_name": "SSB"
+ }
+ ]'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
diff --git a/website/docs/restapi/acl_api/intro.md b/website/docs/restapi/acl_api/intro.md
new file mode 100644
index 0000000000..2def65af7e
--- /dev/null
+++ b/website/docs/restapi/acl_api/intro.md
@@ -0,0 +1,18 @@
+---
+title: ACL Management API
+language: en
+sidebar_label: ACL Management API
+pagination_label: ACL Management API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - acl management api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+Kylin provides REST APIs on Access Control List Management to help users strictly manage the ACL on projects, tables, users, user group and so on.
+
diff --git a/website/docs/restapi/acl_api/project_acl_api.md b/website/docs/restapi/acl_api/project_acl_api.md
new file mode 100644
index 0000000000..14aa630b52
--- /dev/null
+++ b/website/docs/restapi/acl_api/project_acl_api.md
@@ -0,0 +1,217 @@
+---
+title: Project ACL API
+language: en
+sidebar_label: Project ACL API
+pagination_label: Project ACL API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - project acl api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](../authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+### Get Project ACL {#get-project-acl}
+
+- `GET http://host:port/kylin/api/access/project`
+
+- Introduced in: 4.1.4
+
+- Request Parameters
+
+ - `project` - `required` `string`, project name.
+ - `name` - `optional` `string`, user or group name.
+ - `page_offset` - `optional` `int`, offset of returned result, 0 by default.
+ - `page_size` - `optional` `int`, quantity of returned result per page, 10 by default.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/access/project?project=Kylin' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ {
+ "type": "user",
+ "name": "ADMIN",
+ "permission": "ADMIN"
+ },
+ {
+ "type": "group",
+ "name": "TEST_GROUP",
+ "permission": "QUERY"
+ }
+ ...
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 4
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Grant Project ACL {#grant-project-acl}
+
+- `POST http://host:port/kylin/api/access/project`
+
+- Introduced in: 4.1.4
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `type` - `required` `string`, represents users or user groups, optional values are `user` or `group`.
+ - `permission` - `required` `string`, represents users or user groups permission, optional values are `QUERY`, `OPERATION`, `MANAGEMENT` and `ADMIN`.
+ - `names` - `required` `array[string]`, name of user or user group.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/access/project' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project": "Kylin",
+ "type": "user",
+ "permission": "QUERY",
+ "names":["test"]
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
+### Update Project ACL (overwrite) {#update-project-acl}
+
+- `PUT http://host:port/kylin/api/access/project`
+
+- Introduced in: 4.1.4
+
+- HTTP Body: JSON Object
+ - `project` - `required` `string`, project name.
+ - `type` - `required` `string`, represents users or user groups, optional values are `user` or `group`.
+ - `permission` - `required` `string`, represents users or user groups permission, optional values are `QUERY`, `OPERATION`, `MANAGEMENT` and `ADMIN`.
+ - `name` - `required` `string`, name of user or user group.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/access/project' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project": "Kylin",
+ "type": "user",
+ "permission": "ADMIN",
+ "name": "test"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
+### Revoke Project ACL {#revoke-project-acl}
+
+- `DELETE http://host:port/kylin/api/access/project`
+
+- Introduced in: 4.1.4
+
+- Request Parameters
+
+ - `project` - `required` `string`, project name.
+ - `type` - `required` `string`, Represents users or user groups, optional are `user` or `group`.
+ - `name` - `required` `string`, name of user or user group.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://localhost:7070/kylin/api/access/project?project=Kylin&type=user&name=test' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
diff --git a/website/docs/restapi/acl_api/user_api.md b/website/docs/restapi/acl_api/user_api.md
new file mode 100644
index 0000000000..3fb7fe77e9
--- /dev/null
+++ b/website/docs/restapi/acl_api/user_api.md
@@ -0,0 +1,343 @@
+---
+title: User Management API
+language: en
+sidebar_label: User Management API
+pagination_label: User Management API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - user management api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](../authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+
+### Get User List {#get-user-list}
+
+- `GET http://host:port/kylin/api/user`
+
+- Introduced in: 4.1.4
+
+- Request Parameters
+
+ - `name` - `optional` `string`, username.
+ - `is_case_sensitive` - `optional` `boolean`, whether case sensitive on user name and `false` is by default.
+ - `page_offset` - `optional` `int`, offset of returned result, 0 by default.
+ - `page_size` - `optional` `int`, quantity of returned result per page, 10 by default.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/user' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ {
+ "username": "test",
+ "authorities": [
+ {
+ "authority": "ALL_USERS"
+ }
+ ...
+ ],
+ "disabled": false,
+ "default_password": false,
+ "locked": false,
+ "uuid": "af11440a-8e9d-4801-8508-5d0ce0e04a2f",
+ "last_modified": 1592296833935,
+ "create_time": 1592296833844,
+ "locked_time": 0,
+ "wrong_time": 0,
+ "first_login_failed_time": 0
+ }
+ ...
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 3
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Create User {#create-user}
+
+- `POST http://host:port/kylin/api/user`
+
+- Introduced in: 4.1.4
+
+- HTTP Body: JSON Object
+
+ - `username` - `required` `string`, username.
+ - `password` - `required` `string`, password.
+ - `disabled` - `required` `boolean`, enable the user or not, `true` means the user is disabled and `false` means the user is enabled.
+ - `authorities` - `required` `array[string]`, the user belongs which user group.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/user' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "username": "test",
+ "password": "Password@123",
+ "disabled": "false",
+ "authorities":["ALL_USERS"]
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
+### Modify the group where user belongs {#modify-user}
+
+**Note:** Modifying a user will overwrite the user group values.
+
+- `PUT http://host:port/kylin/api/user`
+
+- Introduced in: 4.1.4
+
+- HTTP Body: JSON Object
+
+ - `username` - `required` `string`, username.
+ - `authorities` - `required` `array[string]`, the user belongs which user group.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/user' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "username": "createuser",
+ "authorities":["ALL_USERS"]
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
+### Change Password {#change-password}
+
+- `PUT http://host:port/kylin/api/user/password`
+
+- Introduced in: 4.1.4
+
+- HTTP Body: JSON Object
+
+ - `username` - `required` `string`, username.
+ - `password` - `required` `string`, original password.
+ - `new_password` - `required` `string`, new password.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/user' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "username": "test",
+ "password": "Password@123",
+ "new_password": "Password@1234"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
+### Delete User{#delete-user}
+
+### {#delete-user}
+
+- `DELETE http://host:port/kylin/api/user`
+
+- Introduced in: 4.2.1
+
+- HTTP Body: JSON Object
+
+ - `username` - `required` `string`, username.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://localhost:7070/kylin/api/user' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ -d '{
+ "username": "testuser"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
+### Get ACL of a User or Group {#get-acl-for-user-or-group}
+
+- `GET http://host:port/kylin/api/access/acls`
+
+- Introduced in: 4.1.4
+
+- Request Parameters
+
+ - `type` - `required` `string`, represents users or user groups, optional values are `user` or `group`.
+ - `name` - `required` `string`, user or group name.
+ - `project` - `optional` `string`, project name, fill in the value to get all the permissions of a user or user group in the specified project.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/access/acls?type=user&name=ADMIN' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": [
+ {
+ "project_name": "test",
+ "project_permission": "ADMIN",
+ "acl_info": [
+ {
+ "tables": [
+ {
+ "authorized": true,
+ "columns": [
+ {
+ "authorized": true,
+ "column_name": "C_ADDRESS"
+ }
+ ...
+ ],
+ "rows": [],
+ "table_name": "CUSTOMER",
+ "authorized_column_num": 8,
+ "total_column_num": 8
+ }
+ ...
+ ],
+ "authorized_table_num": 6,
+ "total_table_num": 6,
+ "database_name": "SSB"
+ }
+ ...
+ ]
+ }
+ ...
+ ],
+ "msg": ""
+ }
+ ```
diff --git a/website/docs/restapi/acl_api/user_group_api.md b/website/docs/restapi/acl_api/user_group_api.md
new file mode 100644
index 0000000000..97f01a0d03
--- /dev/null
+++ b/website/docs/restapi/acl_api/user_group_api.md
@@ -0,0 +1,276 @@
+---
+title: User Group Management API
+language: en
+sidebar_label: User Group Management API
+pagination_label: User Group Management API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - user group management api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](../authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains `&` or other special chars.
+
+
+
+### Get User Group List {#get-user-group-list}
+
+- `GET http://host:port/kylin/api/user_group/groups`
+
+- Introduced in: 4.1.4
+
+- Request Parameters
+
+ - `group_name` - `optional` `string`, group name.
+ - `is_case_sensitive` - `optional` `boolean`, whether case-sensitive on user group name. The default value is ` false`.
+ - `page_offset` - `optional` `int`, offset of returned result, 0 by default.
+ - `page_size` - `optional` `int`, quantity of returned result per page, 10 by default.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/user_group/groups`' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ "ALL_USERS",
+ "ROLE_ADMIN",
+ "ROLE_ANALYST",
+ "ROLE_MODELER"
+ ...
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 7
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Get User List of Specified User Group {#get-user-list-of-specified-user-group}
+
+- `GET http://host:port/kylin/api/user_group/group_members/{group_name}`
+
+- Introduced in: 4.1.4
+
+- URL Parameters
+
+ - `group_name` - `required` `string`, group name.
+
+- Request Parameters
+
+ - `username` - `optional` `string`, username.
+ - `page_offset` - `optional` `int`, offset of returned result, 0 by default.
+ - `page_size` - `optional` `int`, quantity of returned result per page, 10 by default.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/user_group/group_members/test' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ {
+ "username": "ADMIN",
+ "authorities": [
+ {
+ "authority": "ROLE_ADMIN"
+ },
+ {
+ "authority": "ALL_USERS"
+ }
+ ],
+ "disabled": false,
+ "default_password": false,
+ "locked": false,
+ "uuid": "aaf02c5d-1605-42fa-abf9-9b0bb5715a6a",
+ "last_modified": 1592555313558,
+ "create_time": 1586744927779,
+ "locked_time": 0,
+ "wrong_time": 0,
+ "first_login_failed_time": 0
+ }
+ ...
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 10
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Add User Group {#add-user-group}
+
+- `POST http://host:port/kylin/api/user_group`
+
+- Introduced in: 4.2.1
+
+- HTTP Body: JSON Object
+
+ - `group_name` - `required` `string`, group name.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/user_group' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "group_name": "test_group"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": "add user group"
+ }
+ ```
+
+
+
+### Update User Group {#update-user-group}
+
+**Note:** Updating group will overwrite the original user list.
+
+- `PUT http://host:port/kylin/api/user_group/users`
+
+- Introduced in: 4.1.4
+
+- HTTP Body: JSON Object
+
+ - `group_name` - `required` `string`, group name.
+ - `users` - `required` `array[string]`, list of users in group.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/user_group/users' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "group_name": "test",
+ "users":["ANALYST", "MODELER"]
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": "modify users in user group"
+ }
+ ```
+
+
+### Delete User Group {#delete-user-group}
+
+- `DELETE http://host:port/kylin/api/user_group`
+
+- Introduced in: 4.2.1
+
+- HTTP Body: JSON Object
+
+ - `group_name` - `required` `string`, group name.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://localhost:7070/kylin/api/user_group' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ -d '{
+ "group_name": "test_group"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": "del user group"
+ }
+ ```
+
diff --git a/website/docs/restapi/async_query_api.md b/website/docs/restapi/async_query_api.md
new file mode 100644
index 0000000000..818f76f3e2
--- /dev/null
+++ b/website/docs/restapi/async_query_api.md
@@ -0,0 +1,414 @@
+---
+title: Async Query API
+language: en
+sidebar_label: Async Query API
+pagination_label: Async Query API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - async query api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+> 3. In this chapter， all get requests and delete by ID request are forward compatible, that is, if the project parameter is not transferred to the URL, the request can also be successful if the project parameter is transferred to the request body.
+> 4. Due to the requirements of the parquet format, the naming of the column can not contain characters such as ",; {} () \\ n\\ t =", so you need to display the definition alias in the SQL query, and the alias does not contain these special characters.
+> 5. The earlier versions of hive do not support reading the parquet date type field, so you can replace the original table data type with timestamp, or upgrade hive to the versions after 1.2 .
+
+
+
+### Submit Async Query {#Submit-Async-Query}
+
+- `POST http://host:port/kylin/api/async_query`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `sql` - `required` `string`, SQL statement
+ - `separator` - `optional` `string`, separator of the exported result, which is `,` by default. Other separators such as `#`, `$`, `@`, `|` are also supported.
+ - `offset` - `optional` `int`, offset of query result
+ - `limit` - `optional` `int `, limit on the quantity of query result
+ - `project` - `required` `string`, project name
+ - `format` - `optional` `string`，file format, the default value is "csv", other optional values are "json", "xlsx", "parquet"
+ > Note: When the file format is "xlsx" or "json", specifying the separator separator is not supported. When the file format is "parquet", it is currently not supported to download the result file through the [Download Query Result](#Download-Query-Result) API, and can only be obtained directly from HDFS.
+ - `encode` - `optional` `string`，file encoding, the default value is "utf-8", other optional values are "gbk"
+ - `file_name` - `optional` `string`，file name, Chinese is not supported temporarily, the default value is "result"
+ - `spark_queue` - `optional` `string`, the cluster queue specified, the default value is "default". It will take effect after enabling `kylin.query.unique-async-query-yarn-queue-enabled`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/async_query' \
+ -H 'Accept: application/vnd.apache.kylin-v4+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{ "sql":"select * from KYLIN_SALES limit 100", "project":"learn_kylin" }'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "query_id": "eb3e837f-d826-4670-aac7-2b92fcd0c8fe",
+ "status": "RUNNING",
+ "info": "query still running"
+ },
+ "msg": ""
+ }
+ ```
+
+- Response Information
+
+ - `query_id` - Query ID of the Async Query
+ - `status` - Status, ie., "SUCCESSFUL", "FAILED", "RUNNING"
+ - `info` - Detailed information about the status
+
+
+
+### Request Query Status {#Request-Query-Status}
+
+- `GET http://host:port/kylin/api/async_query/{query_id}/status?project=learn_kylin`
+
+- URL Parameters
+
+ - `query_id` - `required` `string`, Query ID of the Async Query
+ - `project` `required` `string`, project name
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/async_query/{query_id}/status?project=learn_kylin' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "query_id": "eb3e837f-d826-4670-aac7-2b92fcd0c8fe",
+ "status": "SUCCESSFUL",
+ "info": "await fetching results"
+ },
+ "msg": ""
+ }
+ ```
+
+- Response Information
+ - `query_id` - Query ID of the Async Query
+ - `status` - Status, ie., "SUCCESSFUL" , "RUNNING", "FAILED" and "MISSING"
+ - `info` - Detailed information about the status
+
+
+
+### Request Query Metadata Info {#Request-Query-Metadata-Info}
+
+- `GET http://host:port/kylin/api/async_query/{query_id}/metadata?project=learn_kylin`
+
+- URL Parameters
+ - `query_id` - `required` `string`, Query ID of the Async Query
+ - `project` `required` `string`, project name
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/async_query/{query_id}/metadata?project=learn_kylin' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": [
+ [
+ "TRANS_ID",
+ "PART_DT"
+ ],
+ [
+ "BIGINT",
+ "DATE"
+ ]
+ ],
+ "msg": ""
+ }
+ ```
+
+- Response Information
+
+ - `data` - data includes two list, the first list is the column name, and the second list is the corresponding data type of the column
+
+
+
+### Request Query Result File Status {#Request-Query-Result-File-Status}
+
+- `GET http://host:port/kylin/api/async_query/{query_id}/file_status?project=learn_kylin`
+
+
+- URL Parameters
+ - `query_id` - `required` `string`, Query ID of the Async Query
+ - `project` `required` `string`, project name
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/async_query/{query_id}/file_status?project=learn_kylin`' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": 7611,
+ "msg": ""
+ }
+ ```
+
+- Response Information
+
+ - `data` - total size of the result
+
+
+
+### Download Query Result {#Download-Query-Result}
+
+> Note: Please make sure the query status is "SUCCESSFUL" before calling this API.
+
+- `GET http://host:port/kylin/api/async_query/{query_id}/result_download?include_header=true&project=learn_kylin`
+
+
+- URL Parameters
+ - `query_id` - `required` `string`, Query ID of the Async Query
+ - `include_header` - `optional` `boolean`, result include header, default is false
+ - `project` `required` `string`, project name
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/async_query/{query_id}/result_download?include_header=true&project=learn_kylin' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -o result.csv
+ ```
+
+- Response Example
+ - returns a document named `result.csv`.
+
+
+
+
+### Request Query HDFS Path {#Request-Query-HDFS-Path}
+
+- `GET http://host:port/kylin/api/async_query/{query_id}/result_path?project=learn_kylin`
+
+
+- URL Parameters
+ - `query_id` - `required` `string`, Query ID of the Async Query
+ - `project` `required` `string`, project name
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/async_query/{query_id}/result_path?project=learn_kylin' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "hdfs://host:8020/{kylin_working_dir}/{kylin_metadata_url}/{project}/async_query_result/{query_id}",
+ "msg": ""
+ }
+ ```
+
+- Response Information
+
+ - `data` - the HDFS Path in which stores the result file
+
+
+
+### Delete All Query Result Files {#Delete-All-Query-Result-Files}
+
+- `DELETE http://host:port/kylin/api/async_query`
+
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://host:port/kylin/api/async_query' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": true,
+ "msg": ""
+ }
+ ```
+
+
+
+### Delete old query result files based on time {#Delete-old-query-result-files-based-on-time}
+
+> Tip: This interface may delete queries that have not yet obtained results.
+
+- `DELETE http://host:port/kylin/api/async_query?project={project}&older_than={time}`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- URL Parameters
+
+ - `project`-`required` `string`, the query result file of which project needs to be deleted
+ - `Older_than`-`required` `string`, the earliest retention time, the asynchronous query result file whose last_modify is earlier than this time will be deleted, the time format is `yyyy-MM-dd HH:mm:ss`, no need to bring it quotation marks. Note: When using Curl request, it needs to perform url escaping by replacing spaces with `%20`.
+
+- Curl request example
+
+ ```sh
+ curl -X DELETE \
+ 'http://host:port/kylin/api/async_query?project=learn_kylin&older_than=2021-04-26%2010:00:00' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+
+- Response example
+
+ ```json
+ {
+ "code": "000",
+ "data": true,
+ "msg": ""
+ }
+ ```
+
+- Response information
+
+ - `data`-Returns true if all old query result files are successfully deleted, otherwise false
+
+
+
+### Delete query result files according to query_id {#Delete-query-result-file-based-on-query_id}
+
+- `DELETE http://host:port/kylin/api/async_query/{query_id}?project=learn_kylin`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- URL Parameters
+ - `query_id`-`required ` `string`, Query ID for asynchronous query
+ - `project`-`required` `string`, project name
+
+- Curl request example
+
+ ```sh
+ curl -X DELETE \
+ 'http://host:port/kylin/api/async_query/{query_id}?project=learn_kylin' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response example
+
+ ```json
+ {
+ "code": "000",
+ "data": true,
+ "msg": ""
+ }
+ ```
+
+- Response information
+
+ - `data`-Return true if the query result file corresponding to `query_id` is successfully deleted, otherwise false
+
+### Known Limitations
+- When the column name contains a comma `,`, if the separator `separator` is specified, the comma `,` in the column name of the header of the download result table will be replaced by the separator.
+- When column names contain separator, the file format may be parsed incorrectly.
diff --git a/website/docs/restapi/authentication.md b/website/docs/restapi/authentication.md
new file mode 100644
index 0000000000..5562d5c780
--- /dev/null
+++ b/website/docs/restapi/authentication.md
@@ -0,0 +1,111 @@
+---
+title: Access and Authentication API
+language: en
+sidebar_label: Access and Authentication API
+pagination_label: Access and Authentication API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - access
+ - authentication
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+### Access Information
+The access prefix of all Kylin APIs is `/kylin/api`. This prefix is required regardless of which module API is accessed. For example, accessing model uses API of `/kylin/api/models`, and the correspondingly complete path is `http://host:port/kylin/api/models`.
+
+
+### Authentication
+All APIs in Kylin are based on [Basic Authentication](http://en.wikipedia.org/wiki/Basic_access_authentication) authentication mechanism. Basic Authentication is a simple access control mechanism, which encodes account and password information based on Base64. Adding these information as request headers to HTTP request commands, the back-end will decode the account and password information from the request header for authentication. Take the account and password `ADMIN:KYLIN` as an examp [...]
+
+
+
+### Authentication Essentials
+* Add `Authorization` to HTTP header
+ * Or users could get authorized through `POST http://host:port/kylin/api/user/authentication`. Once the authentication passes, the authentication information would be stored in cookie files for the following visit.
+
+
+- HTTP Header
+ - `Authorization:Basic QURNSU46S1lMSU4=`
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/user/authentication' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "username":"ADMIN",
+ "authorities":[{"authority": "ROLE_ADMIN"}, {"authority": "ALL_USERS"}],
+ "disabled":false,
+ "locked":false,
+ "uuid":"0205dac6-215a-4454-84ae-3dcc85b9675c",
+ "last_modified":1574756819619,
+ "create_time":1563346648008,
+ "version":"4.0.0.0",
+ "mvcc":24,
+ "locked_time":0,
+ "wrong_time":2,
+ "first_login_failed_time":1574756817981
+ },
+ "msg":""
+ }
+ ```
+
+
+
+### JavaScript Authentication Request Example
+
+> Note: You can download "jquery.base64.js" at [https://github.com/yckart/jquery.base64.js](https://github.com/yckart/jquery.base64.js)
+
+```javascript
+var authorizationCode = $.base64('encode', 'NT_USERNAME' + ":" + 'NT_PASSWORD');
+
+$.ajaxSetup({
+ headers: {
+ 'Authorization': "Basic " + authorizationCode,
+ 'Content-Type': 'application/json;charset=utf-8',
+ 'Accept': 'application/vnd.apache.kylin-v4-public+json'
+ }
+});
+```
+
+```javascript
+$.ajaxSetup({
+ headers: {
+ 'Authorization': "Basic eWFu**********X***ZA==",
+ 'Content-Type': 'application/json;charset=utf-8',
+ 'Accept': 'application/vnd.apache.kylin-v4-public+json'
+ } // use your own authorization code here
+ });
+ var request = $.ajax({
+ url: "http://host:port/kylin/api/query",
+ type: "POST",
+ data: '{"sql":"select count(*) from SUMMARY;","offset":0,"limit":50000,"acceptPartial":true,"project":"test"}',
+ dataType: "json"
+ });
+ request.done(function( msg ) {
+ alert(msg);
+ });
+ request.fail(function( jqXHR, textStatus ) {
+ alert( "Request failed: " + textStatus );
+ });
+```
+
diff --git a/website/docs/restapi/callback_api.md b/website/docs/restapi/callback_api.md
new file mode 100644
index 0000000000..c8e5be8fd1
--- /dev/null
+++ b/website/docs/restapi/callback_api.md
@@ -0,0 +1,153 @@
+---
+title: Use callback API to monitor job status
+language: en
+sidebar_label: Use callback API to monitor job status
+pagination_label: Use callback API to monitor job status
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - callback api
+ - monitor job
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+The success returned by calling the required job API only indicates the job is successfully started. Kylin supports callback API to confirm the job execution state, it will return success if the job is successfully completed; it will return error if the job fails and also the error code to help troubleshooting.
+
+### How to use
+
+Deploy an online service that accepts POST requests and add the configuration parameter `kylin.job.finished-notifier-url` to kylin.properties with the service URL as its parameter value.
+
+### POST message example
+
+Suppose the value of parameter `kylin.job.finished-notifier-url` is set to `http://127.0.0.1:7777`. When Kylin finishes the job, it will send the following message to `http://127.0.0.1:7777`.
+
+**Header**
+
+```sh
+Content-Type: application/json
+Content-Length: 262
+Host: 127.0.0.1:7777
+Connection: Keep-Alive
+User-Agent: Apache-HttpClient/4.5.3 (Java/1.8.0_171)
+Accept-Encoding: gzip,deflate
+```
+
+**Body**
+
+```sh
+{
+"job_id": "6044ac09-27bc-4968-a6b0-881c4c0abf89",
+"project": "test",
+"model_id": "91402da4-b991-4cd8-b164-7f62b9e91b69",
+"segment_ids": [
+ "9e5a89b9-00f1-4296-adc4-139113030e83",
+ "f26dcbc9-b786-4112-a4f6-94280bf10265",
+ "f3dc9864-0e1a-417b-a46d-3255e8cb6473"
+],
+"index_ids": [
+ 20001,
+ 10001
+],
+"duration": 27699,
+"job_state": "ERROR",
+"job_type": "INDEX_BUILD",
+"segment_time_range": [
+ {
+ "segment_id": "9e5a89b9-00f1-4296-adc4-139113030e83",
+ "data_range_start": 1338480000000,
+ "data_range_end": 1354291200000
+ },
+ {
+ "segment_id": "f26dcbc9-b786-4112-a4f6-94280bf10265",
+ "data_range_start": 1325347200000,
+ "data_range_end": 1338480000000
+ },
+ {
+ "segment_id": "f3dc9864-0e1a-417b-a46d-3255e8cb6473",
+ "data_range_start": 1354291200000,
+ "data_range_end": 1370016000000
+ }
+],
+"segment_partition_info": [{
+ "segment_id": "9e5a89b9-00f1-4296-adc4-139113030e83",
+ "partition_info": [{
+ "id": 0,
+ "values": ["674"],
+ "status": "ONLINE",
+ "last_modified_time": 1623390508821,
+ "source_count": 34,
+ "bytes_size": 1772
+ }, {
+ "id": 1,
+ "values": ["22"],
+ "status": "ONLINE",
+ "last_modified_time": 1623390508821,
+ "source_count": 30,
+ "bytes_size": 1800
+ }]
+ }],
+ "snapshot_job_info":null,
+ "err_code":"KE-060100201",
+ "msg":"KE-060100201: An Exception occurred outside Kylin.",
+ "suggestion":"Please check whether the external environment(other systems, components, etc.) is normal.",
+ "start_time":1656604800000,
+ "end_time":1656608400000,
+ "tag":null,
+ "code":"999",
+ "stacktrace":"KE-060100201: An Exception occurred outside Kylin. org.apache...."
+}
+```
+
+### Callback API parameters
+
+- HTTP method: POST
+- Content-Type: application/json
+- URL Parameters
+ - `project` - `string`, project name
+ - `job_id - String`, job ID
+ - `model_id` - `string`, model UUID
+ - `segment_ids` - `list<string>`, segment UUID
+ - `index_ids` - `list<string>`, index UUID
+ - `duration` - `long`, build duration in milliseconds
+ - `job_state` - `string`, job state when the job ends, value: `SUCCEED` if the job succeeds, `DISCARDED` if the job is discarded, `ERROR` if the job fails, or `PAUSED` if the job is paused
+ - `job_type` - `string`，job type，including：`INDEX_BUILD`、`INDEX_REFRESH`、`INDEX_MERGE`、`INC_BUILD`、`SUB_PARTITION_BUILD`、`SUB_PARTITION_REFRESH`、`SNAPSHOT_BUILD`、`SNAPSHOT_REFRESH`
+ - `segment_time_range` - `list`，segment time range
+ - `segment_id` - `string`，segment UUID
+ - `data_range_start` - `long`, timestamp in milliseconds, start time of segment build
+ - `date_range_end` - `long`, timestamp in milliseconds, end time of segment build
+ - `segment_partition_info` - `list`, information about segment under partition including:
+ - `segment_id`: ` string`, segment UUID
+ - `partition_info`: `list`, partition information
+ - `id`: `long`, partition ID
+ - `values`: `list<long>`, partition values
+ - `status`: `string`, partition status, value:`ONLINE` if the partition is online,`LOADING` if the partition is being built, or `REFRESHING` if the partition is being refreshed
+ - `last_modified_time`: `long`, timestamp in milliseconds, last modified time
+ - `source_count`: `long`, number of rows
+ - `bytes_size`: `long`, storage capacity
+ - `snapshot_job_info` - `object`，information about snapshot job, including:
+ - `table`：`string`，table name of the snapshot
+ - `database`：`string`，database name of the snapshot
+ - `total_rows`：`long`，total rows of the snapshot
+ - `storage`： `long`，disk space of the snapshot
+ - `last_modified_time`：timestamp, last time the snapshot was built
+ - `status`：status of the snapshot, value: `ONLINE` and `OFFLINE`
+ - `select_partition_col`：partition column of the snapshot, it takes effect only for incremental snapshots and `null` for full snapshots
+ - `msg` - `string` , job error message
+ - `error_code` - `string` , error code in Kylin
+ - `suggestion` - `string`, suggestion on how to solve the issue
+ - `start_time` - long, timestamp in milliseconds, the start time of the task
+ - `end_time` - long, timestamp in milliseconds, the end time of the task
+ - `tag` - string，customized tag, used for system integration
+ - `code` - string，status code，value: `null` if job succeeds, `401` if unauthorized, `999` other unrecognized status
+ - `stacktrace` - string，stack information of exceptions
+
+### Known limitations
+
+- For a job in PAUSED or ERROR state, if the job is canceled directly, the job state will be changed to DISCARDED, but the job state change will not be reported via the callback API.
+
+- If a job in PAUSED or ERROR state is rerun and then canceled when running, the job state change will be reported via the callback API.
+
diff --git a/website/docs/restapi/data_source_api.md b/website/docs/restapi/data_source_api.md
new file mode 100644
index 0000000000..e53ac925bc
--- /dev/null
+++ b/website/docs/restapi/data_source_api.md
@@ -0,0 +1,541 @@
+---
+title: Data Source API
+language: en
+sidebar_label: Data Source API
+pagination_label: Data Source API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - data source api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminder:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+>
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+### Load Hive Table {#Load-Hive-Table}
+
+> Call this API to load Hive table metadata to Kylin. By default, when a new Hive table is added, the table metadata will not be loaded to Kylin.
+
+- `POST http://host:port/kylin/api/tables`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `project` - `required` `string`, project name
+
+ - `need_sampling` - `required` `boolean`, whether to enable table sampling
+
+ - `sampling_rows` - `optional` `integer`, indicates the max number of sampling rows and the range is [10,000 - 20,000,000] .
+
+ > Note: if you enable need_sampling, this parameter will be required.
+
+ - `databases` - `optional` `[string]`, load all the tables under this database
+
+ - `tables` - `optional` `[string]`, load tables with the format `DB.TABLE`
+
+ **Note:**
+
+ * If the loaded table already exists in the system, it will be reloaded.
+
+ * The above two parameters `databases` and `tables` cannot be empty at the same time, which means you must use one of them to load tables.
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/tables' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb","tables":["SSB.LINEORDER"],"need_sampling":false}'
+ ```
+
+- Response Details
+
+ - `loaded`, successfully loaded tables
+ - `failed`, failed to load tables
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "loaded":["SSB.LINEORDER"],
+ "failed":[]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Prepare Reload Hive Table {#Prepare-Reload-Hive-Table}
+
+> Call this API to compare the Hive table metadata in Kylin and that in the data source. For Hive table already loaded to Kylin and already used in model and index building, if some columns are deleted, Kyligence will return a failure when reading these columns. Use this API to find the metadata differences and evaluate whether to update the metadata in Kylin by reloading the Hive table.
+
+- `GET http://host:port/kylin/api/tables/pre_reload`
+
+- Introduced in: 4.1.9
+
+- URL Parameters
+
+ - `project` - `required` `string`，project name
+
+ - `table` - `required` `string`， reload table with the format `DB.TABLE`
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/tables/pre_reload?project=ssb&table=SSB.LINEORDER' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Details
+
+- `has_datasource_changed`, source table structure has changed
+ - `has_effected_jobs`, has unfinished jobs related to the table
+ - `has_duplicated_columns`, has duplicated columns
+ - `add_column_count`, number of new columns
+ - `remove_column_count`, number of reduce columns
+ - `data_type_change_column_count`, number of column type changes
+ - `broken_model_count`, number of broken models
+ - `remove_measures_count`, number of impact measures
+ - `remove_dimensions_count`, number of dimensions affected
+ - `remove_layouts_count`, number of deleted indexes
+ - `add_layouts_count`, increased number of indexes
+ - `refresh_layouts_count`, number of indexes refreshed
+ - `snapshot_deleted`, snapshot is deleted
+ - `duplicated_columns`, duplicate column whose format is database.table.column
+ - `effected_jobs`，effected Job ID
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "has_datasource_changed": false,
+ "has_effected_jobs": true,
+ "has_duplicated_columns": true,
+ "add_column_count": 0,
+ "remove_column_count": 0,
+ "data_type_change_column_count": 0,
+ "broken_model_count": 0,
+ "remove_measures_count": 0,
+ "remove_dimensions_count": 0,
+ "remove_layouts_count": 0,
+ "add_layouts_count": 0,
+ "refresh_layouts_count": 0,
+ "snapshot_deleted": true,
+ "dumplicated_columns": ["SSB.LINEORDER.PROFIT", "SSB.LINEORDER.LO_DISCOUNT"],
+ "effected_jobs": ["266c9086-7ffe-44a1-9d5e-f9f9941b891d", "f42e5dd3-78e6-43f8-9bcb-edcb2c09312d"]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Reload Hive Table {#Reload-Hive-Table}
+
+
+- `POST http://host:port/kylin/api/tables/reload`
+
+- Request Permission: MANAGEMENT permission and above
+
+- Introduced in: 4.2.0
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name
+
+ - `table` - `required` `string`, specify the table, format：DB.TABLE
+
+ - `need_sampling` - `required` `boolean`, whether to enable table sampling
+
+ - `sampling_rows` - `optional` `integer`, indicates the max number of sampling rows and the range is [10,000 - 20,000,000]
+
+ > Note: if you enable need_sampling, this parameter will be required
+
+ - `need_building` - `optional` `boolean`, whether to build a new index, `true` means to build, `false` means not to build, default value is `false`
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/tables/reload' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb","table":"SSB.LINEORDER","need_sampling":false,"need_building":false}'
+ ```
+
+- Response Field
+
+ - `sampling_id`, ids of table sampling jobs
+ - `job_ids`, ids of the building jobs
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "sampling_id":"",
+ "job_ids":["1234","1234"]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Prepare Unload Table {#Prepare-Unload-Table}
+
+> Call this API to evaluate the risks of unloading Hive table metadata. There are cases where you need to offline some Hive tables from Kylin. Use this API to evaluate the impact of unloading Hive table metadata on related Kyligence models and jobs.
+
+- `GET http://host:port/kylin/api/tables/{database}/{table}/prepare_unload`
+
+- URL Parameters
+
+ - `database` - `required` `string`，database name of the table to be deleted
+
+ - `table` - `required` `string`，table name to be deleted
+
+ - `project` - `required` `string`，project name
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/tables/SSB/LINEORDER/prepare_unload?project=ssb' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Field
+
+ - `has_job`, whether there are running jobs of sampling or building snapshot in the current table
+ - `has_model`, is the current table used by the model
+ - `has_snapshot`, does the current table have a snapshot
+ - `storage_size`, storage size of the current table snapshot (Byte)
+ - `models`, model list
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "has_job": false,
+ "has_model": true,
+ "has_snapshot": true,
+ "storage_size": 16616,
+ "models": [
+ "model"
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+
+### Unload Table {#Unload-Table}
+
+> Call this API to unload Hive table metadata from Kylin. After the API call, Hive table metadata will be unloaded from Kylin, and Kyligence can no longer read the table data, or update the index data related to the table. It's recommended calling the "Prepare Unload Table" API before calling this API.
+
+- `DELETE http://host:port/kylin/api/tables/{database}/{table}`
+
+- URL Parameters
+
+ - `database` - `required` `string`, database name of the table to be deleted
+
+ - `table` - `required` `string`, table name to be deleted
+
+ - `project` - `required` `string`, project name
+
+ - `cascade` - `optional` `boolean`, delete all. default value is `false`
+ - true: Delete this source table with the snapshot, attached Kafka/Hive table, the referenced models, and stop/delete related jobs.
+ - false: Only delete this source table with the snapshot, and stop related jobs. While the referenced models will be kept (BROKEN, can be fixed by reloading the table).
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://host:port/kylin/api/tables/SSB/LINEORDER?project=ssb' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Field
+
+ - `date`, deleted table name
+
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "SSB.LINEORDER",
+ "msg": ""
+ }
+ ```
+
+
+
+### Table Sampling {#Sampling-Table}
+
+> Call this API to enable data sampling to reflect the characteristics of Hive table data.
+
+- `POST http://host:port/kylin/api/tables/sampling_jobs`
+
+- Request Permission: MANAGEMENT permission and above
+
+- Introduced in: 4.2.0
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name
+ - `qualified_table_name` - `required` `string`,specify the table,format:DB.TABLE
+ - `rows` - `required` `integer`, indicates the max number of sampling rows and the range is [10,000 - 20,000,000]
+ - `priority` - `optional` `integer`, set job priority with range `0-4` which indicates the priority from high to low. Default value is `3`
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/tables/sampling_jobs' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb","qualified_table_name":"SSB.LINEORDER","rows":20000,"priority":0}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
+
+
+
+### Get Partition Format of A Column {#Get-Partition-Column-Format}
+
+> When a column is used as a partition column in a model in Kylin, get the partition format of the column.
+
+- `GET http://host:port/kylin/api/tables/column_format`
+
+- Request Permission: Operation permission and above
+
+- Introduced in: 4.2.0
+
+- Request Parameters
+
+ - `project` - `required` `string`, project name
+ - `table` - `required` `string`, table name, format as DB.TABLE
+ - `column_name` - `required` `string`, column name
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/tables/column_format?project=test&table=DEFAULT.KYLIN_SALES&column_name=PART_DT' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Field
+
+ - `column_name` , column name
+ - `column_format` , column format
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "partition_column": "PART_DT",
+ "format": "yyyy-MM-dd"
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Get Table Information{#Get-Table-Information}
+
+> Call this API to get the metadata of a specified Hive table.
+
+- `GET http://host:port/kylin/api/tables`
+
+- Request Permission: READ permission and above.
+
+- Introduced in: 4.2.0
+
+- Request Parameters
+
+ - `project` - `required` `string`, project name
+ - `database` - `optional` `string`, database name, case sensitive
+ - `table` - `optional` `string`, table name, case sensitive
+ - `is_fuzzy` - `optional` `boolean`, whether to enable fuzzy matching for table names, `true` means to enable, `false` means to close, default value is `false`
+ - `ext` - `optional` `boolean`, specify whether the table's extension information is returned, `true` means to enable, `false` means to close, default value is `true`
+ - `page_offset` - `optional` `int`, offset of returned result, `0` by default
+ - `page_size` - `optional` `int`, quantity of returned result per page, `10` by default
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/tables?project=test&database=SSB&table=KYLIN_SALES' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ {
+ "uuid": "6e638305-1a44-42dc-a161-5e06338dcb14",
+ "last_modified": 1600335525521,
+ "create_time": 1600335525522,
+ "version": "4.0.0.0",
+ "mvcc": 0,
+ "name": "KYLIN_SALES",
+ "columns": [
+ {
+ "id": "1",
+ "name": "TRANS_ID",
+ "datatype": "bigint",
+ "cardinality": null,
+ "min_value": null,
+ "max_value": null,
+ "null_count": null
+ }
+ ],
+ "source_type": 9,
+ "kafka_bootstrap_servers": null,
+ "subscribe": null,
+ "starting_offsets": null,
+ "table_type": "MANAGED",
+ "top": false,
+ "increment_loading": false,
+ "last_snapshot_path": null,
+ "database": "DEFAULT",
+ "exd": {
+ "owner": "root",
+ "create_time": "1524213799000",
+ "total_file_size": "0",
+ "hive_inputFormat": "org.apache.hadoop.mapred.TextInputFormat",
+ "hive_outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+ "location": "hdfs://sandbox.hortonworks.com:8020/apps/hive/warehouse/kylin_sales",
+ "partition_column": "",
+ "total_file_number": "0",
+ "last_access_time": "0"
+ },
+ "root_fact": false,
+ "lookup": false,
+ "primary_key": [],
+ "foreign_key": [],
+ "partitioned_column": null,
+ "partitioned_column_format": null,
+ "segment_range": null,
+ "storage_size": -1,
+ "total_records": 0,
+ "sampling_rows": [],
+ "last_build_job_id": null
+ }
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 3
+ },
+ "msg": ""
+ }
+ ```
+>Note: Prior to KE version 4.5.17.0, the `total_size` value was the total number of tables the requesting user had permissions on in the project. Due to performance issues, in KE 4.5.17.0 and later, the `total_size` value is adjusted to the total number of all loaded tables in the project and no longer correlates to the actual table permissions the requesting user has in the project.
+
diff --git a/website/docs/restapi/error_code.md b/website/docs/restapi/error_code.md
new file mode 100644
index 0000000000..198241237c
--- /dev/null
+++ b/website/docs/restapi/error_code.md
@@ -0,0 +1,156 @@
+---
+title: Error Code
+language: en
+sidebar_label: Error Code
+pagination_label: Error Code
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - error code
+draft: true
+last_update:
+ date: 12/08/2022
+---
+
+[comment]: <#TODO: renew the error code> (#TODO)
+
+If an error occurs when calling the Kylin API or using the built-in tools, you can troubleshoot the problem based on the error codes and messages. This article summarizes the error codes and error messages for the APIs and built-in tools.
+### API error code
+#### Project
+| Error code | Error message |
+|:-----------------|:-------------------------------------------------------------------------------------------------------------|
+| KE-010001201 | Can't find project. Please check and try again. |
+| KE-010001208 | Can't use, the multilevel partitioning is not enabled for the project. Please turn it on and try again. |
+
+#### Model
+| Error code | Error message |
+|:--------------------------|:--------------------------------------------------------------------------------|
+| KE-010002201 | Can't find model. Please check and try again. |
+| KE-010002202 | Can't find model id. Please check and try again. |
+| KE-010002203 | Can't find model name. Please check and try again. |
+| KE-010002204 | The name can't be empty. |
+| KE-010002205 | The model name is invalid. Please use letters, numbers and underlines only. |
+| KE-010002206 | Model already exists. Please rename it. |
+
+#### Authentication
+| Error code | Error message |
+| :----------- | :----------------------------------------------------------- |
+| KE-010003207 | Can't authenticate. Please login again. |
+| KE-010003208 | Invalid username or password. Please check and try again. |
+| KE-010003209 | Can't find authentication information. |
+| KE-040005201 | Can't find PASSWORD ENCODER. Please check configuration item <br />kylin.security.user-password-encoder. |
+| KE-040005202 | Can't initialize PASSWORD ENCODER. Please check configuration item <br />kylin.security.user-password-encoder. |
+
+#### Index
+| Error code | Error message |
+|:-----------------|:--------------------------------------------------------------------------------------------------------------------|
+| KE-010012201 | Index metadata might be inconsistent. Please try refreshing all segments in the following model. |
+| KE-010012202 | Can’t add this index, as the same index already exists. Please modify. |
+| KE-010012203 | The parameter is not supported. |
+
+#### Segment
+| Error code | Error message |
+| :----------- | :----------------------------------------------------------- |
+| KE-010022201 | Can't refresh. The segment range exceeds the range of loaded data. Please modify and <br />try again. |
+| KE-010022202 | Can't build segment. The specified data range overlaps with the built segments. Please <br />modify and try again. |
+| KE-010022203 | Can‘t refresh. Please select segment and try again. |
+| KE-010022204 | Can‘t refresh, some segments are being built. Please try again later. |
+| KE-010022205 | Can‘t refresh, the selected segment range is empty. Please reselect and try again. |
+| KE-010022206 | Please select at most one segment to refresh. |
+| KE-010022207 | Please select at least two segment to merge. |
+| KE-010022208 | Can't merge the selected segments, as there are gap(s) in between. Please check and try again. |
+| KE-010022209 | Can't find the segment by name. Please check and try again. |
+| KE-010022210 | Can't find the segment by ID. Please check and try again. |
+| KE-010022211 | Can't find the segment(s). Please check and try again. |
+| KE-010022212 | You should choose at least one segment! |
+| KE-010022213 | You should choose at least one segment to drop! |
+| KE-010022214 | Can't enter segment ID and name at the same time. Please re-enter. |
+| KE-010022215 | Please enter segment ID or name. |
+| KE-010022216 | Can't remove, refresh or merge segment, as it's LOCKED. Please try again later. |
+| KE-010022217 | Can't refresh or merge segment in current status. Please try again later. |
+| KE-010022218 | The indexes included in the selected segments are not consistent. Please build index first <br />and try merging again. |
+| KE-010022219 | The partitions included in the selected segments are not consistent. Please build the<br /> partitions first and try merging again. |
+
+#### Job
+| Error code | Error message |
+| :----------- | :----------------------------------------------------------- |
+| KE-010032201 | Can't add the job, as the subpartition value is empty. Please check and try again. |
+| KE-010032202 | Can't submit the job, as no index is included in the segment. Please check and try again. |
+| KE-010032203 | Can't submit the job, as there are no segments in READY status. Please try again later. |
+| KE-010032204 | Can't find executable jobs at the moment. Please try again later. |
+| KE-010032205 | Can't add the job. Please ensure that the operation is valid for the current object. |
+| KE-010032206 | Can't submit the job at the moment, as a building job for the same object already exists.<br /> Please try again later. |
+| KE-010032207 | Can't submit the job, as the indexes are not identical in the selected segments. Please<br /> check and try again. |
+| KE-010032208 | Can't submit the job to this node, as it’s not a job node. Please check and try again. <br/> |
+| KE-010032209 | Can't add the job, as there are duplicate subpartition values. Please check and try again. |
+| KE-010032210 | No index is available to be refreshed. Please check and try again. |
+| KE-010032211 | Can’t perform job in current status. |
+| KE-010032212 | The selected job status is invalid. The value of status must be "PENDING", "RUNNING", <br />"FINISHED", "ERROR” or "DISCARDED". Please check and try again. |
+| KE-010032213 | At least one job should be selected. |
+| KE-010032214 | The number of sampling rows are out of range. Please modify it. |
+| KE-010032215 | Can't perform current operation, as an error occurred when updating the job status. |
+| KE-010032216 | Can't submit building jobs, as it exceeds the concurrency limit. Please try submitting <br />fewer jobs at a time. |
+| KE-010032217 | Invalid value in parameter "action" or "statuses" or "job_ids", The value of "statuses" or the<br /> status of jobs specified by "job_ids", this status of jobs can only perform the following actions. |
+| KE-010032218 | No storage quota available. The system failed to submit building job, while the query engine <br />will still be available. Please clean up low-efficient storage in time, increase the low-efficient <br />storage threshold, or notify the administrator to increase the storage quota for this project. |
+| KE-010032219 | Can't find job. Please check and try again. |
+
+
+#### Parameter Check
+
+| Error code | Error message |
+| :----------- | :----------------------------------------------------------- |
+| KE-010043201 | Request parameter is empty or value is empty. Please check the request parameters are <br />filled in correctly. |
+| KE-010043202 | The entered parameter value is invalid. The timestamp of the start time and end time<br /> conversion must be greater than or equal to 0. Please check and try again. |
+| KE-010043203 | The entered parameter value is invalid. The end time must be greater than the start time. <br />Please check and try again. |
+| KE-010043204 | The entered parameter value is invalid. The end time must be greater than or equal to <br />the start time. Please check and try again. |
+| KE-010043205 | The entered parameter format is invalid. The timestamp format must be milliseconds. <br />Please check and try again. |
+| KE-010043206 | The entered parameter value is invalid. The start time and end time must exist or empty at<br /> the same time. Please check and try again. |
+| KE-010043207 | The entered parameter is invalid, only supports specific values at the moment. Please check <br />and try again. |
+| KE-010043208 | The entered parameter value is invalid. The parameter value must be a non-negative <br />integer. Please check and try again. |
+| KE-010043209 | The entered parameter value is invalid. Only support specific values at the moment. Please <br />check and try again. |
+| KE-010043210 | The parameter can't be empty. Please enter the time partition column format. |
+| KE-010043211 | The type of the time partition column is invalid. Please enter the supported format, refer <br />to the [user manual](../../Designers-Guide/model/model_design/data_modeling.en.md#faq). |
+| KE-010043212 | The parameter can't be empty. Please enter layout(s) id. |
+| KE-010043213 | Can't find layout. Please check and try again. |
+| KE-010043214 | Can't refresh the value, the time units are only supported in d (days), h (hours), <br />or m (minutes). Please check and try again. |
+| KE-010043215 | Can't modify this configuration. Please contact admin. |
+| KE-010043216 | Can't delete this configuration. Please contact admin. |
+
+
+#### System
+| Error code | Error message |
+|:---------------|:----------------------------------------------------------------------------|
+| KE-040021201 | System is trying to recover service. Please try again later. |
+| KE-040023201 | Can't execute this request on job node. Please check and try again. |
+| KE-040023202 | Job node is unavailable for queries. Please select a query node. |
+| KE-040023203 | Can't execute this request on query node. Please check and try again. |
+| KE-040024201 | Can't operate Metadata at the moment, as system is in maintenance mode. |
+| KE-040024202 | System is already in maintenance mode. |
+| KE-040024203 | System is not in maintenance mode. |
+| KE-040025201 | System profile abnormal data. |
+
+#### Other
+| Error code | Error message |
+| :----------- | :----------------------------------------------------------- |
+| KE-010007204 | The data type of column from the source table has changed. Please remove the column from <br />model , or modify the data type. |
+| KE-010007208 | The table metadata can't be reloaded now. There are ongoing jobs with the following target <br />subjects(s). Please try reloading until all the jobs are completed, or manually discard the jobs. |
+| KE-010025201 | Can't find the cube. |
+| KE-010031201 | Can’t get query result, as the rows of query result exceeds the maximum limit. Please add <br />filters, or contact admin to adjust the maximum limit. |
+| KE-030001201 | Can't build, the expected number of rows for index does not match the actual built number of rows. |
+| KE-060100201 | An exception occurred outside Kylin. |
+
+
+### Built-in tools error code
+> [!NOTE]
+> Build-in tools include [junk file clean tools](../../Operation-and-Maintenance-Guide/junk_file_clean.en.md), [diagnostic package generate tools](../../Operation-and-Maintenance-Guide/system-operation/cli_tool/diagnosis.en.md), etc.
+
+| Error code | Error message |
+|:-------------------|:--------------------------------------------------------------------------------------|
+| KE-050040201 | Required parameter is empty. |
+| KE-050040202 | Required parameter is not specified. |
+| KE-050040203 | Required parameter is not specified (milliseconds). |
+| KE-050040204 | Parameter "-endTime" <= Parameter "-startTime". |
+| KE-050041201 | The path does not exist. |
+| KE-050041202 | The path already exists. |
diff --git a/website/docs/restapi/image/job_api_en.png b/website/docs/restapi/image/job_api_en.png
new file mode 100644
index 0000000000..a760f31fba
Binary files /dev/null and b/website/docs/restapi/image/job_api_en.png differ
diff --git a/website/docs/restapi/image/metrics_store_en.png b/website/docs/restapi/image/metrics_store_en.png
new file mode 100644
index 0000000000..3595063e76
Binary files /dev/null and b/website/docs/restapi/image/metrics_store_en.png differ
diff --git a/website/docs/restapi/image/work_follow_en.png b/website/docs/restapi/image/work_follow_en.png
new file mode 100644
index 0000000000..784ad483b0
Binary files /dev/null and b/website/docs/restapi/image/work_follow_en.png differ
diff --git a/website/docs/restapi/job_api.md b/website/docs/restapi/job_api.md
new file mode 100644
index 0000000000..9fbcf8472b
--- /dev/null
+++ b/website/docs/restapi/job_api.md
@@ -0,0 +1,168 @@
+---
+title: Job API
+language: en
+sidebar_label: Job API
+pagination_label: Job API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - job api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+## Job API
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+
+* [Get Job List](#Get-Job-List)
+* [Operate Job](#operate-job)
+
+
+
+### Get Job List {#Get-Job-List}
+
+- `GET http://host:port/kylin/api/jobs`
+
+- URL Parameters
+ - `time_filter` - `required` `int`
+
+ 
+
+ - `project` - `optional` `string`, project name
+
+ - `statuses` - `optional` `string`，job status，Optional values：`PENDING`,`RUNNING`,`FINISHED`,`ERROR`,`DISCARDED`,`STOPPED`,Separate multiple values with commas
+
+ - `page_offset` - `optional` `int`, offset of returned result, 0 by default
+
+ - `page_size` - `optional` `int`, quantity of returned result per page, 10 by default
+
+ - `sort_by` - `optional` `string`, sort field, optional values：`last_modified` by default, `project id`,`job_name`,`target_subject`,`create_time`,`total_duration`
+
+ - `reverse` - `optional` `boolean`, whether sort reverse, "true" by default
+
+ - `key` - `optional` `string`, filter field, support job id or object name
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/jobs?time_filter=0&page_size=1' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "value":[
+ {
+ "id":"232bf69f-6cdf-4dcc-a3aa-b6ca7651e98c",
+ "last_modified":0,
+ "duration":91020,
+ "exec_start_time":1577764731199,
+ "steps":null,
+ "job_status":"FINISHED",
+ "job_name":"INDEX_BUILD",
+ "data_range_start":0,
+ "data_range_end":9223372036854775807,
+ "target_model":"5b54898a-dd75-4146-abbe-de77c0cf77fb",
+ "target_segments":[
+ "3344c9bb-83fa-4128-803b-e18f27b0ccf8"
+ ],
+ "step_ratio":1,
+ "create_time":1577764730069,
+ "wait_time":1130,
+ "target_subject":"AUTO_MODEL_KYLIN_ACCOUNT_1",
+ "target_subject_error":false,
+ "project":"test",
+ "submitter":"ADMIN",
+ "exec_end_time":1577764822219,
+ "tag":"mark"
+ }
+ ],
+ "offset":0,
+ "limit":1,
+ "total_size":1364
+ },
+ "msg":""
+ }
+ ```
+
+
+### Operate Job {#operate-job}
+
+- `PUT http://host:port/kylin/api/jobs/status`
+
+- URL Parameters
+ - `action` - `required` `string`, action types for jobs. Optional values are below:
+ - `RESUME`, resume selected jobs from paused/error status
+ - `DISCARD`, discard selected jobs
+ - `PAUSE`, pause selected jobs
+ - `RESTART`, restart selected jobs
+
+ - `project` - `optional` `string`, project name. If only `project` is defined, it will operate all jobs under this project. Note: `project` and `job_ids` cannot be empty at the same time.
+
+ - `job_ids` - `optional` `array<string>`, job id. If only `job_ids` is defined, it will operate all jobs with those ids. Note: `project` and `job_ids` cannot be empty at the same time.
+
+ - - `statuses` - `optional` `array<string>`, filter jobs by statuses based on the filtering results of `project` and `job_ids`.
+ - `PENDING`, pending jobs
+ - `RUNNING`, running jobs
+ - `FINISHED`, finished jobs
+ - `ERROR`, error jobs
+ - `DISCARDED`, discarded jobs
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/jobs/status' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "action" : "PAUSE",
+ "job_ids" : [
+ "d7e4a098-10b6-4961-85b4-9eebfe29eb25",
+ "80f4d168-1074-4218-875c-4c70a4334029"
+ ],
+ "project" : "ssb"
+ }'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
diff --git a/website/docs/restapi/model_api/intro.md b/website/docs/restapi/model_api/intro.md
new file mode 100644
index 0000000000..0f49e728ff
--- /dev/null
+++ b/website/docs/restapi/model_api/intro.md
@@ -0,0 +1,20 @@
+---
+title: Model API
+language: en
+sidebar_label: Model API
+pagination_label: Model API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+## Model API
+
+Kylin provides REST APIs on checking model information, index build and model management.
diff --git a/website/docs/restapi/model_api/model_build_api.md b/website/docs/restapi/model_api/model_build_api.md
new file mode 100644
index 0000000000..90a30648f0
--- /dev/null
+++ b/website/docs/restapi/model_api/model_build_api.md
@@ -0,0 +1,80 @@
+---
+title: Model Build API
+language: en
+sidebar_label: Model Build API
+pagination_label: Model Build API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model build api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](../authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+### Load Segment{#Load-Segment-Expert}
+
+- `POST http://host:port/kylin/api/models/{model_name}/segments`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name.
+
+- HTTP Body: JSON Object
+ - `project` - `required` `string`, project name.
+ - `start` - `optional` `string`, when the model is built in full, it is not required, and it is required when the model is incrementally built. start time of segment (partition column exist), type: timestamp, unit: ms. For example, `694195200000` means `1992-01-01 00:00:00`.
+ - `end` - `optional` `string`, when the model is built in full, it is not required, and it is required when the model is incrementally built. end time of segment (partition column exist), type: timestamp, unit: ms. For example, `883584000000` means `1998-01-01 00:00:00`.
+ - `build_all_indexes` - `optional` `boolean`, build all indexes in the new segment, default value is `true`
+ - `sub_partition_values` - `optional` `Array` , sub-partition value, used for multi-level partition model. the default is empty. For multi-leve partition model, when `build_all_indexes` is `true` (all indexes need to be built), this value is required. When `build_all_indexes` is `false`(when creating an empty segment), the value must be empty.
+ - `priority` - `optional` `int`, set job priority with range `0-4` which indicates the priority from high to low. Default value is `3`
+ - `build_all_sub_partitions` - `optional` `boolean`, build all sub partition values for multi-level partition model,default value is `false`
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](../job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/SSB_LINEORDER/segments' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"doc_expert", "start": "694195200000", "end": "883584000000","build_all_indexes":true,"sub_partition_values":[["1"],["2"]],"build_all_sub_partitions":false}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "jobs": [
+ {
+ "job_name": "INC_BUILD",
+ "job_id": "61f4d697-e648-4ace-8e52-155829417b2a"
+ },
+ {
+ "job_name": "INDEX_BUILD",
+ "job_id": "0217b970-4525-468b-ba25-58bbb168d612"
+ }
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
diff --git a/website/docs/restapi/model_api/model_import_and_export_api.md b/website/docs/restapi/model_api/model_import_and_export_api.md
new file mode 100644
index 0000000000..277183860c
--- /dev/null
+++ b/website/docs/restapi/model_api/model_import_and_export_api.md
@@ -0,0 +1,310 @@
+---
+title: Model Import And Export API
+language: en
+sidebar_label: Model Import And Export API
+pagination_label: Model Import And Export API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model import and export api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](../authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+
+### Get Batch Exportable Models {#Get-Batch-Exportable-Models}
+
+- `GET http://host:port/kylin/api/metastore/previews/models?project=test&model_names=model1,model2`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+ - `model_names` - `optional` `array[string]`, model's name, separated by commas.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl --location --request GET 'http://localhost:7070/kylin/api/metastore/previews/models?project=target_project&model_names=model_index' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ - Field description
+ - `code` - `string`, return code, `000` Means successful import, `999` Means import failed.
+ - `data` - `json`, return data
+ - `uuid` - `string` uuid;
+ - `name` - `string` model's name;
+ - `status` - `string` model's status;
+ - `has_recommendations` - `boolean` whether model has recommendations;
+ - `has_override_props` - `boolean` whether model has override properties;
+ - `has_multiple_partition_values` - `boolean` whether model has multiple columns;
+
+ ```json
+ {
+ "code": "000",
+ "data": [
+ {
+ "uuid": "10d5eb7c-d854-4f72-9e4b-9b1f3c65bcda",
+ "name": "model_index",
+ "status": "ONLINE",
+ "has_recommendations": true,
+ "has_override_props": true,
+ "has_multiple_partition_values": true,
+ "tables": [
+ {
+ "name": "SSB.P_LINEORDER",
+ "kind": "FACT"
+ },
+ {
+ "name": "SSB.CUSTOMER",
+ "kind": "LOOKUP"
+ }
+ ]
+ }
+ ],
+ "msg": ""
+ }
+ ```
+
+
+
+### Model Metadata Export {#Model-Metadata-Export}
+
+- `POST http://host:port/kylin/api/metastore/backup/models?project=test`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Body: JSON Object
+
+ - `names` - `required` `array[string]`, model name list.
+ - `export_recommendations` - `optional` `boolean`, true/false, default value is false,whether export model's recommendations.
+ - `export_over_props` - `optional` `boolean`, true/false, default value is false, whether export model's override props.
+ - `export_multiple_partition_values` - `optional` `boolean`, true/false, default is false, whether export model's multiple partition values.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl --remote-name --remote-header-name --location --request POST 'http://localhost:7070/kylin/api/metastore/backup/models?project=original_project' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ --data '{
+ "names": [
+ "ssb_model"
+ ],
+ "export_recommendations": true,
+ "export_over_props": true,
+ "export_multiple_partition_values": true
+ }'
+ ```
+
+- Response Example
+
+ return zip file on success
+
+> Note: The remote-header-name parameter is available in curl 7.20.0 and above
+
+### Model Metadata Import Validation{#Model-Metadata-Import-Validation}
+
+- `POST http://host:port/kylin/api/metastore/validation/models?project=test`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Body: form-data
+
+ - `file` - `required` `MultipartFile`, model metadata file.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+
+- Curl Request Example
+
+ ```sh
+ curl --location --request POST 'http://localhost:7070/kylin/api/metastore/validation/models?project=original_project' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ --form 'file=@metadata.zip'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "models": {
+ "ssb_model": {
+ "differences": 3,
+ "missing_items": [],
+ "new_items": [],
+ "update_items": [],
+ "reduce_items": [
+ {
+ "reason": null,
+ "attributes": {
+ "name": "LO_SUPPLYCOST",
+ "alias_dot_column": "P_LINEORDER.LO_SUPPLYCOST"
+ },
+ "detail": "LO_SUPPLYCOST",
+ "type": "MODEL_DIMENSION",
+ "model_alias": "ssb_model",
+ "importable": true,
+ "creatable": true,
+ "overwritable": true
+ },
+ {
+ "reason": null,
+ "attributes": {
+ "col_orders": [
+ "P_LINEORDER.LO_CUSTKEY",
+ "P_LINEORDER.LO_DISCOUNT",
+ "P_LINEORDER.LO_LINENUMBER",
+ "P_LINEORDER.LO_ORDERDATE",
+ "P_LINEORDER.LO_ORDERKEY",
+ "P_LINEORDER.LO_PARTKEY",
+ "P_LINEORDER.LO_QUANTITY",
+ "P_LINEORDER.LO_SUPPKEY",
+ "P_LINEORDER.LO_SUPPLYCOST"
+ ]
+ },
+ "detail": "20000050001",
+ "type": "WHITE_LIST_INDEX",
+ "model_alias": "ssb_model",
+ "importable": true,
+ "creatable": true,
+ "overwritable": true
+ },
+ {
+ "reason": null,
+ "attributes": {
+ "col_orders": [
+ "P_LINEORDER.LO_CUSTKEY",
+ "P_LINEORDER.LO_QUANTITY",
+ "P_LINEORDER.LO_SUPPKEY",
+ "LO_SUPPKEY_SUM",
+ "COUNT_ALL"
+ ]
+ },
+ "detail": "180001",
+ "type": "WHITE_LIST_INDEX",
+ "model_alias": "ssb_model",
+ "importable": true,
+ "creatable": true,
+ "overwritable": true
+ }
+ ],
+ "importable": true,
+ "overwritable": true,
+ "creatable": true
+ }
+ }
+ },
+ "msg": ""
+ }
+ ```
+
+ - Field description
+
+ - `code` - `string`, return code, `000` Means successful import, `999` Means import failed.
+ - `data` - `json`, return data
+ - `models` - `array[object]`, validate information of each import model
+ - `key` - `string`, model's name
+ - `differences` - `int`, the total differences of model metadata between zip file and target model
+ - `missing_items` - `array[object]`, table or table column is required but target project didn't have
+ - `new_items` - `array[object]`, the new items between import and target model
+ - `reduce_items` - `array[object]`, the reduced items between import and target model
+ - `update_items` - `array[object]`, the updated items between import and target model
+ - `has_same_name` - `boolean`, whether has same name model
+ - `importable` - `boolean`, whether the model can import
+ - `overwritable` - `boolean`, whether the model can overwrite target model
+ - `creatable` - `boolean`, whether the model can create a new model
+
+### Model Metadata Import {#Model-Metadata-Import}
+
+- `POST http://host:port/kylin/api/metastore/import/models?project=test`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Body: form-data
+
+ - `file` - `required` `MultipartFile`, model metadata file.
+
+ - `request` - `required` `MultipartFile`, json file. file content is:
+
+ ```
+ {
+ "models":[
+ {
+ "original_name":"ssb_model",
+ "target_name":"ssb_model",
+ "import_type":"OVERWRITE"
+ }
+ ]
+ }
+ ```
+
+ - `original_name` -`string` original model's name
+ - `target_name` -`string` target model's name, set value while import_type is NEW
+ - `import_type` - `string` optional values are `NEW`, `OVERWRITE`, `UN_IMPORT`. `NEW` means create new model, `OVERWRITE` means overwrite existing model, `UN_IMPORT` means not import model.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+
+- Curl Request Example
+
+ ```sh
+ curl --location --request POST 'http://localhost:7070/kylin/api/metastore/import/models?project=original_project' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ --form 'file=@metadata.zip;type=application/octet-stream' \
+ --form 'request=@request.json;type=application/json'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
diff --git a/website/docs/restapi/model_api/model_management_api.md b/website/docs/restapi/model_api/model_management_api.md
new file mode 100644
index 0000000000..7d0b4352a1
--- /dev/null
+++ b/website/docs/restapi/model_api/model_management_api.md
@@ -0,0 +1,1835 @@
+---
+title: Model Management API
+language: en
+sidebar_label: Model Management API
+pagination_label: Model Management API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model management api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+> * On how authentication works, see [Access and Authentication REST API](../authentication.md).
+> * On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+> * On Curl command line, escape `%25` if it contains the special char `%`.
+
+
+
+
+### Create Model {#Create_Model}
+
+- `POST http://host:port/kylin/api/models`
+
+- Request Permission: MANAGEMENT permission and above
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name
+ - `fact_table` - `required` `string`, fact table name
+ - `uuid` - `optional` `string`, model ID, default auto generated
+ - `alias` - `required` `string`, model alias name
+ - `management_type` - `required` `string`, for model creation please use: MODEL_BASED
+ - `simplified_measures` - `required` `JSON Object[]`, measures
+ - `name` - `required` `string`, measure name
+ - `expression` - `required` `string`, function, including: SUM,MIN,MAX,TOP_N,COUNT,COUNT_DISTINCT,CORR,PERCENTILE_APPROX,COLLECT_SET
+ - `parameter_value` - `required` `JSON Object[]`, measure parameters
+ - `type` - `required` `string`, parameter type: constant,column
+ - `value` - `required` `int|string`, parameter value (Set value 1 when 'expression' is 'COUNT' and 'type' is 'constant', while others should be: TABLE.COLUMN)
+ - `return_type` - `required` `string`, function return type with arguments including: topn(10)、topn(100)、percentile(100) etc., using empty string("") when no return type needed
+
+ ```
+ Supported return_type values:
+
+ TOP_N:
+ * Top 10: topn(10)
+ * Top 100: topn(100)
+ * Top 1000: topn(1000)
+
+ COUNT_DISTINCT:
+ * Error Rate < 9.75%: hllc(10)
+ * Error Rate < 4.88%: hllc(12)
+ * Error Rate < 2.44%: hllc(14)
+ * Error Rate < 1.72%: hllc(15)
+ * Error Rate < 1.22%: hllc(16)
+ * Precisely: bitmap
+
+ PERCENTILE_APPROX:
+ * percentile(100)
+ * percentile(1000)
+ * percentile(10000)
+ ```
+
+ - `comment` - `optional` `string`, comments
+ - `simplified_dimensions` - `required` `JSON Object[]`, dimensions info
+ - `column` - `required` `string`, format: TABLE.COLUMN
+ - `name` - `required` `string`, column alias, may set the same as column name
+ - `status` - `required` `string`, fixed value: DIMENSION
+ - `computed_columns` - `optional` `JSON Object[]`, computed columns info
+ - `columnName` - `required` `string`, the new column name
+ - `expression` - `required` `string`, expression
+ - `datatype` - `required` `string`, data type: VARCHAR,INT,BIGINT ...standard sql types
+ - `tableIdentity` - `required` `string`, format: SCHEMA.TABLE
+ - `tableAlias` - `required` `string`, table alias
+ - `join_tables` - `required` `JSON Object[]`, join info
+ - `table` - `required` `string`, table name
+ - `join` - `required` `JSON Object`, join info
+ - `type` - `required` `string`, join type: INNER,LEFT
+ - `foreign_key` - `required` `string[]`, foreign key
+ - `primary_key` - `required` `string[]`, primary key
+ - `simplified_non_equi_join_conditions` - `optional` `JSON Object`, non-equivalent join conditions
+
+ (note1: The support of this settings should have 'Support History Table' enabled in advance. Seeing [Slowly Changing Dimension](../../../Designers-Guide/model/model_design/slowly_changing_dimension.en.md))
+
+ (note2: Join relationship >= and < must be used in pairs, and same column must be joint in both conditions)
+
+ - `foreign_key` - `string`, foreign key
+ - `primary_key` - `string`, primary key
+ - `op` - `string`, join relationship: LESS_THAN,GREATER_THAN_OR_EQUAL
+ - `kind` - `optional` `string`, table kind: FACT,LOOKUP, default: LOOKUP
+ - `alias` - `optional` `string`, alias
+ - `flattenable` - `required` `string`, precomputing associations(precomputing: flatten, no-precomputing: normalized), flatten is suggested
+ - `join_relation_type` - `optional` `string`, join type: MANY_TO_ONE,MANY_TO_MANY, default: MANY_TO_ONE
+ - `partition_desc` - `optional` `JSON Object`, partition columns info
+ - `partition_date_column` - `required` `string`, partition date column, format: TABLE.COLUMN
+ - `partition_date_format` - `required` `string`, partition date column format, including: yyyy-MM-dd, yyyyMMdd... Supported date format please check "[Model Design Basics](../../../Designers-Guide/model/model_design/data_modeling.en.md#faq)"
+ - `partition_type` - `optional` `string`, partition type, including: APPEND, default: APPEND
+ - `owner` - `optional` `string`, the owner of model, default current user
+ - `description` - `optional` `string`, model description
+ - `capacity` - `optional` `string`, model capacity, including: SMALL,MEDIUM,LARGE, default: MEDIUM
+ - `filter_condition` - `optional` `string`, data filter condition. Data filter condition is an additional data filter during data loading.
+ - `with_base_index` - `optional` `boolean`, adding base indexes or not. Base indexes include all dimensions and measures of the model and automatically update as the model changes by default. default: false
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ # Stringify the following JSON Object when use
+ -d '{
+ "project": "pj01",
+ "fact_table": "SSB.P_LINEORDER",
+ "alias": "test_model_01",
+ "management_type": "MODEL_BASED",
+ "simplified_measures": [
+ {
+ "name": "COUNT_ALL",
+ "expression": "COUNT",
+ "parameter_value": [
+ {
+ "type": "constant",
+ "value": 1
+ }
+ ],
+ "return_type": ""
+ }
+ ],
+ "simplified_dimensions": [
+ {
+ "column": "P_LINEORDER.LO_ORDERKEY",
+ "name": "LO_ORDERKEY",
+ "status": "DIMENSION"
+ },
+ {
+ "column": "P_LINEORDER.LO_CUSTKEY",
+ "name": "LO_CUSTKEY",
+ "status": "DIMENSION"
+ },
+ {
+ "column": "P_LINEORDER.LO_ORDERDATE",
+ "name": "LO_ORDERDATE",
+ "status": "DIMENSION"
+ },
+ {
+ "column": "P_LINEORDER.LO_ORDERPRIOTITY",
+ "name": "LO_ORDERPRIOTITY",
+ "status": "DIMENSION"
+ },
+ {
+ "column": "P_LINEORDER.V_REVENUE",
+ "name": "V_REVENUE",
+ "status": "DIMENSION"
+ },
+ {
+ "column": "DATES.D_YEAR",
+ "name": "D_YEAR",
+ "status": "DIMENSION"
+ },
+ {
+ "column": "CUSTOMER.C_NAME",
+ "name": "C_NAME",
+ "status": "DIMENSION"
+ },
+ {
+ "column": "CUSTOMER.C_PHONE",
+ "name": "C_PHONE",
+ "status": "DIMENSION"
+ }
+ ],
+ "join_tables": [
+ {
+ "table": "SSB.DATES",
+ "join": {
+ "type": "LEFT",
+ "foreign_key": [
+ "P_LINEORDER.LO_ORDERDATE"
+ ],
+ "primary_key": [
+ "DATES.D_DATE"
+ ]
+ },
+ "alias": "DATES",
+ "flattenable": "flatten",
+ "join_relation_type": "MANY_TO_ONE"
+ },
+ {
+ "table": "SSB.CUSTOMER",
+ "join": {
+ "type": "LEFT",
+ "foreign_key": [
+ "P_LINEORDER.LO_CUSTKEY"
+ ],
+ "primary_key": [
+ "CUSTOMER.C_CUSTKEY"
+ ]
+ },
+ "alias": "CUSTOMER",
+ "flattenable": "flatten",
+ "join_relation_type": "MANY_TO_ONE"
+ }
+ ],
+ "with_base_index": true
+ }'
+ ```
+
+- Response Details
+
+ - `code` - `string`, response code, succeed: `000`, failed: `999`
+ - `msg` - `string`, response message
+ - `data` - `JSON Object`, response data
+ - `base_table_index` - `JSON Object`, base index info
+ - `dimension_count` - `int`, dimension count
+ - `measure_count` - `int`, measure count
+ - `layout_id` - `long`, layout id
+ - `operate_type` - `string`, operation type, including: UPDATE,CREATE
+ - `base_agg_index` - `JSON Object`, base aggregation index info, same structure as `base_table_index`
+ - `warn_code` - `string`, warning code message
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "msg": "",
+ "data": {
+ "base_table_index": {
+ "dimension_count": 8,
+ "measure_count": 0,
+ "layout_id": 20000000001,
+ "operate_type": "CREATE"
+ },
+ "base_agg_index": {
+ "dimension_count": 8,
+ "measure_count": 1,
+ "layout_id": 1,
+ "operate_type": "CREATE"
+ },
+ "warn_code": null
+ }
+ }
+ ```
+
+- Failed Response Example
+
+ ```json
+ {
+ "code": "999",
+ "data": null,
+ "msg": "KE-010001002(Empty Project Name):Can’t find project information. Please select a project.",
+ "stacktrace": "KE-010001002(Empty Project Name) \norg.apache.kylin.common.exception.KylinException: KE-010001002(Empty Project Name):Can’t find project information. ...",
+ "exception": "KE-010001002(Empty Project Name):Can’t find project information. Please select a project.",
+ "url": "http://localhost:7070/kylin/api/models"
+ }
+ ```
+
+- Error Code (Specific error message please check real api return.)
+
+ - `KE-010001001`: Project Not Exist
+ - `KE-010001002`: Empty Project Name
+ - `KE-010006002`: Invalid Partition Column
+ - `KE-010000003`: Invalid Parameter
+ - `KE-010000002`: Invalid Range
+ - `KE-010000004`: Invalid Name
+ - `KE-010006005`: Timestamp Column Not Exist
+ - `KE-010002010`: Failed to Add Models
+ - `KE-010011001`: Duplicate Computed Column Name
+ - `KE-010007001`: Table Not Exist
+
+
+
+### Get Model List {#Get-Model-List}
+
+> Call this API to get a list of models under specified project that satisfies certain conditions.
+
+- `GET http://host:port/kylin/api/models`
+
+- URL Parameters
+
+ - `project` - `required`, `string`, project name.
+ - `page_offset` - `optional`, `int`, offset of returned result, `0` by default.
+ - `page_size` - `optional`, `int`, quantity of returned result per page, `10` by default.
+ - `status` - `optional`, `string`, model status.
+ - `model_name` - `optional`, `string`, model name.
+ - `exact` - `optional`, `boolean`, whether exactly match the model name, `true` by default.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/models?project=doc_expert' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Details
+
+ - `Code`: `String`, response code. **Returned value**: `000` (request processing success), `999 ` (request processing failed)
+ - `data`: `JSON`, returned data
+ - `value`: details of the returned data, which consists of:
+ - `name`: `String`, model name
+ - `lookups`: `JSON`, detailed list of all dimension tables
+ - `size_kb`: `Long`, total size of all segments of the models
+ - `input_records_count`: `Long`, number of flattened tables within all segments
+ - `input_records_size`: `Long`, size of flattened tables within all segments
+ - `project`: `String`, project name
+ - `uuid`: `String`, model ID
+ - `last_modified`: `Long`, last modified time of the model
+ - `create_time`: `Long`, model creation time
+ - `mvcc`: `Long`, version number with metadata modified
+ - `alias`: `String`, model alias
+ - `owner`: `String`, model owner
+ - `config_last_modifier`: `String`, last user who modified the configuration
+ - `fact_table`: `String`, fact table (one model contains only one fact table)
+ - `fact_table_alias`: `String`, fact table alias
+ - `join_tables`: `JSON`, joined tables
+ - `partition_desc`:`JSON`, partition column
+ - `all_measures`: `JSON`, all measures within the model
+ - `multi_partition_desc`: `JSON`, multiple partitions
+ - `computed_columns`: `JSON`, computed columns
+ - `canvas`: `JSON`, position of model canvas
+ - `status`: `String`, model status; **Returned value**: `online`, `offline`, `broken`, `warning`
+ - `last_build_end`: `String`, building time of the last segment
+ - `storage`: `Long`, total storage size of the model; **Unit:** byte
+ - `source`: `Long`, sourceByte sum of all segments in the model
+ - `expansion_rate`: `String`, model expansion rate; **Unit:** %
+ - `usage`: `Long`, queried times of the model
+ - `model_broken`: `Boolean`, if the model is broken or not
+ - `root_fact_table_deleted`: `Boolean`, if the fact table is deleted
+ - `segments`: `JSON`, segment lists
+ - `recommendations_count`: `Integer`, number of recommendations
+ - `simplified_measures`: `JSON`, measure list of the models
+ - `simplified_dimensions`: `JSON`, dimension list of the models
+ - `simplified_tables`: `JSON`, all tables in the model
+ - `offset`: page number
+ - `limit`: models listed in each page
+ - `total_size`: total number of models
+
+- Response Example
+
+ > [!NOTE]
+ >
+ > For parameters not listed in this page, you may go to the [Kylin Console](../../../Designers-Guide/model/model_concepts_operations.en.md) for details.
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "value":[
+ {
+ "name":"Model_03",
+ "lookups":Array[1],
+ "is_streaming":false,
+ "size_kb":0,
+ "input_records_count":0,
+ "input_records_size":0,
+ "project":null,
+ "uuid":"0464241b-fd7d-49a9-a3c9-b4f7e32cf489",
+ "last_modified":1574750372949,
+ "create_time":1574761225505,
+ "version":"4.0.0.0",
+ "mvcc":4,
+ "alias":"Model_03",
+ "owner":"ADMIN",
+ "config_last_modifier":null,
+ "config_last_modified":0,
+ "description":"",
+ "fact_table":"SSB.LINEORDER",
+ "fact_table_alias":null,
+ "management_type":"MODEL_BASED",
+ "join_tables":Array[1],
+ "filter_condition":"",
+ "partition_desc":Object{...},
+ "capacity":"MEDIUM",
+ "segment_config":Object{...},
+ "data_check_desc":null,
+ "semantic_version":0,
+ "all_named_columns":Array[26],
+ "all_measures":Array[2],
+ "column_correlations":Array[0],
+ "multilevel_partition_cols":Array[0],
+ "computed_columns":Array[0],
+ "canvas":Object{...},
+ "status":"ONLINE",
+ "last_build_end":"902016000000",
+ "storage":24694,
+ "source":5585164,
+ "expansion_rate":"0.44",
+ "usage":0,
+ "model_broken":false,
+ "root_fact_table_deleted":false,
+ "segments":null,
+ "recommendations_count":0,
+ "simplified_measures":Array[2],
+ "simplified_dimensions":Array[9],
+ "simplified_tables":Array[2],
+ "multi_partition_desc": {
+ "columns": ["KYLIN_SALES.LSTG_SITE_ID"],
+ "partitions": [
+ {
+ "id": 0,
+ "values": [
+ "1"
+ ]
+ },
+ {
+ "id": 1,
+ "values": [
+ "2"
+ ]
+ }
+ ],
+ "max_partition_id": 1
+ }
+ },
+ Object{...},
+ Object{...}
+ ],
+ "offset":0,
+ "limit":10,
+ "total_size":3
+ },
+ "msg":""
+ }
+ ```
+
+### Get Model Description {#Get-Model-Description}
+
+> Call this API to get the descriptions of a model, for example, dimension, measure, fact table, and table join relations.
+>
+
+- `GET http://host:port/kylin/api/models/{project}/{model_name}/model_desc`
+
+ > The API response does not contain dimensions and indexes recommended by the system.
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+ - `model_name` - `required` `string`, model name.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/models/Kylin/AUTO_MODEL/model_desc'\
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Details
+
+ - `code`: `String`, response code. **Returned value**: `000` (request processing success), `999 ` (request processing failed)
+ - `data`: `JSON`, returned data
+ - `uuid`: `String`, model ID
+ - `last_modified`: `Long`, last modified time of the model
+ - `name`: `String`, model name
+ - `project`: `String`, project name
+ - `dimensions`: `JSON`, dimension information
+ - `id`: `Integer`, dimension ID
+ - `name`: `String`, dimension name
+ - `column`: `String`, column name
+ - `status`: `String`, status
+ - `derived`: `Boolean`, if the column is derived dimension
+ - `measures`: `JSON`, measure information
+ - `name`: `String`, measure name
+ - `function`: `JSON`, functions
+ - `expression`: `String`, expressions
+ - `parameters`: `JSON`, parameters
+ - `type`: `String`, parameter type
+ - `value`: `String`, parameter value
+ - `returntype`: `String`, returned type
+ - `id`: `Integer`, measure ID
+ - `aggregation_groups`: `JSON`, aggregation groups
+ - `includes`: `JSON`, columns in aggregation groups
+ - `select_rule`: `JSON`, type of aggregation groups
+ - `hierarchy_dims`: `JSON`, dimension hierarchy
+ - `mandatory_dims`: `JSON`, required hierarchy
+ - `joint_dims`: `JSON`, joint dimensions
+ - `computed_columns`: `JSON`, [Computed Column](../../../Designers-Guide/model/model_design/computed_column/README.md)
+ - `tableIdentity`: `String`, table ID
+ - `tableAlias`: `String`, table alias
+ - `columnName`: `String`, column name
+ - `expression`: `String`, expression of computed columns
+ - `datatype`: `String`, column type
+ - `comment`: `String`, notes of the columns
+ - `rec_uuid`: `String`, primary key of the column
+ - `fact_table`: `String`, fact table
+ - `join_tables`: `JSON`, joined tables
+ - `table`: `String`, table name
+ - `kind`: `String`, table type; **Returned Value**: `FACT`, `LOOKUP`
+ - `alias`: `String`, table alias
+ - `join`: `String`, join relation
+ - `type`: `String`, join type; **Returned Value**: `INNER`, `LEFT`, `RIGHT`, `OUTER`
+ - `primary_key`: `JSON`, column that is a reference to other tables **Data Type**: JSON
+ - `foreign_key`: `JSON`, columns in table that are joined with this table
+ - `non_equi_join_condition`: `JSON`, non-equi join conditions (object)
+ - `primary_table`: `String`, current table
+ - `foreign_table`: `String`, tables joined to current table
+ - `flattenable`: `String`, if dimension tables are flattened tables, **Data Type**: String
+ - `join_relation_type`: `String`, join relations; **Returned Value**: `MANY_TO_ONE`, `ONE_TO_ONE`, `ONE_TO_MANY`, `MANY_TO_MANY`
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "uuid":"24dc8eae-e613-40ce-b601-26f065f24070",
+ "last_modified":1577436020423,
+ "version":"4.0.0.0",
+ "name":"Kylin",
+ "project":"test_gy",
+ "description":"",
+ "dimensions":[
+ {
+ "id":8,
+ "name":"P_LINEORDER_LO_CUSTKEY",
+ "column":"P_LINEORDER.LO_CUSTKEY",
+ "status":"DIMENSION",
+ "derived":false
+ }
+ ],
+ "measures":[
+ {
+ "name":"COUNT_ALL",
+ "function":{
+ "expression":"COUNT",
+ "parameters":[
+ {
+ "type":"constant",
+ "value":"1"
+ }
+ ],
+ "returntype":"bigint"
+ },
+ "id":100000
+ },
+ {
+ "name":"TEST",
+ "function":{
+ "expression":"SUM",
+ "parameters":[
+ {
+ "type":"column",
+ "value":"P_LINEORDER.LO_QUANTITY"
+ }
+ ],
+ "returntype":"bigint"
+ },
+ "id":100001
+ }
+ ],
+ "aggregation_groups":[
+ {
+ "includes":[
+ "P_LINEORDER.LO_CUSTKEY"
+ ],
+ "select_rule":{
+ "hierarchy_dims":[
+
+ ],
+ "mandatory_dims":[
+
+ ],
+ "joint_dims":[
+
+ ]
+ }
+ }
+ ],
+ "computed_columns": [{
+ "tableIdentity": "SSB.P_LINEORDER",
+ "tableAlias": "P_LINEORDER",
+ "columnName": "CASTCOL",
+ "expression": "CAST(P_LINEORDER.LO_PARTKEY as bigint)",
+ "innerExpression": "CAST(`P_LINEORDER`.`LO_PARTKEY` as bigint)",
+ "datatype": "BIGINT",
+ "comment": null,
+ "rec_uuid": null
+ },{...}],
+ "fact_table": "SSB.P_LINEORDER",
+ "join_tables": [{
+ "table": "SSB.CUSTOMER",
+ "kind": "LOOKUP",
+ "alias": "CUSTOMER",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "CUSTOMER.C_CUSTKEY"
+ ],
+ "foreign_key": [
+ "P_LINEORDER.LO_CUSTKEY"
+ ],
+ "non_equi_join_condition": null,
+ "primary_table": null,
+ "foreign_table": null
+ },
+ "flattenable": "flatten",
+ "join_relation_type": "MANY_TO_ONE"
+ },{...}]
+ },
+ "msg":""
+ }
+ ```
+
+### Define Partition Column{#Define Partition Colum}
+
+> Call this API to set partition columns for certain model of certain project. To enable incremental building for a model, you should first define a partition column for this model. After this operation, Kylin will complete incremental building.
+
+- `PUT http://host:port/kylin/api/models/{model_name}/partition`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name.
+
+- HTTP Body: JSON Object
+ - `project` - `required` `string`，project name.
+
+ - `partition_desc` - `optional`， partition colunm information
+ - `partition_date_column` - `optional` `string`, partition colunm
+ - `partition_date_format` - `optional` `string`，partition format
+
+ - `multi_partition_desc` - `optional`, multi-level partitioning model sub-partition colunm information
+ - `columns` - `array[string]`, multi-level partitioning model sub-partition colunm
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ http://host:port/kylin/api/models/multi_partition_model/partition \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "project": "multi_partition",
+ "partition_desc": {
+ "partition_date_column": "KYLIN_SALES.PART_DT",
+ "partition_date_format": "yyyy-MM-dd"
+ },
+ "multi_partition_desc": {
+ "columns": [
+ "KYLIN_SALES.LSTG_SITE_ID"
+ ]
+ }
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
+
+### Define Partition Column(Deprecated){#Define-Partition-Column-Deprecated}
+
+- `PUT http://host:port/kylin/api/models/{project}/{model_name}/partition_desc`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name.
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - `partition_desc` - `optional` `string`, indicate the column name and data format of partition column.
+
+ - `start` - `optional` `string`, start time of segment (partition column exist), default value is `1`, type: timestamp, unit: ms.
+
+ - `end` - `optional` `string`, end time of segment (partition column exist), default value is`9223372036854775806`, type: timestamp, unit: ms.
+
+ > If you need to set the model partition column, the above parameters need to be filled in. Otherwise, if you want the model to be defined as a non-partition, please leave these parameters blank.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/models/ssb/test/partition_desc' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "partition_desc":{
+ "partition_date_column":"LINEORDER.LO_ORDERDATE",
+ "partition_date_format":"yyyy-MM-dd"},
+ "start":"0",
+ "end":"1111"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
+
+
+
+### Model Validation{#Model-Validation}
+
+> Used to determine whether the SQLs can be answered by an existing model which includes the original model and the models with recommendations.
+>
+
+- `POST http://host:port/kylin/api/models/model_validation`
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `sqls` - `required` `array`, query list.
+
+* Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/model_validation' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb", "sqls":["select LO_CUSTKEY, count(1) from SSB.P_LINEORDER GROUP BY LO_CUSTKEY1", "SELECT LO_SHIPPRIOTITY,count(1) FROM SSB.LINEORDER GROUP BY LO_SHIPPRIOTITY"]}'
+ ```
+
+* Response Details
+
+ * `valid_sqls`, valid queries and the corresponding model. If there is no answerable model, the list will be empty.
+ * `error_sqls`, queries failed to pass the syntax check.
+ * `error_sqls_detail`, detailed error message of queries failed to pass the syntax check
+
+* Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "valid_sqls":{
+ "SELECT LO_SHIPPRIOTITY,count(1) FROM SSB.LINEORDER GROUP BY LO_SHIPPRIOTITY":[
+ "AUTO_MODEL_LINEORDER_2"
+ ]
+ },
+ "error_sqls":[
+ "select LO_CUSTKEY, count(1) from SSB.P_LINEORDER GROUP BY LO_CUSTKEY1"
+ ],
+ "error_sqls_detail": [
+ {
+ "sql": "select LO_CUSTKEY, count(1) from SSB.P_LINEORDER GROUP BY LO_CUSTKEY1",
+ "sql_advices": [
+ {
+ "suggestion": "Please add column LO_CUSTKEY1 to data source.",
+ "incapable_reason": "Column 'LO_CUSTKEY1' not found in any table."
+ }
+ ]
+ }
+ ]
+ },
+ "msg":""
+ }
+ ```
+
+
+
+### Get index list{#get-index-list}
+
+> Get indexes of given model.
+>
+
++ `GET http://host:port/kylin/api/models/{model_name}/indexes`
+
++ URL Parameters
+
+ + `project` - `required` `string`, project name
+ + `model_name` - `required ` `string`，model alias
+ + `status` - `optional` `string`, index status，support `NO_BUILD`, `BUILDING`, `LOCKED`, `ONLINE`, empty by default
+ + `page_offset` - `optional` `int`, offset of returned result, `0` by default.
+ + `page_size` - `optional` `int`, quantity of returned result per page, `10` by default.
+ + `sources` - `optional` `string`, type of index，support `RECOMMENDED_AGG_INDEX`, `RECOMMENDED_TABLE_INDEX`, `CUSTOM_AGG_INDEX`, `CUSTOM_TABLE_INDEX`, empty by default
+ + `sort_by` - `optional` `string`, sort by field，support `last_modified`, `usage`, `data_size`, `last_modified` by default
+ + `reverse` - `optional` `boolean`, whether sort reverse, `true` by default
+ + `batch_index_ids` - `optional` `array[long]`,specify indexId, empty by default
+
++ HTTP Header
+
+ + `Accept: application/vnd.apache.kylin-v4-public+json`
+ + `Accept-Language: en`
+ + `Content-Type: application/json;charset=utf-8`
+
++ Curl Request Example
+
+ ```shell
+ curl -X GET 'http://localhost:7070/kylin/api/models/m1/indexes?project=ssb&batch_index_ids=1,10001,20001' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+* Response Details
+
+ * `absent_batch_index_ids`, not found indexId
+ * `indexes`, index list
+ * `related_tables`, index related table
+
+* Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "project": "ssb",
+ "uuid": "383cd655-89f4-4986-85cf-1d299e287e84",
+ "model_name": "m1",
+ "total_size": 2,
+ "offset": 0,
+ "limit": 10,
+ "owner": "ADMIN",
+ "absent_batch_index_ids": [20001],
+ "indexes": [
+ {
+ "id": 1,
+ "status": "NO_BUILD",
+ "source": "CUSTOM_AGG_INDEX",
+ "col_order": [
+ {
+ "key": "LINEORDER.LO_ORDERKEY",
+ "value": "column",
+ "cardinality": null
+ },
+ {
+ "key": "LINEORDER.LO_LINENUMBER",
+ "value": "column",
+ "cardinality": null
+ },
+ {
+ "key": "COUNT_ALL",
+ "value": "measure",
+ "cardinality": null
+ }
+ ],
+ "shard_by_columns": [],
+ "sort_by_columns": [],
+ "data_size": 0,
+ "usage": 0,
+ "last_modified": 1610462519405,
+ "related_tables": ["SSB.LINEORDER"],
+ "storage_type": 20
+ },
+ {
+ "id": 10001,
+ "status": "NO_BUILD",
+ "source": "CUSTOM_AGG_INDEX",
+ "col_order": [
+ {
+ "key": "LINEORDER.LO_ORDERKEY",
+ "value": "column",
+ "cardinality": null
+ },
+ {
+ "key": "COUNT_ALL",
+ "value": "measure",
+ "cardinality": null
+ }
+ ],
+ "shard_by_columns": [],
+ "sort_by_columns": [],
+ "data_size": 0,
+ "usage": 0,
+ "last_modified": 1610462519405,
+ "related_tables": ["SSB.LINEORDER"],
+ "storage_type": 20
+ }
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Create a Model by SQL{#sql}
+
+- `POST http://host:port/kylin/api/models/model_suggestion`
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `sqls` - `required` `array`, queries to create models
+ - `with_segment` - `optional`, `bool`, create an empty segment, default is true
+ - `with_model_online` - `optional`, `bool`, model online or not, default is false
+
+* Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/model_suggestion' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb", "sqls":["select count(*) from ssb.P_LINEORDER", "SELECT 1"]}'
+ ```
+
+* Response Details
+
+ * `models`, model details
+ * `uuid`, model UUID
+ * `alias`, model name
+ * `version`, model version
+ * `rec_items`, new model information. Please note that there will be some model detail info such as dimensions or measure shown as a recommendation, but it will not truly generate such a recommendation.
+ * `sqls`, sqls used by one recommendation item
+ * `index_id`, reused index id, if the value is -1, a new model created
+ * `dimensions`, all dimensions used, `new` used to indicate whether it is newly created or reused
+ * `measues`, all measures used, `new` used to indicate whether it is newly created or reused
+ * `computed_columns`, all computed_columns used, `new` used to indicate whether it is newly created or reused
+ * `error_sqls`, failed SQLs
+
+* Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "models": [
+ {
+ "uuid": "364e4485-433c-4fe2-be57-02c59170b5d4",
+ "alias": "AUTO_MODEL_P_LINEORDER_1",
+ "version": "4.0.0.0",
+ "rec_items": [
+ {
+ "sqls": [
+ "select count(*) from ssb.P_LINEORDER"
+ ],
+ "index_id": -1,
+ "dimensions": [],
+ "measures": [
+ {
+ "measure": {
+ "name": "COUNT_ALL",
+ "function": {
+ "expression": "COUNT",
+ "parameters": [
+ {
+ "type": "constant",
+ "value": "1"
+ }
+ ],
+ "returntype": "bigint"
+ },
+ "id": 100000
+ },
+ "new": true
+ }
+ ],
+ "computed_columns": []
+ }
+ ]
+ }
+ ],
+ "error_sqls": [
+ "SELECT 1"
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Model Optimization{#Optimize-Model}
+
+
+- `POST http://host:port/kylin/api/models/model_optimization`
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+
+ - `Content-Type: application/json;charset=utf-8`
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `sqls` - `required` `array`, queries to optimize models.
+ - `accept_recommendation` - `optionalboolean`, default value is `false`, in which case the recommendations will be saved into the recommendation list. When it is set as `true`, the recommendations will be transferred to index directly.
+
+
+* Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/model_optimization' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb", "sqls":["select LO_CUSTKEY, count(1) from SSB.P_LINEORDER GROUP BY LO_CUSTKEY", "SELECT 1"]}'
+ ```
+
+* Response Details
+
+ * `models` , model details
+ * `alias`, model name
+ * `rec_items`, recommendation details
+ * `sqls`, sqls used by one recommendation
+ * `index_id`, a temporary index id
+ * `dimensions`, all dimensions used by this recommendation
+ * `measures`, all measures used by this recommendation
+ * `computed_column`, all computed columns used by this recommendation
+ * `error_sqls`, failed SQLs
+
+* Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "models": [
+ {
+ "uuid": "364e4485-433c-4fe2-be57-02c59170b5d4",
+ "alias": "AUTO_MODEL_P_LINEORDER_1",
+ "version": "4.0.0.0",
+ "rec_items": [
+ {
+ "sqls": [
+ "select LO_CUSTKEY, count(1) from SSB.P_LINEORDER GROUP BY LO_CUSTKEY"
+ ],
+ "index_id": 10001,
+ "dimensions": [
+ {
+ "dimension": {
+ "id": 1,
+ "name": "P_LINEORDER_0_DOT_0_LO_CUSTKEY",
+ "column": "P_LINEORDER.LO_CUSTKEY",
+ "status": "DIMENSION"
+ },
+ "dataType": "integer",
+ "new": true
+ }
+ ],
+ "measures": [
+ {
+ "measure": {
+ "name": "COUNT_ALL",
+ "function": {
+ "expression": "COUNT",
+ "parameters": [
+ {
+ "type": "constant",
+ "value": "1"
+ }
+ ],
+ "returntype": "bigint"
+ },
+ "id": 100000
+ },
+ "new": false
+ }
+ ],
+ "computed_columns": []
+ }
+ ]
+ }
+ ],
+ "error_sqls": [
+ "SELECT 1"
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Accelerate SQL{#Accelerate-sql}
+
+ - `POST http://host:port/kylin/api/models/sql_acceleration`
+ - HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+ - HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `sqls` - `required` `array`, queries to optimize models.
+ - `accept_recommendation` - `optionalboolean`, default value is `false`, in which case the recommendations will be saved into the recommendation list. When it is set as `true`, the recommendations will be transferred to index directly.
+ - `with_segment` - `optional`, `bool`, create an empty segment, default is true
+ - `with_model_online` - `optional`, `bool`, model online or not, default is false
+
+ * Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/sql_acceleration' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb", "sqls":["select count(*) from SSB.LINEORDER","select LO_CUSTKEY, count(1) from SSB.P_LINEORDER GROUP BY LO_CUSTKEY", "SELECT 1"]}'
+ ```
+
+ * Response Details
+
+ * `optimized_models` , optimized model details
+ * `alias`, model name
+ * `rec_items`, recommendation details
+ * `sqls`, sqls used by one recommendation
+ * `index_id`, a temporary index id
+ * `dimensions`, all dimensions used by this recommendation
+ * `measures`, all measures used by this recommendation
+ * `computed_columns`, all computed columns used by this recommendation
+
+ * `created_models`, created model details
+ * `uuid`, model UUID
+ * `alias`, model name
+ * `version`, model version
+ * `rec_items`, new model information. Please note that there will be some model detail info such as dimensions or measure shown as a recommendation, but it will not truly generate such a recommendation.
+ * `sqls`, sqls used by one recommendation item
+ * `index_id`, reused index id, if the value is -1, a new model created
+ * `dimensions`, all dimensions used, `new` used to indicate whether it is newly created or reused
+ * `measures`, all measures used, `new` used to indicate whether it is newly created or reused
+ * `computed_columns`, all computed_columns used, `new` used to indicate whether it is newly created or reused
+
+* `optimal_models`, optimal models details
+
+ * `uuid`, model UUID
+
+ - `alias`, model name
+
+ - `version`, model version
+
+ - `rec_items`, query response details
+ - `sqls`, sql used by one query
+ - `index_id`, index used by one query
+ - `dimensions`, all dimensions about index
+ - `measures`, all measures about index
+ - `computed_columns`, all coputed_columns about index
+
+ * `error_sqls`, failed SQLs
+
+ * Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "optimized_models": [
+ {
+ "uuid": "364e4485-433c-4fe2-be57-02c59170b5d4",
+ "alias": "AUTO_MODEL_P_LINEORDER_1",
+ "version": "4.0.0.0",
+ "rec_items": [
+ {
+ "sqls": [
+ "select LO_CUSTKEY, count(1) from SSB.P_LINEORDER GROUP BY LO_CUSTKEY"
+ ],
+ "index_id": 10001,
+ "dimensions": [
+ {
+ "dimension": {
+ "id": 1,
+ "name": "P_LINEORDER_0_DOT_0_LO_CUSTKEY",
+ "column": "P_LINEORDER.LO_CUSTKEY",
+ "status": "DIMENSION"
+ },
+ "dataType": "integer",
+ "new": true
+ }
+ ],
+ "measures": [
+ {
+ "measure": {
+ "name": "COUNT_ALL",
+ "function": {
+ "expression": "COUNT",
+ "parameters": [
+ {
+ "type": "constant",
+ "value": "1"
+ }
+ ],
+ "returntype": "bigint"
+ },
+ "id": 100000
+ },
+ "new": false
+ }
+ ],
+ "computed_columns": []
+ }
+ ]
+ }
+ ],
+ "created_models": [
+ {
+ "uuid": "364e4485-433c-4fe2-be57-02c59170b5d4",
+ "alias": "AUTO_MODEL_LINEORDER_1",
+ "version": "4.0.0.0",
+ "rec_items": [
+ {
+ "sqls": [
+ "select count(*) from SSB.LINEORDER"
+ ],
+ "index_id": -1,
+ "dimensions": [],
+ "measures": [
+ {
+ "measure": {
+ "name": "COUNT_ALL",
+ "function": {
+ "expression": "COUNT",
+ "parameters": [
+ {
+ "type": "constant",
+ "value": "1"
+ }
+ ],
+ "returntype": "bigint"
+ },
+ "id": 100000
+ },
+ "new": true
+ }
+ ],
+ "computed_columns": []
+ }
+ ]
+ }
+ ],
+ "optimal_models":[
+ {
+ "uuid":"6731928e-0f6c-6a79-f430-9e2c938689de",
+ "alias":"CONSTANT",
+ "version":"4.0.0.0",
+ "rec_items":[
+ {
+ "sqls":[
+ "select 1"
+ ],
+ "index_id":-1,
+ "dimensions":[
+
+ ],
+ "measures":[
+
+ ],
+ "computed_columns":[
+
+ ]
+ }
+ ]
+ },
+ ],
+ "error_sqls": [
+ "SELECT SELECT"
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Get Model Recommendations{#Get_Model_Recommendations}
+
+> Call this API to get model recommendations for certain models. For SQL queries answered by pushdown rather than the models, Kylin will generate corresponding recommendations. Making this API call returns recommendations for specified models.
+
+- `GET http://host:port/kylin/api/models/{model_name}/recommendations`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name.
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```shell
+ curl -X GET \
+ 'http://host:port/kylin/api/models/ssb_test/recommendations?project=ssb'\
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+* Response Details
+
+ * `layouts`, all index recommendations
+ * `add`, recommendation used for adding or removing index
+ * `agg`, index related to this recommendation is AggIndex or TableIndex
+ * `item_id`, recommendation id
+ * `is_agg`, the same to `agg`
+ * `is_add`, the same to `add`
+ * `type`, recommendation type, including ADD_AGG_INDEX, REMOVE_AGG_INDEX, ADD_TABLE_INDEX, REMOVE_TABLE_INDEX
+ * `create_time`, create time of recommendation
+ * `last_modified`, last modified time of recommendation
+ * `hit_count`, used times of this recommendation in history
+ * `index_id`, ID of indexes in recommendations and if parameter `add` is true, this one should be ignored
+ * `data_size`, data size of indexes in recommendations and if parameter `add` is true, this one will be `-1`
+ * `memo_info`, comments of recommendations of deleting indexes
+ * `size`, all index recommendation size
+ * `model_id`, model id
+ * `project`, project name
+
+* Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "layouts": [{
+ "add": true,
+ "agg": true,
+ "item_id": 34,
+ "is_agg": true,
+ "is_add": true,
+ "type": "ADD_AGG_INDEX",
+ "create_time": 1601027171262,
+ "last_modified": 1601028480033,
+ "hit_count": 20,
+ "index_id": 0,
+ "data_size": -1,
+ "memo_info": {}
+ }, {
+ "add": false,
+ "agg": true,
+ "item_id": 35,
+ "is_agg": true,
+ "is_add": false,
+ "type": "REMOVE_AGG_INDEX",
+ "create_time": 1601028374011,
+ "last_modified": 1601028491121,
+ "hit_count": 2,
+ "index_id": 1,
+ "data_size": 155219,
+ "memo_info": {
+ "index_opt_reason": "INCLUDED"
+ }
+ }],
+ "size": 2,
+ "model_id": "931c95ba-a6c1-4d7e-8f30-b4599dae16ed",
+ "project": "ssb"
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Approve Model Recommendation in batch{#Approve_Model_Recommendation_in_batch}
+
+> Call this API to accept recommendations for certain models in batch. The recommendations are provided on a model level, that is, you can accept or reject all recommendations for certain model.
+
+- `PUT http://host:port/kylin/api/models/recommendations/batch`
+- HTTP Body: JSON Object
+ - `project` - `required` `string`, project name.
+ - `filter_by_models ` - `required` `boolean` , whether to approve recommendations by models. The default value is `true`, which means you only approve the recommendations under the models specified by `model_names` . Otherwise, all the recommendations in this project will be approved.
+ - `model_names` - `optional` `array`, model name list.
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+
+ - `Content-Type: application/json;charset=utf-8`
+
+* Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/models/recommendations/batch' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project": "ssb",
+ "filter_by_models": "true",
+ "model_names": ["ssb_test"]
+ }'
+ ```
+
+* Response Details
+
+ - project, project name
+ - models, information of affected models
+ - uuid, model uuid
+ - alias, model name
+ - added_indexes：the added index IDs among approved recommendations
+ - deleted_indexes：the removed index IDs among approved recommendations
+
+* Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "project": "ssb",
+ "models": [
+ {
+ "uuid": "364e4485-433c-4fe2-be57-02c59170b5d4",
+ "alias": "ssb_test",
+ "added_indexes": [20001,30001],
+ "deleted_indexes": []
+ }]
+ },
+ "msg": ""
+ }
+ ```
+
+### Delete Model {#Delete_Model}
+
+- `DELETE http://host:port/kylin/api/models/{model_name}`
+
+- Request Permission: MANAGEMENT permission and above
+
+- Introduced in: 4.2.0
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name
+
+- Request Parameters
+
+ - `project` - `required` `string`, project name
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://host:port/kylin/api/models/{model_name}?project=test' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
+
+### Export TDS File {#EXPORT_TDS}
+- `GET http://host:port/kylin/api/models/{model_name}/export`
+- URL Parameters
+
+ - `model_name` - `required` `string`, the model name
+ - `project` - `required` `string`, the project name
+ - `export_as` - `required` `string`, the format to export to
+ - `TABLEAU_ODBC_TDS` export as Tableau TDS with Kyligence ODBC datasource
+ - `TABLEAU_CONNECTOR_TDS` export as Tableau TDS with Kyligence Tableau Connector as the datasource
+ - `element` - `string` `optional` the model elements to export
+ - `AGG_INDEX_COL` (default) export only dimensions and meausres defined in Aggregate Groups
+ - `AGG_INDEX_AND_TABLE_INDEX_COL` export only dimensions and meausres defined in Aggregate Groups or Table Indexes
+ - `ALL_COLS` export all dimensions and measures defined in the model
+ - `server_host` - `string` `optional`, the host name of the exported Kylin datasource, default to the current requesting host
+ - `server_port` - `string` `optional`, the port of the export Kylin datasource, default to the current requesting port
+- HTTP Header
+
+ - Accept:application/vnd.apache.kylin-v4-public+json
+ - Accept-Language: en
+ - Content-Type: application/json;charset=utf-8
+- Curl Request Example
+
+```sh
+curl -X GET \
+ 'http://localhost:7070/kylin/api/models/a3/export?project=test_project&export_as=TABLEAU_ODBC_TDS&element=AGG_INDEX_COL&server_host=host&server_port=7080' \
+ -H 'accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'authorization: Basic QURNSU46S1lMSU4=' \
+ -H'Content-Type: application/json;charset=utf-8' \
+
+```
+
+### Model Rename {#Model_Rename}
+
+- `PUT http://host:port/kylin/api/models/{model_name}/name`
+
+- Introduced in: 4.5.4
+
+- URL Parameters
+
+ - `model_name` - `required` `string`，model name。
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: cn`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - `project_name` - `required` `string`, project name.
+ - `new_model_name` - `required` `string`, the new name of the model.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/models/sytest/name' \
+ -H'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H'Accept-Language: cn' \
+ -H'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d'{"project":"ssb", "new_model_name":"testNewName"}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+
+
+### Model Online/Offline {#Model_Online_Offline}
+
+- `PUT http://host:port/kylin/api/models/{model_name}/status`
+
+- Request Permission: MANAGEMENT permission and above
+
+- Introduced in: 4.5.11
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - project - `required` `string`, project name
+ - status - `required` `string`, update model status, including: ONLINE,OFFLINE
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/models/model_test1/status' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ -d '{"project": "pj01", "status": "OFFLINE"}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+- Failed Response Example
+
+ ```json
+ {
+ "code": "999",
+ "data": null,
+ "msg": "KE-010001002(Empty Project Name):Can’t find project information. Please select a project.",
+ "stacktrace": "KE-010001002(Empty Project Name) \norg.apache.kylin.common.exception.KylinException: KE-010001002(Empty Project Name):Can’t find project information. ...",
+ "exception": "KE-010001002(Empty Project Name):Can’t find project information. Please select a project.",
+ "url": "http://localhost:7070/kylin/api/models"
+ }
+ ```
+
+- Error Code (Specific error message please check real api return.)
+
+ - `KE-010001001`: Project Not Exist
+ - `KE-010001002`: Empty Project Name
+ - `KE-010000003`: Invalid Parameter
+
+### Export TDS File based on user rights {#Export_TDS_File_based_on_user_rights}
+
+- `GET http://host:port/kylin/api/models/bi_export`
+- URL Parameters
+
+ - `model_name` - `required` `string`, the model name
+ - `project` - `required` `string`, the project name
+ - `export_as` - `required` `string`, the format to export to
+ - `TABLEAU_ODBC_TDS` export as Tableau TDS with Kyligence ODBC datasource
+ - `TABLEAU_CONNECTOR_TDS` export as Tableau TDS with Kyligence Tableau Connector as the datasource
+ - `element` - `string` `optional` the model elements to export
+ - `AGG_INDEX_COL` (default) export only dimensions and meausres defined in Aggregate Groups based on user rights
+ - `AGG_INDEX_AND_TABLE_INDEX_COL` export only dimensions and meausres defined in Aggregate Groups or Table Indexes based on user rights
+ - `ALL_COLS` export all dimensions and measures defined in the model based on user rights
+ - `CUSTOM_COLS` export custom dimensions and measures defined in the model based on user rights, When using this method, the `dimensions`, ` measures` parameter must be add
+ - `server_host` - `string` `optional`, the host name of the exported Kylin datasource, default to the current requesting host
+ - `server_port` - `string` `optional`, the port of the export Kylin datasource, default to the current requesting port
+ - `dimensions` - `List<String>` `optional`, An exported dimension list in the format of `${table_alias}.${columnName1},${table_alias}.${columnName2} `, Parameter takes effect of `element=CUSTOM_COLS`
+ - `measures` - `List<String>` `optional`, An exported measures list in the format of `${measureName1},${measureName2`, Parameter takes effect of `element=CUSTOM_COLS`
+
+- HTTP Header
+
+ - `Accept:application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+- Curl Request Example
+
+```sh
+curl -X GET \
+ 'http://{host}:{port}/kylin/api/models/bi_export?model_name={modelName}&project={project}&export_as=TABLEAU_ODBC_TDS&server_host=localhost&server_port=8080&dimensions=CUSTOMER.C_NAME,CUSTOMER.CC_6,CUSTOMER.CC_7,CUSTOMER.CC_9,LINEORDER_1.LO_CUSTKEY&measures=m_aa&element=CUSTOM_COLS' \
+ -H 'accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'authorization: Basic QURNSU46S1lMSU4=' \
+```
+
+- Failed Response Example(When all columns in the model have no permissions or table-joined columns have no permissions)
+
+```json
+{
+ "code": "999",
+ "data": null,
+ "msg": "KE-010002022(The table not contains unauthenticated columns):Please add permissions to columns in the table!",
+ "stacktrace": "KE-010002022(The table not contains unauthenticated columns) \norg.apache.kylin.common.exception.KylinException: KE-010002022(The table not contains unauthenticated columns):Please add permissions to columns in the table!8)...。",
+ "exception": "KE-010002022(The table not contains unauthenticated columns):Please add permissions to columns in the table!",
+ "url": "http://localhost:7070/kylin/api/models/bi_export"
+}
+```
+
+- Error Code (Specific error message please check real api return.)
+ - `KE-010002022`: Please add permissions to columns in the table
+- Successful Code (exported as TDS file contents)
+
+```xml
+<?xml version='1.0' encoding='UTF-8'?>
+<datasource formatted-name="federated.0e6gjbn18cj0a41an9pi309itkyi" inline="true" source-platform="win" version="10.0">
+ <connection class="federated">
+ <named-connections>
+ <named-connection caption="localhost" name="genericodbc.11du78x0szfyb51b703es1ocv315">
+ <connection class="genericodbc" dbname="" odbc-connect-string-extras="PROJECT=KE_36166;CUBE=test_model_3_cc" odbc-dbms-name="MySQL" odbc-driver="KyligenceODBCDriver" odbc-dsn="" odbc-suppress-connection-pooling="" odbc-use-connection-pooling="" port="8080" schema="DEFAULT" server="localhost" username="ADMIN"/>
+ </named-connection>
+ </named-connections>
+ <relation join="inner" type="join">
+ <clause type="join">
+ <expression op="=">
+ <expression op="[CUSTOMER].[C_CUSTKEY]"/>
+ <expression op="[CUSTOMER_1].[C_CUSTKEY]"/></expression>
+ </clause>
+ <relation join="inner" type="join">
+ <clause type="join">
+ <expression op="=">
+ <expression op="[CUSTOMER].[C_CUSTKEY]"/>
+ <expression op="[LINEORDER_1].[LO_CUSTKEY]"/></expression>
+ </clause>
+ <relation type="table" connection="genericodbc.11du78x0szfyb51b703es1ocv315" name="CUSTOMER" table="[SSB].[CUSTOMER]"/>
+ <relation type="table" connection="genericodbc.11du78x0szfyb51b703es1ocv315" name="LINEORDER_1" table="[SSB].[LINEORDER]"/></relation>
+ <relation type="table" connection="genericodbc.11du78x0szfyb51b703es1ocv315" name="CUSTOMER_1" table="[SSB1].[CUSTOMER]"/></relation>
+ <cols>
+ <map key="[LO_ORDERPRIOTITY]" value="[LINEORDER_1].[LO_ORDERPRIOTITY]"/>
+ <map key="[C_ADDRESS (CUSTOMER_1)]" value="[CUSTOMER_1].[C_ADDRESS]"/>
+ <map key="[C_NAME (CUSTOMER_1)]" value="[CUSTOMER_1].[C_NAME]"/>
+ <map key="[C_REGION (CUSTOMER_1)]" value="[CUSTOMER_1].[C_REGION]"/>
+ <map key="[C_NATION (CUSTOMER)]" value="[CUSTOMER].[C_NATION]"/>
+ <map key="[C_CUSTKEY (CUSTOMER)]" value="[CUSTOMER].[C_CUSTKEY]"/>
+ <map key="[C_MKTSEGMENT (CUSTOMER)]" value="[CUSTOMER].[C_MKTSEGMENT]"/>
+ <map key="[LO_PARTKEY]" value="[LINEORDER_1].[LO_PARTKEY]"/>
+ <map key="[C_CITY (CUSTOMER)]" value="[CUSTOMER].[C_CITY]"/>
+ <map key="[LO_LINENUMBER]" value="[LINEORDER_1].[LO_LINENUMBER]"/>
+ <map key="[C_REGION (CUSTOMER)]" value="[CUSTOMER].[C_REGION]"/>
+ <map key="[LO_ORDERKEY]" value="[LINEORDER_1].[LO_ORDERKEY]"/>
+ <map key="[C_PHONE (CUSTOMER)]" value="[CUSTOMER].[C_PHONE]"/>
+ <map key="[CC_10]" value="[CUSTOMER].[CC_10]"/>
+ <map key="[C_ADDRESS (CUSTOMER)]" value="[CUSTOMER].[C_ADDRESS]"/>
+ <map key="[CC_4]" value="[CUSTOMER].[CC_4]"/>
+ <map key="[LO_SHIPMODE]" value="[LINEORDER_1].[LO_SHIPMODE]"/>
+ <map key="[C_CITY (CUSTOMER_1)]" value="[CUSTOMER_1].[C_CITY]"/>
+ <map key="[CC_6]" value="[CUSTOMER].[CC_6]"/>
+ <map key="[C_NATION (CUSTOMER_1)]" value="[CUSTOMER_1].[C_NATION]"/>
+ <map key="[CC_5]" value="[CUSTOMER].[CC_5]"/>
+ <map key="[C_PHONE (CUSTOMER_1)]" value="[CUSTOMER_1].[C_PHONE]"/>
+ <map key="[CC_7]" value="[CUSTOMER].[CC_7]"/>
+ <map key="[CC_9]" value="[CUSTOMER].[CC_9]"/>
+ <map key="[LO_QUANTITY]" value="[LINEORDER_1].[LO_QUANTITY]"/>
+ <map key="[LO_SUPPLYCOST]" value="[LINEORDER_1].[LO_SUPPLYCOST]"/>
+ <map key="[C_CUSTKEY (CUSTOMER_1)]" value="[CUSTOMER_1].[C_CUSTKEY]"/>
+ <map key="[LO_CUSTKEY]" value="[LINEORDER_1].[LO_CUSTKEY]"/>
+ <map key="[LO_ORDTOTALPRICE]" value="[LINEORDER_1].[LO_ORDTOTALPRICE]"/>
+ <map key="[C_NAME (CUSTOMER)]" value="[CUSTOMER].[C_NAME]"/>
+ <map key="[LO_COMMITDATE]" value="[LINEORDER_1].[LO_COMMITDATE]"/>
+ <map key="[LO_EXTENDEDPRICE]" value="[LINEORDER_1].[LO_EXTENDEDPRICE]"/>
+ <map key="[LO_REVENUE]" value="[LINEORDER_1].[LO_REVENUE]"/>
+ <map key="[LO_DISCOUNT]" value="[LINEORDER_1].[LO_DISCOUNT]"/>
+ <map key="[LO_SHIPPRIOTITY]" value="[LINEORDER_1].[LO_SHIPPRIOTITY]"/>
+ <map key="[LO_SUPPKEY]" value="[LINEORDER_1].[LO_SUPPKEY]"/>
+ <map key="[LO_TAX]" value="[LINEORDER_1].[LO_TAX]"/>
+ <map key="[LO_ORDERDATE]" value="[LINEORDER_1].[LO_ORDERDATE]"/>
+ <map key="[C_MKTSEGMENT (CUSTOMER_1)]" value="[CUSTOMER_1].[C_MKTSEGMENT]"/>
+ </cols>
+ </connection>
+ <aliases enabled="yes"/>
+ <column caption="LO_ORDERPRIOTITY" datatype="string" name="[LO_ORDERPRIOTITY]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="C_ADDRESS_CUSTOMER_1" datatype="string" name="[C_ADDRESS (CUSTOMER_1)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="C_NAME_CUSTOMER_1" datatype="string" name="[C_NAME (CUSTOMER_1)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="C_REGION_CUSTOMER_1" datatype="string" name="[C_REGION (CUSTOMER_1)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="C_NATION" datatype="string" name="[C_NATION (CUSTOMER)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="C_CUSTKEY" datatype="integer" name="[C_CUSTKEY (CUSTOMER)]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_MKTSEGMENT" datatype="string" name="[C_MKTSEGMENT (CUSTOMER)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="LO_PARTKEY" datatype="integer" name="[LO_PARTKEY]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_CITY" datatype="string" name="[C_CITY (CUSTOMER)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="LO_LINENUMBER" datatype="integer" name="[LO_LINENUMBER]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_REGION" datatype="string" name="[C_REGION (CUSTOMER)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="LO_ORDERKEY" datatype="integer" name="[LO_ORDERKEY]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_PHONE" datatype="string" name="[C_PHONE (CUSTOMER)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="CC_10" datatype="integer" name="[CC_10]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_ADDRESS" datatype="string" name="[C_ADDRESS (CUSTOMER)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="CC_4" datatype="integer" name="[CC_4]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_SHIPMODE" datatype="string" name="[LO_SHIPMODE]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="C_CITY_CUSTOMER_1" datatype="string" name="[C_CITY (CUSTOMER_1)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="CC_6" datatype="integer" name="[CC_6]" role="dimension" type="ordinal"/>
+ <column caption="C_NATION_CUSTOMER_1" datatype="string" name="[C_NATION (CUSTOMER_1)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="CC_5" datatype="integer" name="[CC_5]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_PHONE_CUSTOMER_1" datatype="string" name="[C_PHONE (CUSTOMER_1)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="CC_7" datatype="integer" name="[CC_7]" role="dimension" type="ordinal"/>
+ <column caption="CC_9" datatype="integer" name="[CC_9]" role="dimension" type="ordinal"/>
+ <column caption="LO_QUANTITY" datatype="integer" name="[LO_QUANTITY]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_SUPPLYCOST" datatype="integer" name="[LO_SUPPLYCOST]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_CUSTKEY_CUSTOMER_1" datatype="integer" name="[C_CUSTKEY (CUSTOMER_1)]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_CUSTKEY" datatype="integer" name="[LO_CUSTKEY]" role="dimension" type="ordinal"/>
+ <column caption="LO_ORDTOTALPRICE" datatype="integer" name="[LO_ORDTOTALPRICE]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_NAME_1" datatype="string" name="[C_NAME (CUSTOMER)]" role="dimension" type="nominal"/>
+ <column caption="LO_COMMITDATE" datatype="date" name="[LO_COMMITDATE]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_EXTENDEDPRICE" datatype="integer" name="[LO_EXTENDEDPRICE]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_REVENUE" datatype="integer" name="[LO_REVENUE]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_DISCOUNT" datatype="integer" name="[LO_DISCOUNT]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_SHIPPRIOTITY" datatype="integer" name="[LO_SHIPPRIOTITY]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_SUPPKEY" datatype="integer" name="[LO_SUPPKEY]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_TAX" datatype="integer" name="[LO_TAX]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="LO_ORDERDATE" datatype="date" name="[LO_ORDERDATE]" role="dimension" type="ordinal" hidden="true"/>
+ <column caption="C_MKTSEGMENT_CUSTOMER_1" datatype="string" name="[C_MKTSEGMENT (CUSTOMER_1)]" role="dimension" type="nominal" hidden="true"/>
+ <column caption="COUNT_ALL" datatype="integer" name="[COUNT_ALL]" role="measure" type="quantitative" hidden="true">
+ <calculation class="tableau" formula="COUNT(1)"/>
+ </column>
+ <column caption="" datatype="integer" name="[m_aa]" role="measure" type="quantitative">
+ <calculation class="tableau" formula="SUM([CC_4])"/>
+ </column>
+ <column caption="" datatype="integer" name="[m_bb]" role="measure" type="quantitative" hidden="true">
+ <calculation class="tableau" formula="SUM([CC_5])"/>
+ </column>
+ <column caption="" datatype="integer" name="[m_cc]" role="measure" type="quantitative" hidden="true">
+ <calculation class="tableau" formula="SUM([C_CUSTKEY (CUSTOMER)])"/>
+ </column>
+ <column caption="" datatype="integer" name="[m_dd]" role="measure" type="quantitative" hidden="true">
+ <calculation class="tableau" formula="SUM([CC_10])"/>
+ </column>
+ <drill-paths>
+ <drill-path name="[C_ADDRESS (CUSTOMER)], [C_NATION (CUSTOMER)], [CC_7]">
+ <field>[C_ADDRESS (CUSTOMER)]</field>
+ <field>[C_NATION (CUSTOMER)]</field>
+ <field>[CC_7]</field>
+ </drill-path>
+ </drill-paths>
+ <semantic-values>
+ <semantic-value key="[Country].[Name]" value=""美国""/>
+ </semantic-values>
+</datasource>
+```
+
diff --git a/website/docs/restapi/model_api/model_multilevel_partitioning_api.md b/website/docs/restapi/model_api/model_multilevel_partitioning_api.md
new file mode 100644
index 0000000000..0af956b037
--- /dev/null
+++ b/website/docs/restapi/model_api/model_multilevel_partitioning_api.md
@@ -0,0 +1,366 @@
+---
+title: Multi-level Partitioning Model API
+language: en
+sidebar_label: Multi-level Partitioning Model API
+pagination_label: Multi-level Partitioning Model API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - multi-level partitioning model api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](../authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+### Set partition column {#Set partition column}
+For multi-level partition model define partition column, please refer to [Define Partition Column](model_management_api.md) directly.
+
+### Add Sub-Partition value {#Add Sub-Partition value}
+You can add it by specifying `sub_partition_values` when [Load Segment](model_build_api.md) directly or execute following API.
+
+- `POST http://host:port/kylin/api/models/{model_name}/segments/multi_partition/sub_partition_values`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`,model name.
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`,project name.
+
+ - `sub_partition_values` - `required` `array`,sub-partition values.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ http://host:port/kylin/api/models/multi_level_partition/segments/multi_partition/sub_partition_values \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project":"multi_level_partition",
+ "sub_partition_values":[["5"]]
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+### Load Segment {#Load Segment}
+For building multi-level partition model, please refer directly to [Load Segment API](model_build_api.md).
+
+### Build Sub-Partition {#Build Sub-Partition}
+
+- `POST http://host:port/kylin/api/models/{model_name}/segments/multi_partition`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`,model name.
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`,project name.
+
+ - `segment_id` - `required` `string`,Segment id.
+
+ - `sub_partition_values` - `required` `array`,sub-partition values.
+
+ - `parallel_build_by_segment` - `optional` `boolean`, whether to build concurrently,the default value is `false`.
+
+ - `build_all_sub_partitions` - `optional` `boolean` whether to build all sub-partition values,the default value is `false`.
+
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](../job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ http://host:port/kylin/api/models/multi_partition_model/segments/multi_partition \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project": "multi_partition_project",
+ "segment_id": "983904fa-d573-4944-acb8-558c59598a48",
+ "sub_partition_values": [
+ [
+ "5"
+ ]
+ ],
+ "parallel_build_by_segment": false,
+ "build_all_sub_partitions": false
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "jobs": [
+ {
+ "job_name": "SUB_PARTITION_BUILD",
+ "job_id": "0282e2c9-63e3-46b0-85f6-6e74ca5bf984"
+ }
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+### Delete Sub-Partition value corresponding index data {#Delete Sub-Partition value corresponding index data}
+
+- `DELETE http://host:port/kylin/api/models/segments/multi_partition`
+
+- URL Parameters
+
+ - `project` - `required` `string`,project name.
+
+ - `model` - `required` `string`,model name.
+
+ - `segment_id` - `required` `string`, Segment Id.
+
+ - `sub_partition_values` - `required` `string`, sub-partition values.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://host:port/kylin/api/models/segments/multi_partition?project=multi_partition_project&model=multi_partition_model&segment_id=ff839b0b-2c23-4420-b332-0df70e36c343&sub_partition_values=0,1' \
+ -H 'accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+### Refresh the index data corresponding to the Sub-Partition value {#Refresh the index data corresponding to the Sub-Partition value}
+
+- `PUT http://host:port/kylin/api/models/{model_name}/segments/multi_partition`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`,model name.
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`,project name.
+
+ - `segment_id` - `required` `string`,Segment id.
+
+ - `sub_partition_values` - `required` `array`,sub-partition values.
+
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](../job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ http://host:port/kylin/api/models/SSB_LINEORDER/segments/multi_partition \
+ -H 'accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"default","segment_id":"60615c5e-2ae1-4ee9-b88d-f00f1f101a8a","sub_partition_values":[["1"], ["2"]]}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "jobs": [
+ {
+ "job_name": "SUB_PARTITION_REFRESH",
+ "job_id": "4fa1a0a1-6a97-4be9-bdf6-8957c5f80114"
+ }
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+### Get Segment List {#Get Segment List}
+Please refer to [Get Segment List](../segment_managment_api.md) for multi-level partition model to get Segment list.
+
+### Get Segment details {#Get Segment details}
+- `GET http://host:port/kylin/api/models/{model_name}/segments/multi_partition`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+ - `segment_id` - `required` `string`, Segment Id.
+
+ - `page_offset` - `optional` `int`, pagination page, the default value is 0.
+
+ - `page_size` - `optional` `int`, page size, the default value is 10.
+
+ - `model_name` - `required` `string`, model name.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/models/multi_partition_model/segments/multi_partition?project=multi_partition_project&segment_id=ffdfc037-7c63-4499-986d-9b680276161e' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Authorization: Basic QURNSU46a3lsaW5AMjAxOQ==' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ {
+ "id": 2,
+ "values": [
+ "3"
+ ],
+ "status": "ONLINE",
+ "last_modified_time": 1609213843048,
+ "source_count": 0,
+ "bytes_size": 3747
+ },
+ {
+ "id": 3,
+ "values": [
+ "4"
+ ],
+ "status": "ONLINE",
+ "last_modified_time": 1609213843048,
+ "source_count": 0,
+ "bytes_size": 3747
+ }
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 2
+ },
+ "msg": ""
+ }
+ ```
+
+### Query mapping settings {#Query mapping settings}
+- `PUT http://host:port/kylin/api/models/{model_name}/multi_partition/mapping`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`,model name.
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`,project name.
+
+ - `alias_columns` - `required`,`array<string>`, The corresponding column name that the sub-partition column needs to be mapped to
+
+ - `multi_partition_columns` - `required`,`array<string>`, sub-partition column names
+
+ - `value_mapping` - `required`,`array<object>`, partition value mapping
+
+ - `origin` - `required`,`array<string>`, sub-partition column value
+
+ - `target` - `required`, `array<string>`, sub-partition column value mapping value
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ http://host:port/kylin/api/models/test_multi/multi_partition/mapping \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Authorization: Basic QURNSU46a3lsaW5AMjAxOQ==' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project": "multi_partition_project",
+ "alias_columns": ["KYLIN_SALES.LEAF_CATEG_ID"],
+ "multi_partition_columns": ["KYLIN_SALES.LSTG_SITE_ID"],
+ "value_mapping": [
+ {
+ "origin": ["Beijing"],
+ "target":["North"]
+ },
+ {
+ "origin":["Shanghai"],
+ "target":["South"]
+ }
+ ]
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
diff --git a/website/docs/restapi/project_api.md b/website/docs/restapi/project_api.md
new file mode 100644
index 0000000000..15fc80476b
--- /dev/null
+++ b/website/docs/restapi/project_api.md
@@ -0,0 +1,700 @@
+---
+title: Project Setting API
+language: en
+sidebar_label: Project Setting API
+pagination_label: Project Setting API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - project setting api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+
+## Project Setting API
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+### <span id="create-project">Create a Project </span>
+
+- `POST http://host:port/kylin/api/projects`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `name` - `required` `string`, project name.
+ - `description` - `optional` `string`, project description.
+ - `maintain_model_type` - `required` `string`, project type, `MANUAL_MAINTAIN` for AI augmented mode.
+
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/projects' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "name": "a",
+ "description": "",
+ "maintain_model_type": "MANUAL_MAINTAIN"
+ }'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "uuid":"c7713bea-7df5-49d5-9713-f0094addbafe",
+ "last_modified":1574389912687,
+ "create_time":1574389912687,
+ "version":"4.0.0.0",
+ "mvcc":0,
+ "name":"a",
+ "owner":"ADMIN",
+ "status":"ENABLED",
+ "create_time_utc":1574389912687,
+ "default_database":"DEFAULT",
+ "description":"",
+ "ext_filters":[
+
+ ],
+ "maintain_model_type":"AUTO_MAINTAIN",
+ "override_kylin_properties":{
+
+ },
+ "segment_config":{
+ "auto_merge_enabled":true,
+ "auto_merge_time_ranges":[
+ "WEEK",
+ "MONTH",
+ "QUARTER",
+ "YEAR"
+ ],
+ "volatile_range":{
+ "volatile_range_number":0,
+ "volatile_range_enabled":false,
+ "volatile_range_type":"DAY"
+ },
+ "retention_range":{
+ "retention_range_number":1,
+ "retention_range_enabled":false,
+ "retention_range_type":"MONTH"
+ }
+ }
+ },
+ "msg":""
+ }
+ ```
+
+
+
+### <span id="delete-project">Delete a Project </span>
+
+- `DELETE http://host:port/kylin/api/projects/{project}`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://host:port/kylin/api/projects/b' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+### General Setting {#general-setting}
+
+- `PUT http://host:port/kylin/api/projects/{project}/project_general_info`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `description` - `optional` `string`, project description.
+ - `semi_automatic_mode` - `optional` `boolean`, whether to turn
+ on Recommendation Mode. `True` means to turn on, `false` means to turn off. The default value is `false`.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/project_general_info' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "description":"description",
+ "semi_automatic_mode":true
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+### Storage Quota {#storage-quota}
+
+- `PUT http://host:port/kylin/api/projects/{project}/storage_quota`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+- `storage_quota_size` - `required` `long`, project storage quota, unit byte. Optional values: positive numbers and be equal or greater than `1099511627776` , which means equal or greater than 1 TB.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/storage_quota' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "storage_quota_size": 1099511627776
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+### Low Usage Storage {#inefficient-storage}
+
+- `PUT http://host:port/kylin/api/projects/{project}/garbage_cleanup_config`
+
+- URL Parameters
+ - `project` - `required` `string`, project name.
+
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `frequency_time_window` - `required` `string`, low usage storage calculation period. Optional values: `MONTH`, `WEEK`, `DAY`.
+ - `low_frequency_threshold` - `required` `int`, the number of use of low usage storage. Optional values: positive integer and `0`. For example, you can set when usage is lower than 10 times, then the storage of indices would be regarded as low usage storage.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/garbage_cleanup_config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "frequency_time_window": "WEEK",
+ "low_frequency_threshold": 7
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+
+### <span id="Pushdown">Pushdown Setting</span>
+
+- `PUT http://host:port/kylin/api/projects/{project}/push_down_config`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Content-Type: application/json;charset=utf-8`
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+
+- HTTP Body: JSON Object
+
+ - `push_down_enabled` - `required` `boolean`, whether to turn on query pushdown, `true` for turning on, `false` for turning off.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/push_down_config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "push_down_enabled":true
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+
+### <span id="pushdown-config">Pushdown Configure Setting</span>
+
+- `PUT http://host:port/kylin/api/projects/{project}/push_down_project_config`
+
+- URL Parameters
+
+ - `project` - `required` `string`，project name。
+
+- HTTP Header
+ - `Content-Type: application/json;charset=utf-8`
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+
+- HTTP Body: JSON Object
+
+- `runner_class_name` - `required` `string`，project config property `kylin.query.pushdown.runner-class-name`. To specify the query engine when query pushdown. For default, when pushdown to the native Spark, the value is `org.apache.kylin.query.pushdown.PushDownRunnerSparkImpl`.
+- `converter_class_names` - `required` `string`，project config SQL conversion property `kylin.query.pushdown.converter-class-names`.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/push_down_project_config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project": "project_name",
+ "runner_class_name":"org.apache.kylin.query.pushdown.PushDownRunnerSparkImpl",
+ "converter_class_names":"org.apache.kylin.query.security.HackSelectStarWithColumnACL,org.apache.kylin.query.util.SparkSQLFunctionConverter"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+
+### Segment Settings {#segment-setting}
+
+- `PUT http://host:port/kylin/api/projects/{project}/segment_config`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `auto_merge_enabled` - `required` `boolean`, whether to turn on segment auto-merge, `true` for turning on, `false` for turning off.
+ - `auto_merge_time_ranges` - `required` `array`, the period of segment auto-merge and the unit of retentio threshold. Optional values: `DAY`, `WEEK`, `MONTH`, `QUARTER`, `YEAR`. For example, you can set auto-merge the segments in 1 week.
+ - `volatile_range` - `optional` `json`, the volatile range of auto-merge, which means 'Auto-Merge' will not merge latest [Volatile Range] days segment. For example, If you set 10 days, it means that segments within 10 days will not be merged automatically.
+ - `volatile_range_number` - `optional` `int`，The time of volatile range. The default value is `0`. Optional values: positive integer and `0`.
+ - `volatile_range_type` - `optional` `string`，The time unit of the volatile range. The default value is `Day`. Optional values: `DAY`, `WEEK`, `MONTH`.
+ - `retention_range` - `optional` `json`, retention threshold, which means to retain the segments in the retention threshold. If you set 1 year, that means the segments exceed the 1 year will be removed by the system automatically.
+ - `retention_range_enabled` - `optional` `boolean`, whether to turn on the retention threshold, `true` for turning on, `false` for turning off, and the default value is `false`.
+ - `retention_range_number` - `optional` `int`，The time to retention threshold. The default value is `1`. Optional values: positive integer and `0`.
+ - `create_empty_segment_enabled` - `optional` `boolean`, whether to allow add new segment. `true` for turning on, `false` for turning off, and the default value is `false`.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/segment_config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "auto_merge_time_ranges":[
+ "WEEK",
+ "MONTH",
+ "QUARTER",
+ "DAY",
+ "YEAR"
+ ],
+ "auto_merge_enabled":true,
+ "volatile_range":{
+ "volatile_range_number":0,
+ "volatile_range_type":"DAY"
+ },
+ "retention_range":{
+ "retention_range_number":2,
+ "retention_range_enabled":false
+ }
+ }'
+ ```
+ > **Notes：**
+ >
+ > - `auto_merge_time_ranges` : array contains at least one value.
+ > - The time unit of `retention_range` is equal to the maximum time unit in the `auto_merge_time_ranges` array. If `"auto_merge_time_ranges":["DAY",MONTH"]`, then the time unit of `retention_range` is `MONTH`. If `"auto_merge_time_ranges":["YEAR",WEEK"]`, then the time unit of `retention_range` is `YEAR`.
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+
+### Default Database {#default-database}
+
+- `PUT http://host:port/kylin/api/projects/{project}/default_database`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+- `default_database` - `required` `string`, default database name.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/default_database' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "default_database":"EDW"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+
+### Job Notification {#job-notification}
+
+- `PUT http://host:port/kylin/api/projects/{project}/job_notification_config`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `data_load_empty_notification_enabled` - `required` `boolean`, whether to turn on empty data load notification. `True` means to notify user if there is an empty data load job.
+ - `job_error_notification_enabled` - `required` `boolean`, whether to turn on error job notification. `True` means to notify user if there is an error job.
+ - `job_notification_emails` - `required` `array`，email address for the job notification. Email format: `xx@xx.xx`.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/job_notification_config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "data_load_empty_notification_enabled":true,
+ "job_error_notification_enabled":false,
+ "job_notification_emails":[
+ "nnnn@kyligence.io","tttt@kyligence.io"
+ ]
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":null,
+ "msg":""
+ }
+ ```
+
+### <span id="yarn">YARN Application Queue</span>
+
+- `PUT http://host:port/kylin/api/projects/{project}/yarn_queue`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - `queue_name` - `required` `string`, the name of YARN application queue.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/b/yarn_queue' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "queue_name":"yarnqueue"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": null,
+ "msg": ""
+ }
+ ```
+
+
+
+### <span id="update-job-engine">Update the Linking Relationship between Projects and Job Engines</span>
+
+**Note:** This Rest API need to cooperate with multi-active job engines. For more details, please refer to [Kylin Multi-Active Job Engines](../../../installation/deploy/cluster_lb.en.md).
+
+- `POST http://host:port/kylin/api/epoch`
+
+- HTTP Body: JSON Object
+
+ - `projects` - `required` `string`, projects name. If the array is empty, it will update the relationship for all projects
+ - `force` - `required` `boolean`, whether to update the relationship directly. For `false`, it will check whether the relationship is outdated. If it is outdated, the system will update it based on current projects and job engines. For `true`, the system will skip the check step and update directly.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/epoch' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"projects":["Project_A"],"force":true}'
+ ```
+
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+ ```
+
+### Project Reset Setting {#project-reset-setting}
+
+The Low Usage Storage, Segment Settings, Storage Quota and Job Notification can be reset to the default values. Only one of the settings can be reset each time.
+
+- `PUT http://host:port/kylin/api/projects/{project}/project_config`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - `reset_item` - `required` `string`, reset project settings item. Optional values: `job_notification_config`, `query_accelerate_threshold`, `garbage_cleanup_config`, `segment_config`, `storage_quota_config`.
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/projects/a/project_config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "reset_item":"job_notification_config"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "project":"a",
+ "description":"",
+ "maintain_model_type":"MANUAL_MAINTAIN",
+ "default_database":"EDW",
+ "semi_automatic_mode":false,
+ "storage_quota_size":10995116277760,
+ "push_down_enabled":true,
+ "auto_merge_enabled":true,
+ "auto_merge_time_ranges":[
+ "WEEK",
+ "MONTH",
+ "QUARTER",
+ "YEAR"
+ ],
+ "volatile_range":{
+ "volatile_range_number":0,
+ "volatile_range_enabled":true,
+ "volatile_range_type":"DAY"
+ },
+ "retention_range":{
+ "retention_range_number":1,
+ "retention_range_enabled":false,
+ "retention_range_type":"YEAR"
+ },
+ "job_error_notification_enabled":false,
+ "data_load_empty_notification_enabled":false,
+ "job_notification_emails":[
+
+ ],
+ "threshold":20,
+ "tips_enabled":true,
+ "frequency_time_window":"MONTH",
+ "low_frequency_threshold":5
+ },
+ "msg":""
+ }
+ ```
+
+- The default values for Low Usage Strage:
+ - `"frequency_time_window":"MONTH"`
+ - `"low_frequency_threshold":0`
+- The default values for Segment Settings:
+ - `"auto_merge_enabled":true`
+ - `"auto_merge_time_ranges":["DAY", "MONTH", "QUARTER", "YEAR"]`
+ - `"volatile_range"`
+ - `"volatile_range_number":0`
+ - `"volatile_range_type":"DAY"`
+ - `retention_range`
+ - `"retention_range_enabled":false`
+ - `"retention_range_number":1`
+- `"create_empty_segment_enabled":false`
+
+- The default values for Job Notification:
+ - `"data_load_empty_notification_enabled":false`
+ - `"job_error_notification_enabled":false`
+ - `"job_notification_emails"` is null
+
+- The default values for Storage Quota
+
+ - `"kylin.storage.quota-in-giga-bytes": 10240`
diff --git a/website/docs/restapi/query_api.md b/website/docs/restapi/query_api.md
new file mode 100755
index 0000000000..44813e2c18
--- /dev/null
+++ b/website/docs/restapi/query_api.md
@@ -0,0 +1,543 @@
+---
+title: Query API
+language: en
+sidebar_label: Query API
+pagination_label: Query API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - query api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+### Query API {#Query-api}
+
+- `POST http://host:port/kylin/api/query`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+ - `sql` - `required` `string`, SQL statement
+ - `project` - `required` `string`, project name
+ - `offset` - `optional` `int`, offset of query result. Must be used in conjunction with `limit`.
+ - `limit` - `optional` `int`, limit on the quantity of returned query result
+ - `forcedToPushDown` - `optional` `boolean`, whether to force queries to pushdown engine, `false` by default. You are not able to force queries to pushdown when the pushdown setting is turned off.
+ - `partialMatchIndex` - `optional` `boolean`, `false` by default, whether to force queries the segments with proper index, when not all segment can answer the query
+ - `forced_to_index` - `optional` `boolean`, whether to force the query to use index, default to `false`. When set to `true`, the query will return the result as normal if a matching index is found, otherwise, an error will be thrown and the query will not be pushded down.
+ - `forcedToTieredStorage` - `optional` `int`, whether the query is forced to use tiered storage, the default value is `0`, which means that when the tiered storage cannot answer, it will be answered by the base table index on HDFS, configured as `1` indicates that when the tiered storage cannot answer the query, the query will be pushdown, configured as `2`, indicates that the query fails when the tiered storage cannot answer the query. When `forcedToPushDown` is `true`, this parameter [...]
+ - Parameters `forcedToPushDown` and `forced_to_index` cannot be `true` at the same time, an error will be reported.
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://host:port/kylin/api/query' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{ "sql":"select count(*) from SSB.P_LINEORDER", "project":"ssb", "partialMatchIndex":"true"}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "columnMetas":[
+ {
+ "isNullable":0,
+ "displaySize":19,
+ "label":"EXPR$0",
+ "name":"EXPR$0",
+ "schemaName":null,
+ "catelogName":null,
+ "tableName":null,
+ "precision":19,
+ "scale":0,
+ "columnType":-5,
+ "columnTypeName":"BIGINT NOT NULL",
+ "caseSensitive":false,
+ "autoIncrement":false,
+ "currency":false,
+ "definitelyWritable":false,
+ "signed":true,
+ "writable":false,
+ "searchable":false,
+ "readOnly":false
+ }
+ ],
+ "results":[
+ [
+ "60175"
+ ]
+ ],
+ "affectedRowCount":0,
+ "exceptionMessage":null,
+ "duration":129,
+ "scanRows":[
+ 2519
+ ],
+ "totalScanRows":2519,
+ "scanBytes":[
+ 6118
+ ],
+ "totalScanBytes":6118,
+ "resultRowCount":1,
+ "shufflePartitions":1,
+ "hitExceptionCache":false,
+ "storageCacheUsed":false,
+ "queryStatistics":null,
+ "traceUrl":null,
+ "queryId":"738fad53-57a9-43fc-a186-b25d0993dfcb",
+ "server":"client134.kcluster:7470",
+ "suite":null,
+ "signature":"1593670148521;1592230756490_1595228217256_1595228217256_1595228217256_1595228217257",
+ "engineType":"NATIVE",
+ "exception":false,
+ "prepare":false,
+ "timeout":false,
+ "partial":false,
+ "isException":false,
+ "appMasterURL":"/kylin/sparder/SQL/execution/?id=27391",
+ "pushDown":false,
+ "is_prepare":false,
+ "is_timeout":false,
+ "realizations":[
+ {
+ "modelId":"c7a7a1d0-71c6-4d42-bbc3-76167e5d2d10",
+ "modelAlias":"ssb_cube",
+ "layoutId":80001,
+ "indexType":"Agg Index",
+ "valid":true,
+ "partialMatchModel":false
+ }
+ ],
+ "traces" : [
+ {
+ "name": "GET_ACL_INFO",
+ "group": "PREPARATION",
+ "duration": 10
+ },
+ {
+ "name" : "SQL_TRANSFORMATION",
+ "group" : "PREPARATION",
+ "duration" : 3
+ },
+ {
+ "name" : "SQL_PARSE_AND_OPTIMIZE",
+ "group" : "PREPARATION",
+ "duration" : 9
+ },
+ {
+ "name" : "MODEL_MATCHING",
+ "group" : "PREPARATION",
+ "duration" : 2
+ },
+ {
+ "name" : "PREPARE_AND_SUBMIT_JOB",
+ "group" : null,
+ "duration" : 64
+ },
+ {
+ "name" : "WAIT_FOR_EXECUTION",
+ "group" : null,
+ "duration" : 2
+ },
+ {
+ "name" : "EXECUTION",
+ "group" : null,
+ "duration" : 26
+ },
+ {
+ "name" : "FETCH_RESULT",
+ "group" : null,
+ "duration" : 7
+ }]
+ },
+ "msg":""
+ }
+ ```
+
+- Response Information
+ - `columnMetas` - metadata information of the columns
+ - `results` - query results
+ - `resultRowCount` - row count of query results
+ - `isException` - whether the query returns exception
+ - `exceptionMessage` - exception message
+ - `queryId` - query ID
+ - `duration` - query duration
+ - `totalScanRows` - total scan count
+ - `totalScanBytes` - total scan bytes
+ - `hitExceptionCache` - whether hit the result cache of an exception query
+ - `storageCacheUsed` - whether hit the result cache of a success query
+ - `server` - which server executed this query
+ - `timeout` - whether query is timeout
+ - `pushDown` - whether query push down to other engine
+ - `traces` - the trace information for each query execution stage
+ - `name` - the stage name
+ - `duration` - duration in milliseconds
+ - `group` - the stage group
+
+### Refresh cached data {#Refresh-cached-data}
+
+Calling this API will refresh the table cache of the Spark SQL Context for all query nodes. Application scenario: Query pushdown fails after the update of data source table.
+
+- `PUT http://host:port/kylin/api/tables/catalog_cache`
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- HTTP Body: JSON Object
+
+ - `tables` - `required` `string`,specify the table you want to load, in the format: DB.TABLE, separate multiple tables with commas
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://host:port/kylin/api/tables/catalog_cache' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"tables":["SSB.LINEORDER","SSB.SUPPLIER"]}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "nodes": [
+ {
+ "server": "slave104.tnt:18001",
+ "refreshed": [
+ "SSB.LINEORDER",
+ "SSB.SUPPLIER"
+ ],
+ "failed": []
+ },
+ {
+ "server": "slave104.tnt:18003",
+ "refreshed": [
+ "SSB.LINEORDER",
+ "SSB.SUPPLIER"
+ ],
+ "failed": []
+ }
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+- Response Information
+
+ - `nodes` - Refresh results of different nodes
+ - `server` - Node information
+ - `refreshed` - Table that refreshed successfully
+ - `failed` - Table that failed to refresh
+ - `msg` - Reasons for refresh failure
+
+
+### Get query histories {#Get-query-histories}
+- `GET http://host:port/kylin/api/query/query_histories`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- URL Parameters
+ - `project` - `required` `string`，project name
+ - `page_offset` - `optional` `int`, offset of returned result, 0 by default
+ - `page_size` - `optional` `int`, quantity of returned result per page, 10 by default
+ - `start_time_from` - `optional` `string`，timestamp of query history start, cannot be used alone with start_time_to
+ - `start_time_to` - `optional` `string`，timestamp of query history end, cannot be used alone with start_time_from
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/query/query_histories?project=kylin_demo&page_offset=5&page_size=1&start_time_from=1656864000000&start_time_to=1656950400000' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+```json
+{
+ "code": "000",
+ "data": {
+ "size": 79,
+ "query_histories": [
+ {
+ "queryRealizations": null,
+ "query_id": "4bfbc8f1-ffcb-ae86-c5f1-2b78c73cb802",
+ "query_history_info": {
+ "exactly_match": true,
+ "scan_segment_num": 1,
+ "state": "FAILED",
+ "execution_error": false,
+ "error_msg": null,
+ "query_snapshots": [],
+ "realization_metrics": [
+ {
+ "queryId": "4bfbc8f1-ffcb-ae86-c5f1-2b78c73cb802",
+ "duration": 568,
+ "layoutId": "1",
+ "indexType": "Agg Index",
+ "modelId": "118ae12b-5198-ade2-ecd9-b7e5f64318a2",
+ "queryTime": 1656914360898,
+ "projectName": "kylin_demo",
+ "snapshots": [],
+ "secondStorage": false,
+ "streamingLayout": false
+ },
+ {
+ "queryId": "4bfbc8f1-ffcb-ae86-c5f1-2b78c73cb802",
+ "duration": 568,
+ "layoutId": "1",
+ "indexType": "Agg Index",
+ "modelId": "6cf4a660-e217-add5-87f4-04493c8df21e",
+ "queryTime": 1656914360898,
+ "projectName": "kylin_demo",
+ "snapshots": [],
+ "secondStorage": false,
+ "streamingLayout": false
+ }
+ ],
+ "traces": [
+ {
+ "name": "HTTP_RECEPTION",
+ "group": null,
+ "duration": 29
+ },
+ {
+ "name": "GET_ACL_INFO",
+ "group": "PREPARATION",
+ "duration": 1
+ },
+ {
+ "name": "SQL_TRANSFORMATION",
+ "group": "PREPARATION",
+ "duration": 12
+ },
+ {
+ "name": "SQL_PARSE_AND_OPTIMIZE",
+ "group": "PREPARATION",
+ "duration": 125
+ },
+ {
+ "name": "MODEL_MATCHING",
+ "group": "PREPARATION",
+ "duration": 10
+ },
+ {
+ "name": "PREPARE_AND_SUBMIT_JOB",
+ "group": "JOB_EXECUTION",
+ "duration": 277
+ },
+ {
+ "name": "WAIT_FOR_EXECUTION",
+ "group": "JOB_EXECUTION",
+ "duration": 13
+ },
+ {
+ "name": "EXECUTION",
+ "group": "JOB_EXECUTION",
+ "duration": 90
+ },
+ {
+ "name": "FETCH_RESULT",
+ "group": "JOB_EXECUTION",
+ "duration": 11
+ }
+ ],
+ "cache_type": null,
+ "query_msg": null
+ },
+ "sql_text": "SELECT *\nFROM\n (SELECT PICKUP_DATE,\n TAXI_ORDER_NUMBER,\n PEOPLE_POSITIVE_NEW_CASES_COUNT,\n round(TAXI_ORDER_NUMBER / PEOPLE_POSITIVE_NEW_CASES_COUNT, 2) AS COVID_IMPACT_INDEX\n FROM\n (SELECT PICKUP_DATE,\n MONTH_START,\n ZONE,\n TAXI_ORDER_NUMBER\n FROM\n (SELECT PICKUP_DATE,\n MONTH_START,\n ZONE,\n sum(TOTAL_AMOUNT) AS TAXI_PR [...]
+ "query_time": 1656914360898,
+ "duration": 568,
+ "server": "snoopy-gw07.kylin.com:7095",
+ "submitter": "ADMIN",
+ "index_hit": true,
+ "query_status": "SUCCEEDED",
+ "result_row_count": 366,
+ "id": 296,
+ "engine_type": "NATIVE",
+ "total_scan_count": 122839,
+ "project_name": "kylin_demo",
+ "realizations": [
+ {
+ "modelId": "118ae12b-5198-ade2-ecd9-b7e5f64318a2",
+ "modelAlias": "AUTO_MODEL_TAXI_TRIP_RECORDS_VIEW_1",
+ "layoutId": 1,
+ "indexType": "Agg Index",
+ "snapshots": [],
+ "valid": true,
+ "partialMatchModel": false
+ },
+ {
+ "modelId": "6cf4a660-e217-add5-87f4-04493c8df21e",
+ "modelAlias": "AUTO_MODEL_COVID_19_ACTIVITY_1",
+ "layoutId": 1,
+ "indexType": "Agg Index",
+ "snapshots": [],
+ "valid": true,
+ "partialMatchModel": false
+ }
+ ],
+ "total_scan_bytes": 849962,
+ "error_type": null,
+ "cache_hit": false
+ }
+ ]
+ },
+ "msg": ""
+}
+```
+
+- Response Information
+ - `query_id` - query ID
+ - `query_history_info` - Query history infomation
+ - `exactly_match` - whether the query returns exception
+ - `scan_segment_num` - scan segments
+ - `state` - the flag whether the query satisfied query recommendation
+ - `execution_error` - whether the query failed
+ - `error_msg` - error message
+ - `query_snapshots` - query snapshots
+ - `realization_metrics`
+ - `queryId` - query ID
+ - `duration` - query duration(ms)
+ - `layoutId` - index ID
+ - `indexType` - index type
+ - `modelId` - model ID
+ - `queryTime` - timestamp of when query executed
+ - `projectName` - project name
+ - `snapshots` - snapshots
+ - `secondStorage` - whether Tied Storage was used
+ - `streamingLayout` - whether hit streaming index
+ - `traces`
+ - `name` - the stage name
+ - `duration` - duration in milliseconds
+ - `group` - the stage group
+ - `cache_type` - cache type
+ - `query_msg` - exception message
+ - `sql_text` - SQL
+ - `query_time` - timestamp of when query executed
+ - `duration` - duration in milliseconds
+ - `server` - which server executed this query
+ - `submitter` - username
+ - `index_hit` - whether the query hit index
+ - `query_status` - query status
+ - `result_row_count` - row count of query results
+ - `engine_type` - engine type
+ - `total_scan_count` - total scan counts
+ - `project_name` - project name
+ - `realizations`
+ - `modelId` - model ID
+ - `modelAlias` - model name
+ - `layoutId` - index ID
+ - `indexType` - index type
+ - `snapshots` - snapshots
+ - `valid` - whether the model is valid
+ - `partialMatchModel` - whether using partial match
+ - `total_scan_bytes` - total scan bytes
+ - `error_type` - error type
+ - `cache_hit` - whether hit cache
+
+
+
+### Download query history SQL {#Download-query-history-SQL}
+
+- `GET http://host:port/kylin/api/query/download_query_histories`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name.
+ - `timezone_offset_hour` - `required` `int`, query the time zone offset of history, the number of hours different from GMT, for example, East 8 is passed in 8, note that only the integer between [-18,18] can be taken.
+ - `start_time_from` - `optional` `string`, query history start time timestamp, units ms, it work only when used together with start_time_to. For example, 1617206400000, if you enter a number other than a number, it will return empty.
+ - `start_time_to` - `optional` `string`, query the end of history timestamp, units ms, it work only when used together with start_time_from. For example, 1620662400000, if you enter a number other than a number, it will return to null.
+ - `latency_from` - `optional` `string`, query delay is greater than latency_from, units s, it work only when used together with latency_to. For example, 10, if you enter a number other than a number, it will return to null.
+ - `latency_to` - `optional` `string`, query delay is less than latency_to, units s, it work only when used together with latency_from. For example, 20, if you enter a number other than a number, it will return to null.
+ - `query_status` - `optional` `List<String>` query status, such as SUCCEEDED、FAILED. If you enter values other than these, it will return empty.
+ - `sql` - `optional` `string`, used to fuzzy match user SQL or query ID.
+ - `realization` - `optional` `List<string>` to query object.
+ - `server` - `optional` `string`, the hostname and port of query node, e.g. myhost:7072 .
+ - `submitter` - `optional` `List<string>` to query user.
+
+- Example of curl request
+
+ ```
+ curl -X GET \
+ ' http://host:port/kylin/api/query/download_ query_ histories?timezone_ offset_ hour=8&realization=&query_ status=&submitter=&project=default&start_ time_ from=&start_ time_ to=&latency_ from=&latency_ to=&sql=' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -o query_history.csv
+ ```
+
+
+### Download query history SQL {#Download-query-history-SQL}
+
+- `GET http://host:port/kylin/api/query/download_query_histories_sql`
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- URL Parameters
+
+ - `project` - `required` `string`, project name
+ - `start_time_from` - `optional` `string`, query history start time timestamp, units ms, it work only when used together with start_time_to. For example, 1617206400000, if you enter a number other than a number, it will return empty.
+ - `start_time_to` - `optional` `string`, query the end of history timestamp, units ms, it work only when used together with start_time_from. For example, 1620662400000, if you enter a number other than a number, it will return to null.
+ - `latency_from` - `optional` `string`, query delay is greater than latency_from, units s, it work only when used together with latency_to. For example, 10, if you enter a number other than a number, it will return to null.
+ - `latency_to` - `optional` `string`, query delay is less than latency_to, units s, it work only when used together with latency_from. For example, 20, if you enter a number other than a number, it will return to null.
+ - `query_status` - `optional` `List<string>` query status, such as succeeded and failed. If you enter values other than these, it will return null.
+ - `sql` - `optional` `string`, used to fuzzy match user SQL or query ID.
+ - `realization` - `optional` `List<string>` to query the object.
+ - `server` - `optional` `string`, query node.
+ - `submitter` - `optional` `List<string>` to query the user.
+
+- Example of curl request
+
+ ```
+ curl -X GET \
+ ' http://host:port/kylin/api/query/download_ query_ histories_ sql?timezone_ offset_ hour=8&realization=&query_ status=&submitter=&project=default&start_ time_ from=&start_ time_ to=&latency_ from=&latency_ to=&sql=' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -o query_history.sql
+ ```
diff --git a/website/docs/restapi/segment_managment_api.md b/website/docs/restapi/segment_managment_api.md
new file mode 100644
index 0000000000..8d3c0ea037
--- /dev/null
+++ b/website/docs/restapi/segment_managment_api.md
@@ -0,0 +1,393 @@
+---
+title: Segment Management API
+language: en
+sidebar_label: Segment Management API
+pagination_label: Segment Management API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - segment management api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+
+### Get Segment List {#Get-Segment-List}
+
+> Call this API to acquire the Segment list of certain model. After calling the "Replenish Indexes among Segments" API, use this API to confirm if segments are successfully built.
+
+- `GET http://host:port/kylin/api/models/{model_name}/segments`
+
+- URL Parameters
+ - `model_name` - `required` `string`, model name.
+ - `project` - `required` `string`, project name.
+ - `page_offset` - `optional` `int`, offset of returned result, `0` by default.
+ - `page_size` - `optional` `int`, quantity of returned result per page, `10` by default.
+ - `start` - `optional` `string`, start time of segments, `1` by default, type: timestamp, unit: ms.
+ - `end` - `optional` `string` , end time of segments, `9223372036854775806` by default, type: timestamp, unit: ms.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://localhost:7070/kylin/api/models/SSB_LINEORDER/segments?project=doc_expert' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ {
+ "id": "99c547b7-1bc1-4b57-b62a-127c080e1fd2",
+ "name": "20120101000000_20130101000000",
+ "create_time_utc": 1609209524110,
+ "status": "READY",
+ "segRange": {
+ "@class": "org.apache.kylin.metadata.model.SegmentRange$TimePartitionedSegmentRange",
+ "date_range_start": 1325347200000,
+ "date_range_end": 1356969600000
+ },
+ "timeRange": null,
+ "parameters": null,
+ "dictionaries": null,
+ "snapshots": null,
+ "last_build_time": 1609209642839,
+ "source_count": 2,
+ "source_bytes_size": 798009,
+ "column_source_bytes": {},
+ "ori_snapshot_size": {},
+ "additionalInfo": {
+ "segment_path": "hdfs://nameservice1/kylin/jrc_kylin/multi_partition/parquet/d5b94380-f84f-481f-b4bb-ffa3f6b3b391/99c547b7-1bc1-4b57-b62a-127c080e1fd2",
+ "file_count": "14"
+ },
+ "is_encoding_data_skew": false,
+ "is_snapshot_ready": false,
+ "is_dict_ready": false,
+ "is_flat_table_ready": false,
+ "is_fact_view_ready": false,
+ "multi_partitions": [],
+ "max_bucket_id": 13,
+ "bytes_size": 12520,
+ "hit_count": 0,
+ "status_to_display": "ONLINE",
+ "index_count": 7,
+ "index_count_total": 7,
+ "multi_partition_count": 2,
+ "multi_partition_count_total": 2,
+ "row_count": 13,
+ "second_storage_nodes":[
+ {
+ "name":"sandbox.hortonworks.com",
+ "ip":"10.1.2.55",
+ "port":9500
+ }
+ ],
+ "second_storage_size":12989,
+ "has_base_table_index":true,
+ "has_base_agg_index":true,
+ "has_base_table_index_data":true,
+ "last_modified_time": 1609209642839
+ }
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 1
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+
+### Refresh/Merge Segments{#Refresh-Segments-Expert}
+
+> Call this API to refresh segments or merge consecutive segments. Use this API to refresh Segment when there are index changes, or to merge multiple consecutive small segments to control file number and improve query performance.
+
+- `PUT http://host:port/kylin/api/models/{model_name}/segments`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name.
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `type` - `optional` `string`, refresh segments or merge continuous segments, the optional value should be either `REFRESH` or ` MERGE` and the default value is `REFRESH`.
+ - `ids` - `optional` `array[string]`, segment id list.
+ - `names` - `optional` `array[string]`, segment name list.
+ > **Notice:** For `ids` and `names`, one of them must be set, and both cannot be set at the same time or both are empty at the same time.
+ >
+ > - `priority` - `optional` `int`, set job priority with range `0-4` which indicates the priority from high to low. Default value is `3`
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+- `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X PUT \
+ 'http://localhost:7070/kylin/api/models/SSB_LINEORDER/segments' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"doc_expert", "ids":["8f3b4040-aa75-4e74-9730-6bf1cae61745"], "type":"REFRESH"}'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
+
+
+
+### Delete Segments{#Delete-Segments-Expert}
+
+- `DELETE http://host:port/kylin/api/models/{model_name}/segments`
+
+- URL Parameters
+ - `model_name` - `required` `string`, model name.
+ - `project` - `required` `string`, project name.
+ - `purge` - `required` `boolean`, whether purge all segments or not.
+ - `ids` - `optional` `array[string]`, segment id list.
+ - `names` - `optional` `array[string]`, segment name list.
+ - `force` - `optional` `boolean`, whether force to delete, default value is "false".
+> **Reminders**：
+>
+> 1.if `purge` is `false`, for `ids` and `names`, one of them must be set, and both cannot be set at the same time or both are empty at the same time.
+>
+> 2.Because the http protocol has a limit on the size of the request header, it is recommended that the count of the `ids` or `names` is less than 100.
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X DELETE \
+ 'http://localhost:7070/kylin/api/models/SSB_LINEORDER/segments?project=doc_expert&ids=291b9926-eaba-42d1-9d70-0a587992bea7&purge=false' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
+
+
+
+### Check Segment {#Check_Segment}
+
+> Verify whether there are overlapping segments within the model
+>
+
+- `POST http://host:port/kylin/api/models/{model_name}/segments/check`
+
+- Request Permission: Operation permission and above
+
+- Effective Versions: 4.1.1 and above
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name.
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `start` - `required` `string`, start time, type: timestamp, unit: ms.
+ - `end` - `required` `string`, end time, type: timestamp, unit: ms.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/ssb_model/segments/check' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project":"ssb",
+ "start":"775785600000",
+ "end":"775789200000"
+ }'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "segments_overlap": [
+ {
+ "segment_id": "17df7def-f06b-4b67-81d8-4c8368e714dc",
+ "segment_name": "19920101000000_19980802000000"
+ },
+ {
+ "segment_id": "1a9c070b-3847-48c6-b938-7109379eef9b",
+ "segment_name": "19980802000000_19980803000000"
+ },
+ {
+ "segment_id": "5cece637-daef-47f5-88e9-85b6a247d357",
+ "segment_name": "19980803000000_19980804000000"
+ }
+ ]
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Replenish Indexes of All Segments {#Replenish-Indexes-All-Segments}
+
+> Note：Specify the model, build all the missing indexes for all Segments of the model
+
+- `POST http://host:port/kylin/api/models/{model_name}/indexes`
+
+- URL Parameters
+
+ - `model_name` - `required` `string`, model name.
+
+- HTTP Body: JSON Object
+
+ - `project` - `required` `string`, project name.
+ - `priority` - `optional` `int`, set job priority with range `0-4` which indicates the priority from high to low. Default value is `3`
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+- HTTP Header
+
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/ssb_test/indexes' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"ssb"}'
+ ```
+
+- Response Details
+
+ - `type`, job type
+ - NORM_BUILD, Build normally
+ - NO_LAYOUT, All indexes to be completed are not online
+ - NO_SEGMENT, All segments to be completed are not online
+ - `job_id`, the job id will be returned if the type is NORM_BUILD. Otherwise this value will be empty if there is no index or segment.
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":{
+ "type":"NORM_BUILD",
+ "job_id":"e3aa809b-5e73-42a5-a1e1-649d53b16e2c"
+ },
+ "msg":""
+ }
+ ```
+
+
+
+### Replenish Indexes among Segments{#Replenish-Indexes-among-Segments}
+
+- `POST http://host:port/kylin/api/models/{model_name}/segments/completion`
+- URL Parameters
+ - `model_name` - `required` `string`, model name
+ - `project` - `required` `string`, project name
+ - `parallel` - `optional` `boolean`, whether build with parallel tasks, by default it is false
+ - `ids` - `optional` `array[string]`, segment id list
+ - `names` - `optional` `array[string]`, segment name list
+ > **Notice:** For `ids` and `names`, one of them must be set, and both cannot be set at the same time or both are empty at the same time.
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/models/m1/segments/completion?project=ssb&names=19900101000000_19950101000000,19950101000000_19970101000000' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "jobs": [
+ {
+ "job_name": "INDEX_BUILD",
+ "job_id": "74e28420-e317-42a9-a221-a7b381b5aeea"
+ }
+ ],
+ "failed_segments": []
+ },
+ "msg": ""
+ }
+ ```
diff --git a/website/docs/restapi/snapshot_management_api.md b/website/docs/restapi/snapshot_management_api.md
new file mode 100644
index 0000000000..a20fb44753
--- /dev/null
+++ b/website/docs/restapi/snapshot_management_api.md
@@ -0,0 +1,304 @@
+---
+title: Snapshot Management API
+language: en
+sidebar_label: Snapshot Management API
+pagination_label: Snapshot Management API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - snapshot management api
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+> Note:
+>
+> 1. Before reading, please finish the [Access and Authentication REST API](authentication.md) chapter to understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+### <span id="switch">Snapshot Management</span>
+
+> **Note**: When the Snapshot Management API is enabled, the system will no longer build or refresh snapshots automatically, please manually manage snapshots according to the snapshot list.
+
+- `PUT http://host:port/kylin/api/projects/{project}/snapshot_config`
+- Request Permission: ADMIN permission and above
+- Introduced in: 4.2.2
+- URL Parameters
+ - `project` - `required` `string`, project name.
+- HTTP Body: JSON Object
+ - `snapshot_manual_management_enabled` - `optional` `boolean`, whether to enable snapshot management. The default value is false.
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+- Curl Request Example
+```sh
+curl -X PUT \
+ 'http://localhost:7070/kylin/api/projects/gc_test/snapshot_config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{ "snapshot_manual_management_enabled": true }'
+```
+
+
+- Response
+```json
+{
+ "code": "000",
+ "data": "",
+ "msg": ""
+}
+```
+
+### <span id="create">Create Snapshots</span>
+
+- `POST http://host:port/kylin/api/snapshots`
+- Request Permission: OPERATION permission and above
+- Introduced in: 4.2.2 (Partition building: since 4.2.6)
+- HTTP Body: JSON Object
+ - `project` - `required` `string`, project name.
+ - `tables` - `optional` `array[string]`, load tables with the format `DB.TABLE`
+ - `databases` - `optional` `array[string]`, load all the tables under this database
+ **Note:** The above two parameters `databases` and `tables` cannot be empty at the same time, which means you must use one of them to load tables.
+ - `priority` - `optional` `integer`, set job priority with range `0-4` which indicates the priority from high to low. Default value is `3`
+ - `options` - `optional` `map[string:args]`, mapping from table (`DB.TABLE`) to argument set, `args` can be set as follows：
+ - `partition_col` - `optional` `string` partition column of responding table, null by default.
+ - `partitions_to_build` - `optional` `string` only refresh the special partitions
+ - `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. The maximum size of value is 1 KB.
+ **Note**: If the loaded table already exists in the system, it will be reloaded.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+- Curl Request Example
+```sh
+curl -X POST \
+ 'http://localhost:7070/kylin/api/snapshots' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"gc_test", "tables": ["SSB.P_LINEORDER", "DEFAULT.TEST_KYLIN_FACT"], "options":{"DEFAULT.TEST_KYLIN_FACT":{"partition_col":"CAL_DT","incremental_build":"true","partitions_to_build":["2012-03-01","2012-03-04"]}}}'
+```
+- Curl Response Example
+```json
+{
+ "code": "000",
+ "data": {
+ "jobs": [{
+ "job_name": "SNAPSHOT_BUILD",
+ "job_id": "65b3b0a4-d4d2-4a5b-af29-b190ca420543"
+ }, {
+ "job_name": "SNAPSHOT_BUILD",
+ "job_id": "24aafb93-1cde-43d1-a627-8cd592f51cfe"
+ }]
+ },
+ "msg": ""
+}
+```
+
+
+### <span id="config">Set Partition Column</span>
+
+
+- `POST http://host:port/kylin/api/snapshots/config`
+- Request Permission: OPERATION permission and above
+- Introduced in: 4.2.6
+- HTTP Body: JSON Object
+ - `project` - `required` `string`, project name.
+ - `table_partition_col` - `required` `map[string:string]` The mapping from table (DB.TABLE) to chosen partition column.
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+- Curl Request Example
+```sh
+ curl -X POST \
+ 'http://localhost:7070/kylin/api/snapshots/config' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"gc_test","table_partition_col":{"DEFAULT.TEST_KYLIN_FACT":"CAL_DT"}}'
+```
+
+
+- Response
+```json
+ {
+ "code": "000",
+ "data": "",
+ "msg": ""
+ }
+```
+
+### <span id="update">Refresh Snapshots</span>
+
+- `PUT http://host:port/kylin/api/snapshots`
+
+- Request Permission: OPERATION permission and above
+
+- Introduced in: 4.2.2 (Partition building: since 4.2.6)
+
+- HTTP Body: JSON Object
+ - `project` - `required` `string`, project name.
+
+ - `tables` - `optional` `array[string]`, load tables with the format `DB.TABLE`
+
+ - `databases` - `optional` `array[string]`, load all the tables under this database
+
+ **Note:** The above two parameters `databases` and `tables` cannot be empty at the same time, which means you must use one of them to load tables.
+
+ - `priority` - `optional` `int`, set job priority with range `0-4` which indicates the priority from high to low. Default value is `3`
+ - `options` - `required` `map[string:args]`, mapping from table (`DB.TABLE`) to arguments, `args` can be set as follows：
+ - `partition_col` - `required` `string`, partition column of responding table, null by default. If you have set partition columns for tables to be refreshed, here **need to enter the corresponding partition column**
+ - `incremental_build` - `optional` `boolean`, whether keep the built partitions, false by default.
+ - `partitions_to_build` - `optional` `array[string]`, only refresh the special partitions.
+- `yarn_queue` - `optional` `string`, specify the YARN queue used by the job, it can be set after these two parameters were set: kylin.engine-yarn.queue.in.task.enabled (whether to allow set specified YARN queue for build task, default value is false), kylin.engine-yarn.queue.in.task.available (available YARN queues, separate them with English commas)
+
+ - `tag` - `optional` `object`, job tag, if the field is set, when calling the [Get Job List](job_api.md) API, the field will be the same back when returning the job. It can be used for system integration, mark the job and deal with it accordingly. By default, the maximum size of value is 1024 KB , which can be set by the configure kylin.job.tag-max-size=1024.
+
+ **Note**:
+
+ - If the loaded table already exists in the system, it will be reloaded
+ - The `partition_col` in the parameter `options` is temporarily **required**. If the interface for setting the partition column has been called before, the correct parameter value must also be filled here
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+```sh
+curl -X PUT \
+ 'http://localhost:7070/kylin/api/snapshots' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{"project":"gc_test", "tables": ["SSB.P_LINEORDER", "DEFAULT.TEST_KYLIN_FACT"],"options":{"DEFAULT.TEST_KYLIN_FACT":{"partition_col":"CAL_DT","incremental_build":true,"partitions_to_build":["2012-03-01","2012-03-04"]}}}'
+```
+- Curl Response Example
+```json
+{
+ "code": "000",
+ "data": {
+ "jobs": [{
+ "job_name": "SNAPSHOT_REFRESH",
+ "job_id": "65b3b0a4-d4d2-4a5b-af29-b190ca420543"
+ }, {
+ "job_name": "SNAPSHOT_REFRESH",
+ "job_id": "24aafb93-1cde-43d1-a627-8cd592f51cfe"
+ }]
+ },
+ "msg": ""
+}
+```
+
+
+### <span id="delete">Delete Snapshots</span>
+
+- `DELETE http://host:port/kylin/api/snapshots`
+- Request Permission: OPERATION permission and above
+- Introduced in: 4.2.2
+- URL Parameters
+ - `project` - `required` `string`, project name.
+ - `tables` - `required` `array[string]`, snapshot tables to be deleted, for example：DB.TABLE, multiple tables are splitted by comma. Because the http protocol has a limit on the size of the request header, it is recommended that the length of url is less than 100.
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+- Curl Request Example
+```sh
+curl -X DELETE \
+ 'http://localhost:7070/kylin/api/snapshots?project=gc_test&tables=SSB.P_LINEORDER%2CDEFAULT.TEST_KYLIN_FACT' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+```
+
+
+- Response Example
+```json
+{
+ "code":"000",
+ "data": {
+ "affected_jobs":[
+ {
+ "database": "DEFAULT",
+ "table": "KYLIN_CAL",
+ "job_id": "e3aa809b-5e73-42a5-a1e1-649d53b16e2c"
+ },
+ {
+ "database": "DEFAULT",
+ "table": "P_LINEORDER",
+ "job_id": "e3aa809b-5e73-42a5-a1e1-649d53b16e2b"
+ }
+ ]
+ },
+ "msg":""
+}
+```
+
+
+### <span id="read">Get Snapshot List</span>
+
+- `GET http://host:port/kylin/api/snapshots`
+- Request Permission: QUERY permission and above
+- Introduced in: 4.2.2
+- URL Parameters
+ - `project` - `required` `string`, project name.
+ - `table` - `optional` `string`, search key word. Default value is an empty string, will display all the snapshots.
+ - `page_offset` -`optional` `int`, offset of returned result, `0` by default.
+ - `page_size` -`optional` `int`, quantity of returned result per page, `10` by default.
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+
+- Curl Request Example
+```sh
+curl -X GET \
+ 'http://localhost:7070/kylin/api/snapshots?project=gc_test' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+```
+- Response Example
+```json
+{
+ "code":"000",
+ "data":{
+ "value":[
+ {
+ "table":"P_LINEORDER",
+ "database":"SSB",
+ "usage": 5,
+ "storage": 8555,
+ "fact_table_count": 2
+ "lookup_table_count": 2,
+ "last_modified_time": 1602315332279,
+ "status": "REFRESHING"
+ },
+ Object{...},
+ Object{...},
+ Object{...}
+ ],
+ "offset":0,
+ "limit":10,
+ "total_size":4
+ },
+ "msg":""
+}
+```
diff --git a/website/docs/restapi/streaming_job_api.md b/website/docs/restapi/streaming_job_api.md
new file mode 100644
index 0000000000..e8da883e79
--- /dev/null
+++ b/website/docs/restapi/streaming_job_api.md
@@ -0,0 +1,282 @@
+---
+title: Streaming Job API
+language: en
+sidebar_label: Streaming Job API
+pagination_label: Streaming Job API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - streaming job api
+draft: true
+last_update:
+ date: 12/08/2022
+---
+
+> Reminders:
+>
+> 1. Please read [Access and Authentication REST API](authentication.md) and understand how authentication works.
+> 2. On Curl command line, don't forget to quote the URL if it contains the special char `&`.
+
+
+### Get Job List
+
+- `GET http://host:port/kylin/api/streaming_jobs`
+
+- Introduced in: 4.5.8.2
+
+- Scenarios
+
+ Obtain detailed information of streaming jobs, and provide operation and maintenance decisions based on information such as task status.
+
+- URL Parameters
+ - `model_name` - `optional` `string`, fuzzy match model name
+
+ - `model_names` - `optional` `array<string>`, exact model name list
+
+ - `job_types` - `optional` `array<string>`, job types, Optional values:`STREAMING_MERGE`,`STREAMING_BUILD`
+
+ - `statuses` - `optional` `array<string>`, job status, Optional values:`STARTING`,`RUNNING`,`STOPPING`,`ERROR` ,`STOPPED`
+
+ - `project` - `optional` `string`, project name
+
+ - `page_offset` - `optional` `int`, offset of returned result, 0 by default
+
+ - `page_size` - `optional` `int`, quantity of returned result per page, 10 by default
+
+ - `sort_by` - `optional` `string`, sort field, optional values: `last_modified` by default,Optional values:`model_alias`、`data_latency`、`last_status_duration`、`last_modified`
+
+ - `reverse` - `optional` `boolean`, whether sort reverse, "true" by default
+
+ - `job_ids` - `optional` `array<string>`, job id list, the parameter `project` is required when `job_ids` is not empty
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl -X GET \
+ 'http://host:port/kylin/api/streaming_jobs' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8'
+ ```
+
+- Response Details
+
+ - `Code`: `String`, response code. **Returned value**: `000` (request processing success), `999 ` (request processing failed)
+ - `data`: `JSON`, response data.
+ - `value`: details of the response data, which consists of:
+ - `uuid`: `String`, job ID
+ - `last_modified`: `Long`, last modified time of the job.
+ - `version`: `String`, system metadata version.
+ - `mvcc`: `Long`, version number with metadata modified.
+ - `model_alias`: `String`, model alias.
+ - `owner`: `String`, job owner
+ - `model_id`: `String`, Model ID.
+ - `last_start_time`: `Long`, last start time of the job.
+ - `last_end_time`: `Long`, last end time of the job.
+ - `last_update_time`: `Long`, last update time of the job.
+ - `last_batch_count`: `Long`, last number of messages processed of the job.
+ - `subscribe`: `String`, topic name.
+ - `fact_table`: `String`, fact table.
+ - `job_status`: `String`, job status, Possible values:`STARTING`,`RUNNING`,`STOPPING`,`ERROR` ,`STOPPED`.
+ - `job_type`: `String`, job types, Possible values:`STREAMING_MERGE`,`STREAMING_BUILD`.
+ - `process_id`: `String`, the process ID of the job.
+ - `node_info`: `String`, the information of the machine that run the job.
+ - `job_execution_id`: `String`, the execution ID of the job.
+ - `yarn_app_id`: `String`, the ApplicationId on yarn.
+ - `yarn_app_url`: `String`, the Application url on yarn.
+ - `project`: `String`, project name.
+ - `skip_listener`: `Boolean`，whether skip to use listener。
+ - `action`: `String`, the action that job is located.
+ - `model_broken`: `Boolean`, whether the model is broken or not.
+ - `data_latency`: `Long`, the minimum latency of data(ms).
+ - `last_status_duration`: `Long`, the last status change time(ms).
+ - `model_indexes`: `Int`, total number of indexes.
+ - `launching_error`: `Boolean`, whether launching error.
+ - `params`: `JSON`, the building parameters of the job.
+ - `partition_desc`: `JSON`, partition description.
+ - `offset`: page number
+ - `limit`: jobs listed in each page
+ - `total_size`: total number of jobs
+ - `msg`: `String`: error message
+
+- Response Example
+
+ ```json
+ {
+ "code": "000",
+ "data": {
+ "value": [
+ {
+ "uuid": "7bccf62d-535c-70b8-8271-eaef3985aa96_merge",
+ "last_modified": 1645186414513,
+ "create_time": 1644823384864,
+ "version": "4.0.0.0",
+ "mvcc": -1,
+ "model_alias": "hy_model",
+ "owner": "ADMIN",
+ "model_id": "7bccf62d-535c-70b8-8271-eaef3985aa96",
+ "last_start_time": null,
+ "last_end_time": null,
+ "last_update_time": "2022-02-18 19:21:27",
+ "last_batch_count": null,
+ "subscribe": null,
+ "fact_table": "BASE.HY_LINEORDER",
+ "job_status": "ERROR",
+ "job_type": "STREAMING_MERGE",
+ "process_id": null,
+ "node_info": null,
+ "job_execution_id": null,
+ "yarn_app_id": "",
+ "yarn_app_url": "",
+ "params": {
+ "spark.executor.memory": "1g",
+ "kylin.streaming.segment-max-size": "32m",
+ "spark.master": "yarn",
+ "spark.driver.memory": "512m",
+ "spark.executor.cores": "2",
+ "kylin.streaming.job-retry-enabled": "false",
+ "spark.executor.instances": "2",
+ "kylin.streaming.segment-merge-threshold": "3",
+ "spark.sql.shuffle.partitions": "8"
+ },
+ "project": "p1",
+ "skip_listener": false,
+ "action": null,
+ "model_broken": false,
+ "data_latency": null,
+ "last_status_duration": 240562073,
+ "model_indexes": 3,
+ "launching_error": false,
+ "partition_desc": {
+ "partition_date_column": "HY_LINEORDER.LO_PARTITIONCOLUMN",
+ "partition_date_start": 0,
+ "partition_date_format": "yyyy-MM-dd HH:mm:ss",
+ "partition_type": "APPEND",
+ "partition_condition_builder": "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
+ }
+ },
+ {
+ "uuid": "7bccf62d-535c-70b8-8271-eaef3985aa96_build",
+ "last_modified": 1645186414089,
+ "create_time": 1644823384857,
+ "version": "4.0.0.0",
+ "mvcc": -1,
+ "model_alias": "hy_model",
+ "owner": "ADMIN",
+ "model_id": "7bccf62d-535c-70b8-8271-eaef3985aa96",
+ "last_start_time": null,
+ "last_end_time": null,
+ "last_update_time": "2022-02-18 17:43:27",
+ "last_batch_count": null,
+ "subscribe": null,
+ "fact_table": "BASE.HY_LINEORDER",
+ "job_status": "ERROR",
+ "job_type": "STREAMING_BUILD",
+ "process_id": null,
+ "node_info": null,
+ "job_execution_id": null,
+ "yarn_app_id": "application_1643095564973_0592",
+ "yarn_app_url": "http://10.1.2.210:8088/proxy/application_1643095564973_0592/",
+ "params": {
+ "spark.executor.memory": "1g",
+ "spark.master": "yarn",
+ "spark.driver.memory": "512m",
+ "kylin.streaming.kafka-conf.maxOffsetsPerTrigger": "0",
+ "kylin.streaming.duration": "30",
+ "spark.executor.cores": "2",
+ "kylin.streaming.job-retry-enabled": "false",
+ "spark.executor.instances": "2",
+ "spark.sql.shuffle.partitions": "8"
+ },
+ "project": "p1",
+ "skip_listener": false,
+ "action": null,
+ "model_broken": false,
+ "data_latency": 0,
+ "last_status_duration": 246442660,
+ "model_indexes": 3,
+ "launching_error": false,
+ "partition_desc": {
+ "partition_date_column": "HY_LINEORDER.LO_PARTITIONCOLUMN",
+ "partition_date_start": 0,
+ "partition_date_format": "yyyy-MM-dd HH:mm:ss",
+ "partition_type": "APPEND",
+ "partition_condition_builder": "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
+ }
+ }
+ ],
+ "offset": 0,
+ "limit": 10,
+ "total_size": 2
+ },
+ "msg": ""
+ }
+ ```
+
+
+
+### Operate Job
+
+- `PUT http://host:port/kylin/api/streaming_jobs/status`
+
+- Introduced in: 4.5.8.2
+
+- Scenarios
+
+ Modify the status of the jobs. For example, you can restart job task after finding that the job is abnormal.
+
+- URL Parameters
+ - `action` - `required` `string`, action types for jobs. Optional values are below:
+ - `START`,start selected jobs
+ - `STOP`, stop selected jobs
+ - `FORCE_STOP`, force to stop the selected jobs
+ - `RESTART`, restart selected jobs
+
+ - `project` - `optional` `string`, project name.
+
+ - `job_ids` - `required` `array<string>`, job id.
+
+- HTTP Header
+ - `Accept: application/vnd.apache.kylin-v4-public+json`
+ - `Accept-Language: en`
+ - `Content-Type: application/json;charset=utf-8`
+
+- Curl Request Example
+
+ ```sh
+ curl --location --request PUT 'http://host:port/kylin/api/streaming_jobs/status' \
+ -H 'Accept: application/vnd.apache.kylin-v4-public+json' \
+ -H 'Accept-Language: en' \
+ -H 'Authorization: Basic QURNSU46S1lMSU4=' \
+ -H 'Content-Type: application/json;charset=utf-8' \
+ -d '{
+ "project": "p1",
+ "action": "RESTART",
+ "job_ids": [
+ "7bccf62d-535c-70b8-8271-eaef3985aa96_merge"
+ ]
+ }'
+ ```
+
+- Response Details
+ - `Code`: `String`, response code. **Returned value**: `000` (request processing success), `999 ` (request processing failed)
+ - `data`: `String`, response data, always be empty.
+ - `msg`：`String`: error message
+
+- Response Example
+
+ ```json
+ {
+ "code":"000",
+ "data":"",
+ "msg":""
+ }
+ ```
diff --git a/website/sidebars.js b/website/sidebars.js
index b216a05067..c971863094 100644
--- a/website/sidebars.js
+++ b/website/sidebars.js
@@ -157,10 +157,106 @@ const sidebars = {
{
type: 'category',
label: 'Rest API',
+ link: {
+ type: 'doc',
+ id: 'restapi/intro',
+ },
items: [
{
type: 'doc',
- id: 'restapi/intro'
+ id: 'restapi/authentication'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/project_api'
+ },
+ {
+ type: 'category',
+ label: 'Model API',
+ link: {
+ type: 'doc',
+ id: 'restapi/model_api/intro',
+ },
+ items: [
+ {
+ type: 'doc',
+ id: 'restapi/model_api/model_management_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/model_api/model_build_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/model_api/model_import_and_export_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/model_api/model_multilevel_partitioning_api'
+ },
+ ],
+ },
+ {
+ type: 'doc',
+ id: 'restapi/segment_managment_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/snapshot_management_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/query_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/data_source_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/async_query_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/job_api'
+ },
+ {
+ type: 'category',
+ label: 'ACL Management API',
+ link: {
+ type: 'doc',
+ id: 'restapi/acl_api/intro',
+ },
+ items: [
+ {
+ type: 'doc',
+ id: 'restapi/acl_api/user_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/acl_api/user_group_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/acl_api/project_acl_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/acl_api/acl_api'
+ },
+ ],
+ },
+ {
+ type: 'doc',
+ id: 'restapi/streaming_job_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/callback_api'
+ },
+ {
+ type: 'doc',
+ id: 'restapi/error_code'
},
],
},
[kylin] 01/03: KYLIN-5221, add deploy mode
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch doc5.0
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8ab848ed25e017ebfc6c0b40c55ea94438f2e40f
Author: Mukvin <bo...@163.com>
AuthorDate: Fri Aug 12 15:41:31 2022 +0800
KYLIN-5221, add deploy mode
---
.../deployment/deploy_mode/cluster_deployment.md | 86 +++++++++++++++
.../deploy_mode/images/cluster_20191231.png | Bin 0 -> 199821 bytes
website/docs/deployment/deploy_mode/intro.md | 18 ++++
.../docs/deployment/deploy_mode/rw_separation.md | 90 ++++++++++++++++
.../deployment/deploy_mode/service_discovery.md | 36 +++++++
.../rdbms_metastore/{ => mysql}/install_mysql.md | 6 +-
.../docs/deployment/rdbms_metastore/mysql/intro.md | 20 ++++
.../rdbms_metastore/mysql/mysql_metastore.md | 118 +++++++++++++++++++++
.../{ => postgresql}/default_metastore.md | 0
.../{ => postgresql}/install_postgresql.md | 4 +-
.../deployment/rdbms_metastore/postgresql/intro.md | 20 ++++
website/docs/intro.md | 14 ++-
website/docs/modeling/intro.md | 25 +++--
website/docs/operations/intro.md | 26 +++--
website/docs/restapi/intro.md | 25 +++--
website/sidebars.js | 86 +++++++++++++--
16 files changed, 530 insertions(+), 44 deletions(-)
diff --git a/website/docs/deployment/deploy_mode/cluster_deployment.md b/website/docs/deployment/deploy_mode/cluster_deployment.md
new file mode 100644
index 0000000000..18b5d4b0ae
--- /dev/null
+++ b/website/docs/deployment/deploy_mode/cluster_deployment.md
@@ -0,0 +1,86 @@
+---
+title: Cluster Deployment
+language: en
+sidebar_label: Cluster Deployment
+pagination_label: Cluster Deployment
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - cluster deployment
+ - cluster
+draft: true
+last_update:
+ date: 12/08/2022
+---
+
+All Kylin state information instance is stored in a RDBMS database, so running Kylin on multiple nodes in a cluster is good practice for better load balance and higher availability. Currently, we only support deployment with one `all` node and multiple `query` nodes.
+
+[comment]: <#TODO> ()
+
+In the above diagram, the components which require user deployment are below:
+
+- RDBMS as Metastore（PostgreSQL/MySQL）
+- Time-Series Database (InfluxDB)
+- Kylin nodes
+- Load Balancer
+
+We will go through each one of them:
+
+### RDBMS and Time-Series Database Introduction
+
+Kylin uses RDBMS to store metadata, please refer to [Use RDBMS as Metastore](../rdbms_metastore/intro.md).
+
+Kylin uses Time-series database to store metrics (mostly for monitor purpose), please refer to [Use InfluxDB as Time-Series Database](../../operations/monitoring/influxdb/influxdb.md).
+
+### Kylin Nodes Introduction
+
+Currently, Kylin supports multiple nodes deployment. Queries will be distributed through all nodes and jobs can be submitted by all types of nodes. However, the building jobs will be submitted to job engine node ( `all` node and `job` node).
+
+- `query` node: You can submit queries or building jobs on `query` nodes but the jobs cannot be executed.
+- `job` node: This node can execute jobs and metadata operations
+- `all` node: `all` node can perform all the work of `query` and `job` node.
+
+### Kylin Instance Configuration
+
+**Note**:
+
+- Please deploy Kylin nodes in the same Hadoop cluster. If you need to deploy a read/write separation environment, please refer to [Read/Write Separation Deployment](rw_separation.md) chapter.
+- It is recommended not to start multiple `all` nodes at the same time.
+
+### Kylin Multi-Active Job Engines
+
+Kylin job engine is used to execute building jobs, schedule jobs and metadata operations. Normally, users will configure one or more node as job engine in the cluster deployment.
+
+Since Kylin 5.0, Kylin works "Multi-Active" mode, which meanings every healthy `all` and `job` node can execute building tasks. When one job engine is stopped, the other job engines will take over the running task on it. That will ensure the high availability of Kylin services.
+
+It should be noted that, due to the existence of project locks, when jobs are allocated to nodes, the project is the smallest unit of allocation, that is, all jobs in a project can only be executed on one job engine. Different projects and jobs can be executed on different job engines.
+
+If you want to enable this function, add `kylin.server.mode=all` or `kylin.server.mode=job` into `$KYLIN_HOME/conf/kylin.properties` for more than one node.
+
+The related config items are shown below:
+```properties
+kylin.server.leader-race.heart-beat-timeout=60
+kylin.server.leader-race.heart-beat-interval=30
+kylin.job.ssh-username=username
+kylin.job.ssh-password=password
+```
+`kylin.server.leader-race.heart-beat-timeout` stands for heart beat timeout，default value is 60, in seconds. When an job engine node didn't send heartbeat for more than this value, this node will be marked as unavailable.
+`kylin.server.leader-race.heart-beat-interval` stands for heart beating's time interval，default value is 30, in seconds, which means each job engine node will send heartbeat for every 30 seconds.
+`kylin.job.ssh-username` stands for ssh login username between Kylin nodes, if passwordless is not configured.
+`kylin.job.ssh-password` stands for ssh login password between Kylin nodes, if passwordless is not configured.
+
+
+**Known Limitations**
+
+1. Enable session sharing is required when using Kylin HA. Add `spring.session.store-type=jdbc` into `$KYLIN_HOME/conf/kylin.properties` for each node.
+2. Please make sure there is no clock skew in the cluster. You can use `ntp` service to sync the clock in the cluster.
+3. When a Kylin job engine node is down, may lead to all projects owned by this node send metadata related requests, the hint is: `System is trying to recover, please try again later.`
+4. When all Kylin job engine nodes are down, you may see the hint when you trying to send metadata related requests in the `query` node: `There is no active All node, please contact to your administrator to check and fix.`.
+5. It is necessary to configure password-free login between each node, or ensure that the ssh login information between each node is consistent.
+
+**Note:**
+
+- The "Multi-Active" mode is enabled by default. If there is only one `all` or `job` node, this mode should be turned off because of performance considerations. If you want to disable this feature, please add `kylin.server.leader-race.enabled=false` in `$KYLIN_HOME/conf/kylin.properties` for the `all` or `job` node.
+- If you want to enable it again, please update the relationship between projects and the job engines. After that, you need call Rest API to update the data. For details, please refer to [Project Settings API](../../restapi/project_api.md)
diff --git a/website/docs/deployment/deploy_mode/images/cluster_20191231.png b/website/docs/deployment/deploy_mode/images/cluster_20191231.png
new file mode 100644
index 0000000000..0d13c43390
Binary files /dev/null and b/website/docs/deployment/deploy_mode/images/cluster_20191231.png differ
diff --git a/website/docs/deployment/deploy_mode/intro.md b/website/docs/deployment/deploy_mode/intro.md
new file mode 100644
index 0000000000..27ef9f3b86
--- /dev/null
+++ b/website/docs/deployment/deploy_mode/intro.md
@@ -0,0 +1,18 @@
+---
+title: Deployment Mode
+language: en
+sidebar_label: Deployment Mode
+pagination_label: Deployment Mode
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - deployment mode
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+This chapter introduces the different Kylin deployment modes and options to achieve high performance, high stability, and high availability.
+
diff --git a/website/docs/deployment/deploy_mode/rw_separation.md b/website/docs/deployment/deploy_mode/rw_separation.md
new file mode 100644
index 0000000000..2f2dbb1fc1
--- /dev/null
+++ b/website/docs/deployment/deploy_mode/rw_separation.md
@@ -0,0 +1,90 @@
+---
+title: Read/Write Separation Deployment
+language: en
+sidebar_label: Read/Write Separation Deployment
+pagination_label: Read/Write Separation Deployment
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - read/write separation
+ - separation
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+Kylin's tasks based on Hadoop are mainly divided into two types: build and query. If these two tasks use the same set of Hadoop resources, resource preemption may occur between the build and the query, which makes them not stable and fast.
+
+Kylin allows you to finish building and query tasks on different Hadoop clusters. There are many write operations in the former, known as the **Build Cluster**, while the latter is dominated by read-only operations, called **Query Cluster**. The build task will be sent to the build cluster. After the build is completed, the system will send the data into the query cluster so as to executing query tasks.
+
+With a read/write separation deployment, you can completely isolate both build and query workloads, allowing them to run independently, avoiding improper interactions between them and possible performance instability.
+
+### Prerequisites
+
+Due to the involvement of two Hadoop environments, please read and comply with the following environmental checks.
+
+1. Please confirm the Hadoop versions of build cluster and query cluster are identical, and they're supported version by Kylin.
+
+2. Please confirm that the Hadoop client of build cluster and query cluster is installed and configured on the **Kylin server**. Check commands like `hdfs`、`hive` are all working properly and can access cluster resources.
+
+3. If the two clusters have enabled the HDFS NameNode HA, please check and make sure their HDFS nameservice names are different. If they are the same, please change one of them to avoid conflict.
+
+4. Please check the two clusters can access each other without manually inputting any user credential.
+
+ > **Tips**: As a test, on any build cluster try to copy some HDFS files from/to the query cluster. The copy must succeed without any extra manual interaction.
+
+5. Please make sure the network latency between the two clusters is low enough, as there will be a large number of data moved back and forth during model build process.
+
+6. If Kerberos is enabled, please check the following:
+
+ - The build cluster and the query cluster belong to different realms.
+ - The cross-realm trust between the two clusters is configured properly.
+
+### Install and Configure Read/Write Separation Deployment
+
+You can follow the below instructions to finish Kylin read/write separation deployment based on hadoop clusters.
+
+1. First of all, on **Kylin server**, uncompress Kylin software package to the same location. This location will be referenced as `$KYLIN_HOME` later.
+
+2. Secondly, you should prepare the hadoop conf file of the two clusters. The hadoop configuration of query cluster should be put into `$KYLIN_HOME/hadoop_conf` directory, while that of build query will be put into `$KYLIN_HOME/write_hadoop_conf` directory. What's more, the `hive-site.xml` of the build cluster should be put into the above two directories, make two copies of hive-site.xml and name them hiveserver2-site.xml and hivemetastore-site.xml respectively.
+
+ If Kerberos authentication enabled, you need to copy the krb5.conf file of build cluster to the `$KYLIN_HOME/write_hadoop_conf` directory and copy the krb5.conf file of query cluster to the `$KYLIN_HOME/hadoop_conf` directory.
+
+3. Set the configuration:
+
+ ```properties
+ kylin.engine.submit-hadoop-conf-dir=$KYLIN_HOME/write_hadoop_conf
+
+ ## Working path of Kylin instance on HDFS. Please replace {working_dir} with the real working path, and use the absolute path, such as hdfs://kylin
+ kylin.env.hdfs-working-dir={working_dir}
+ ```
+
+4. If Kerberos is enabled, you will need to do additional configuration:
+
+ ```properties
+ kylin.storage.columnar.spark-conf.spark.yarn.access.hadoopFileSystems=hdfs://readcluster,hdfs://writecluster
+ kylin.engine.spark-conf.spark.yarn.access.hadoopFileSystems=hdfs://readcluster,hdfs://writecluster
+ ```
+5. If the Kerberos authentication mechanism enabled, the following checks need to be done to avoid errors in the segment build job:
+
+ ```
+ java.lang.IllegalArgumentException: Can't get Kerberos realm
+ ...
+ Caused by: KrbException: Cannot locate default realm
+ ```
+
+ 1. Make sure the `keytab` file and `krb5.conf` file required for authentication exist in the directory `$KYLIN_HOME/conf`
+ 2. If the directory `$KYLIN_HOME/hadoop_conf` exists, make sure there is a `krb5.conf` file in this directory
+ 3. If the directory `$KYLIN_HOME/write_hadoop_conf` exists, make sure there is a `krb5.conf` file in this directory
+
+Now read/write separation deployment is configured.
+
+**Note:**
+
+* `$KYLIN_HOME/bin/check-env.sh` and `$KYLIN_HOME/bin/sample.sh` are not available in this deployment mode.
+
+* In this mode, `kylin.engine.spark-conf.spark.yarn.queue` in `kylin.properties` should be configured as the queue of the build cluster.
+
+
diff --git a/website/docs/deployment/deploy_mode/service_discovery.md b/website/docs/deployment/deploy_mode/service_discovery.md
new file mode 100644
index 0000000000..a08758deff
--- /dev/null
+++ b/website/docs/deployment/deploy_mode/service_discovery.md
@@ -0,0 +1,36 @@
+---
+title: Service Discovery
+language: en
+sidebar_label: Service Discovery
+pagination_label: Service Discovery
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - service discovery
+ - ha
+draft: false
+last_update:
+ date: 12/08/2022
+---
+
+Multiple Kylin instances can work together as a cluster. When a Kylin instance is started, stopped, or lost, other instances in the cluster will be updated automatically. Kylin has a new implementation based on Apache Curator framework, which is more convenient and more stable.
+
+If you want to enable service discovery, please set the correct ZooKeeper address in the configuration file `$KYLIN_HOME/conf/kylin.properties`. For example:
+
+```properties
+kylin.env.zookeeper-connect-string=host1:2181,host2:2181,host3:2181
+```
+
+After the configuration, please restart Kylin. Each Kylin instance will register itself in ZooKeeper, and each one will discover other instances from ZooKeeper.
+
+By default, Kylin will use hostname as the node name. If the cluster you run Kylin on cannot parses the host name to IP address through DNS and other services, you could set an additional configuration in `$KYLIN_HOME/conf/kylin.properties` as below (this can avoid communication errors between Kylin servers). For example:
+
+```properties
+spring.cloud.zookeeper.discovery.instance-host=10.1.2.3
+```
+
+This config enforces Kylin to use IP address for service registry, making sure that other instances can find it by IP address instead of hostname. In case you use this config, please specify this config for **each of the Kylin instance** in `kylin.properties`.
+
+**Note:** Please replace the IP address with actual IP address on each server.
diff --git a/website/docs/deployment/rdbms_metastore/install_mysql.md b/website/docs/deployment/rdbms_metastore/mysql/install_mysql.md
similarity index 97%
rename from website/docs/deployment/rdbms_metastore/install_mysql.md
rename to website/docs/deployment/rdbms_metastore/mysql/install_mysql.md
index b3bced9cd6..d3464debea 100644
--- a/website/docs/deployment/rdbms_metastore/install_mysql.md
+++ b/website/docs/deployment/rdbms_metastore/mysql/install_mysql.md
@@ -8,7 +8,9 @@ toc_max_heading_level: 6
pagination_prev: null
pagination_next: null
keywords:
- - postgresql
+ - mysql
+ - install mysql
+ - install
draft: true
last_update:
date: 11/08/2022
@@ -21,7 +23,7 @@ last_update:
- MySQL 5.1 to 5.7, MySQL 5.7 is recommended
- MySQL 8
-2. JDBC driver of MySQL is needed in the Kyligence Enterprise running environment.
+2. JDBC driver of MySQL is needed in the Kylin running environment.
3. You can download the JDBC driver jar package of MySQL 8 via the link below, that compatible with the version after 5.6:
diff --git a/website/docs/deployment/rdbms_metastore/mysql/intro.md b/website/docs/deployment/rdbms_metastore/mysql/intro.md
new file mode 100644
index 0000000000..fd0cf7a152
--- /dev/null
+++ b/website/docs/deployment/rdbms_metastore/mysql/intro.md
@@ -0,0 +1,20 @@
+---
+title: MySQL
+language: en
+sidebar_label: MySQL
+pagination_label: MySQL
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - mysql
+draft: false
+last_update:
+ date: 11/08/2022
+---
+
+Kylin supports to use MySQL as Metastore, this chapter will discuss how to:
+
+- [Install MySQL](install_mysql.md)
+- [Use MySQL as Metastore](mysql_metastore.md)
diff --git a/website/docs/deployment/rdbms_metastore/mysql/mysql_metastore.md b/website/docs/deployment/rdbms_metastore/mysql/mysql_metastore.md
new file mode 100644
index 0000000000..408edf4ceb
--- /dev/null
+++ b/website/docs/deployment/rdbms_metastore/mysql/mysql_metastore.md
@@ -0,0 +1,118 @@
+---
+title: Use MySQL as Metastore
+language: en
+sidebar_label: Use MySQL as Metastore
+pagination_label: Use MySQL as Metastore
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - mysql
+ - mysql metastore
+ - metastore
+draft: false
+last_update:
+ date: 11/08/2022
+---
+
+
+### <span id ="Prerequisite">Prerequisite</span>
+
+1. Please install MySQL, If the MySQL has not been installed, please refer to [Insatall MySQL](install_mysql.md) for more details.
+
+### <span id ="ConfigurationSteps">Configuration Steps</span>
+
+The following steps illustrate how to connect MySQL as metastore. Here is an example for MySQL 5.7 .
+
+1. Create database `kylin` in MySQL
+
+2. Set configuration item `kylin.metadata.url = {metadata_name}@jdbc` in `$KYLIN_HOME/conf/kylin.properties`,
+ please replace `{metadata_name}` with your metadata name in MySQL, for example, `kylin_default_instance@jdbc`, the maximum length of `{metadata_name}` allowed is 29.
+
+ > **Note**: If the metadata name doesn't exist, it will be automatically created in MySQL. Otherwise, Kylin will use the existing one.
+
+ For example:
+
+ ```properties
+ kylin.metadata.url={metadata_name}@jdbc,driverClassName=com.mysql.jdbc.Driver,url=jdbc:mysql://{host}:{port}/kylin?useUnicode=true&characterEncoding=utf8,username={user},password={password},maxTotal=50,maxIdle=8
+ ```
+
+ The meaning of each parameter is as below, `url`, `username`, and `password` are required parameters. For others, default values will be used if they are not indicated.
+
+ * **driverClassName**: JDBC's driver class name, default value is `com.mysql.jdbc.Driver`;
+ * **url**: JDBC's url;
+ * **host**：MySQL ip address, whose default value is `localhost`;
+ * **port**：MySQL port, whose default value is `3306`. Please use the actual port to replace.
+ * **kylin**: Metabase name. Make sure this database `kylin` has been created in MySQL;
+ * **username**: JDBC's username;
+ * **password**: JDBC's password;
+ * **maxTotal**: max number of database's connection number, default value is 50;
+ * **maxIdle**: max number of database's waiting connection number, default value is 8;
+
+> **Note**: if your query SQL contains chinese, please configure the character encoding to utf8 in kylin.metadata.url to avoid confusing query history: useunicode = true & character encoding = utf8
+
+3. Encrypt JDBC password
+
+ If you need to encrypt JDBC's password, you can do it like this：
+
+ **i.** run following commands in ${KYLIN_HOME}, it will print encrypted password
+ ```shell
+ ./bin/kylin.sh org.apache.kylin.tool.general.CryptTool -e AES -s <password>
+ ```
+
+ **ii.** config properties kylin.metadata.url's password like this
+ ```properties
+ password=ENC('${encrypted_password}')
+ ```
+
+ For example, the following assumes that the JDBC password is kylin:
+
+ First, we need to encrypt kylin using the following command
+ ```shell
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.general.CryptTool -e AES -s kylin
+ AES encrypted password is:
+ YeqVr9MakSFbgxEec9sBwg==
+ ```
+ Then, config kylin.metadata.url like this：
+ ```properties
+ kylin.metadata.url={metadata_name}@jdbc,driverClassName=com.mysql.jdbc.Driver,url=jdbc:mysql://{host}:{port}/kylin?useUnicode=true&characterEncoding=utf8,username={user},password=ENC('YeqVr9MakSFbgxEec9sBwg=='),maxTotal=20,maxIdle=20
+ ```
+
+4. If you need to use MySQL cluster deployment, please add `replication` or `loadbalance` in url with `"`. For example:
+
+ ```properties
+ #use replication in cluster deployment
+ kylin.metadata.url=kylin_default_instance@jdbc,driverClassName=com.mysql.jdbc.Driver,url="jdbc:mysql:replication://localhost:3306,localhost:3307/kylin?useUnicode=true&characterEncoding=utf8",username=root,password=,maxTotal=20,maxIdle=20
+
+ #use loadbalance in cluster deployment
+ kylin.metadata.url=kylin_default_instance@jdbc,driverClassName=com.mysql.jdbc.Driver,url="jdbc:mysql:loadbalance://localhost:3306,localhost:3307/kylin?useUnicode=true&characterEncoding=utf8",username=root,password=,maxTotal=20,maxIdle=20
+ ```
+5. Make sure that the storage engine used with MySQL is **InnoDB** is not **MyISAM** and that the default storage engine is modified as follows:
+
+ ```Properties
+ [mysqld]
+ default-storage-engine=InnoDB
+ ```
+
+### <span id="faq">FAQ</span>
+
+**Q: After the JDK is upgraded to jdk 8u261, the startup of Kylin fails, indicating that the creation of the admin user failed, what should I do? **
+
+A: When you use JDK 8u261 and use MySQL 5.6 or 5.7 as metastore. Since the version before TLS 1.2 has been disabled since the JDK 8u261, and MySQL 5.6 and 5.7 use TLS 1.0 or TLS 1.1 by default, and MySQL must establish an SSL connection by default, which causes conflicts with the TLS protocol, resulting in the startup of Kylin fails, you will see the error message on the terminal as `Create Admin user failed`.
+
+ You have 2 solutions:
+
+Method 1: Modify the metadata configuration parameters and add `useSSL=false`
+
+```properties
+kylin.metadata.url={metadata_name}@jdbc,driverClassName=com.mysql.jdbc.Driver,url=jdbc:mysql://{host}:{port}/kylin?useSSL=false,useUnicode=true&characterEncoding=utf8,username={user},password={password},maxTotal=50,maxIdle=8
+```
+
+Method 2: Modify the java security file `java.security`, find the following configuration, delete TLSv1, TLSv1.1
+
+```sh
+# jdk.tls.disabledAlgorithms=MD5, SSLv3, DSA, RSA keySize < 2048
+jdk.tls.disabledAlgorithms=SSLv3, TLSv1, TLSv1.1, RC4, DES, MD5withRSA, DH keySize < 1024, EC keySize < 224, 3DES_EDE_CBC, anon, NULL, include jdk.disabled.namedCurves
+```
+
diff --git a/website/docs/deployment/rdbms_metastore/default_metastore.md b/website/docs/deployment/rdbms_metastore/postgresql/default_metastore.md
similarity index 100%
rename from website/docs/deployment/rdbms_metastore/default_metastore.md
rename to website/docs/deployment/rdbms_metastore/postgresql/default_metastore.md
diff --git a/website/docs/deployment/rdbms_metastore/install_postgresql.md b/website/docs/deployment/rdbms_metastore/postgresql/install_postgresql.md
similarity index 98%
rename from website/docs/deployment/rdbms_metastore/install_postgresql.md
rename to website/docs/deployment/rdbms_metastore/postgresql/install_postgresql.md
index 392c4987d0..17d47f61d8 100644
--- a/website/docs/deployment/rdbms_metastore/install_postgresql.md
+++ b/website/docs/deployment/rdbms_metastore/postgresql/install_postgresql.md
@@ -125,7 +125,7 @@ last_update:
**v.** <span id="metadata">Run `\l` to check if the database was created successfully. If you see picture as below, you have just created a database named `kylin`.</span>
- 
+ 
@@ -162,7 +162,7 @@ The following example is that Linux user `abc` installs and configures PostgreSQ
As the picture shows below, input password after run the command above, the password is the password for user `postgres`, say the password is `kylin`.
- 
+ 
**ii.** Edit configuration file
diff --git a/website/docs/deployment/rdbms_metastore/postgresql/intro.md b/website/docs/deployment/rdbms_metastore/postgresql/intro.md
new file mode 100644
index 0000000000..b84ad6905a
--- /dev/null
+++ b/website/docs/deployment/rdbms_metastore/postgresql/intro.md
@@ -0,0 +1,20 @@
+---
+title: PostgreSQL
+language: en
+sidebar_label: PostgreSQL
+pagination_label: PostgreSQL
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - postgresql
+draft: false
+last_update:
+ date: 11/08/2022
+---
+
+Kylin supports to use PostgreSQL as Metastore, this chapter will discuss how to:
+
+- [Install PostgreSQL](install_postgresql.md)
+- [Use PostgreSQL as Metastore](default_metastore.md)
diff --git a/website/docs/intro.md b/website/docs/intro.md
index e29e6b6094..e17893a55f 100644
--- a/website/docs/intro.md
+++ b/website/docs/intro.md
@@ -1,5 +1,17 @@
---
-sidebar_position: 0
+title: Intro to Kylin 5.0
+language: en
+sidebar_label: Intro to Kylin 5.0
+pagination_label: Intro to Kylin 5.0
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - rest api
+draft: false
+last_update:
+ date: 12/08/2022
---
# Intro to Kylin 5.0
diff --git a/website/docs/modeling/intro.md b/website/docs/modeling/intro.md
index ccf4259219..533404ab92 100644
--- a/website/docs/modeling/intro.md
+++ b/website/docs/modeling/intro.md
@@ -1,14 +1,17 @@
---
-sidebar_position: 1
+title: Model
+language: en
+sidebar_label: Model
+pagination_label: Model
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model
+draft: false
+last_update:
+ date: 12/08/2022
---
-# Tutorial Intro
-
-Let's discover ** Kylin 5.0 in than 15 minutes **.
-
-## Getting Started
-
-```shell
-
-docker pull apachekylin/apache-kylin-standalone:5.0.0
-```
+This chapter introduces the core concepts of Kylin: model and index design basics, advanced measures and load data.
diff --git a/website/docs/operations/intro.md b/website/docs/operations/intro.md
index ccf4259219..e8feeac788 100644
--- a/website/docs/operations/intro.md
+++ b/website/docs/operations/intro.md
@@ -1,14 +1,18 @@
---
-sidebar_position: 1
+title: Operation and Maintenance Guide
+language: en
+sidebar_label: Operation and Maintenance Guide
+pagination_label: Operation and Maintenance Guide
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - operation
+ - maintenance
+draft: false
+last_update:
+ date: 12/08/2022
---
-# Tutorial Intro
-
-Let's discover ** Kylin 5.0 in than 15 minutes **.
-
-## Getting Started
-
-```shell
-
-docker pull apachekylin/apache-kylin-standalone:5.0.0
-```
+This Operation and Maintenance Guide will introduce a suite of capabilities that can help you to manage your Kylin projects, do system maintenance, work with logs.
diff --git a/website/docs/restapi/intro.md b/website/docs/restapi/intro.md
index ccf4259219..cc09056c66 100644
--- a/website/docs/restapi/intro.md
+++ b/website/docs/restapi/intro.md
@@ -1,14 +1,17 @@
---
-sidebar_position: 1
+title: Rest API
+language: en
+sidebar_label: Rest API
+pagination_label: Rest API
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - rest api
+draft: false
+last_update:
+ date: 12/08/2022
---
-# Tutorial Intro
-
-Let's discover ** Kylin 5.0 in than 15 minutes **.
-
-## Getting Started
-
-```shell
-
-docker pull apachekylin/apache-kylin-standalone:5.0.0
-```
+Kylin provides various REST APIs, which can be used to execute queries, trigger a build job and so on. With those APIs, the third-party system could integrate with Kylin system seamlessly.
diff --git a/website/sidebars.js b/website/sidebars.js
index 0fddf1a701..b216a05067 100644
--- a/website/sidebars.js
+++ b/website/sidebars.js
@@ -81,18 +81,64 @@ const sidebars = {
type: 'doc',
id: 'deployment/rdbms_metastore/intro',
},
+ items: [
+ {
+ type: 'category',
+ label: 'MySQL',
+ link: {
+ type: 'doc',
+ id: 'deployment/rdbms_metastore/mysql/intro',
+ },
+ items: [
+ {
+ type: 'doc',
+ id: 'deployment/rdbms_metastore/mysql/install_mysql'
+ },
+ {
+ type: 'doc',
+ id: 'deployment/rdbms_metastore/mysql/mysql_metastore'
+ },
+ ],
+ },
+ {
+ type: 'category',
+ label: 'PostgreSQL',
+ link: {
+ type: 'doc',
+ id: 'deployment/rdbms_metastore/postgresql/intro',
+ },
+ items: [
+ {
+ type: 'doc',
+ id: 'deployment/rdbms_metastore/postgresql/install_postgresql'
+ },
+ {
+ type: 'doc',
+ id: 'deployment/rdbms_metastore/postgresql/default_metastore'
+ },
+ ],
+ },
+ ]
+ },
+ {
+ type: 'category',
+ label: 'Deployment Mode',
+ link: {
+ type: 'doc',
+ id: 'deployment/deploy_mode/intro',
+ },
items: [
{
type: 'doc',
- id: 'deployment/rdbms_metastore/default_metastore'
+ id: 'deployment/deploy_mode/cluster_deployment'
},
{
type: 'doc',
- id: 'deployment/rdbms_metastore/install_postgresql'
+ id: 'deployment/deploy_mode/service_discovery'
},
{
type: 'doc',
- id: 'deployment/rdbms_metastore/install_mysql'
+ id: 'deployment/deploy_mode/rw_separation'
},
]
}
@@ -130,11 +176,39 @@ const sidebars = {
},
{
type: 'category',
- label: 'Operations',
+ label: 'Operation and Maintenance Guide',
+ link: {
+ type: 'doc',
+ id: 'operations/intro',
+ },
items: [
{
- type: 'doc',
- id: 'operations/intro'
+ type: 'category',
+ label: 'Monitoring',
+ link: {
+ type: 'doc',
+ id: 'operations/monitoring/intro',
+ },
+ items: [
+ {
+ type: 'category',
+ label: 'InfluxDB',
+ link: {
+ type: 'doc',
+ id: 'operations/monitoring/influxdb/intro',
+ },
+ items: [
+ {
+ type: 'doc',
+ id: 'operations/monitoring/influxdb/influxdb'
+ },
+ {
+ type: 'doc',
+ id: 'operations/monitoring/influxdb/influxdb_maintenance'
+ },
+ ],
+ },
+ ],
},
],
},