You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@yunikorn.apache.org by wi...@apache.org on 2023/02/07 03:22:13 UTC
[yunikorn-site] branch master updated: [YUNIKORN-1519] addendum pr (#263)
This is an automated email from the ASF dual-hosted git repository.
wilfreds pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-site.git
The following commit(s) were added to refs/heads/master by this push:
new 9de9b07d0 [YUNIKORN-1519] addendum pr (#263)
9de9b07d0 is described below
commit 9de9b07d0ae3d5e84628746e5b8377665440f736
Author: Manikandan R <ma...@gmail.com>
AuthorDate: Tue Feb 7 14:20:35 2023 +1100
[YUNIKORN-1519] addendum pr (#263)
merge conflicts resolved manually as part of commit
Closes: #263
Co-authored-by: Wilfred Spiegelenburg <wi...@apache.org>
Signed-off-by: Wilfred Spiegelenburg <wi...@apache.org>
---
docs/developer_guide/env_setup.md | 32 +-
docs/user_guide/gang_scheduling.md | 2 +-
docs/user_guide/sorting_policies.md | 2 +-
docs/user_guide/usergroup_resolution.md | 20 +-
.../version-1.2.0/api/cluster.md | 86 +++
.../version-1.2.0/assets | 1 +
.../version-1.2.0/get_started/core_features.md | 71 ++
.../version-1.2.0/get_started/get_started.md | 76 ++
.../evaluate_perf_function_with_kubemark.md | 120 +++
.../version-1.2.0/performance/metrics.md | 105 +++
.../performance/performance_tutorial.md | 451 +++++++++++
.../version-1.2.0/performance/profiling.md | 115 +++
.../version-1.2.0/user_guide/deployment_modes.md | 48 ++
.../version-1.2.0}/user_guide/gang_scheduling.md | 6 +-
.../version-1.2.0/user_guide/placement_rules.md | 296 +++++++
.../version-1.2.0/user_guide/queue_config.md | 382 ++++++++++
.../version-1.2.0/user_guide/troubleshooting.md | 189 +++++
.../user_guide/usergroup_resolution.md | 121 +++
.../user_guide/workloads/run_flink.md | 26 +-
.../version-1.2.0/user_guide/workloads/run_mpi.md | 107 +++
.../user_guide/workloads/run_spark.md | 159 ++++
.../user_guide/workloads/run_tensorflow.md | 229 ++++++
.../user_guide/workloads/workload_overview.md | 20 +-
versioned_docs/version-1.2.0/api/scheduler.md | 62 +-
.../archived_design/cross_queue_preemption.md | 126 +++
.../version-1.2.0/archived_design/k8shim.md | 74 ++
.../archived_design/namespace_resource_quota.md | 183 +++++
.../version-1.2.0/archived_design/predicates.md | 80 ++
.../archived_design/scheduler_core_design.md | 401 ++++++++++
.../version-1.2.0/assets/RunningSparkOnK8s.png | Bin 0 -> 25878 bytes
.../version-1.2.0/assets/external_userinfo.png | Bin 0 -> 112563 bytes
.../version-1.2.0/assets/gang_scheduling_intro.png | Bin 0 -> 43907 bytes
.../version-1.2.0/assets/priority-flows.png | Bin 0 -> 233513 bytes
.../version-1.2.0/assets/priority-tree.png | Bin 0 -> 142135 bytes
.../version-1.2.0/assets/sparkResult.png | Bin 0 -> 38170 bytes
.../version-1.2.0/assets/tf-job-gpu-on-logs.png | Bin 0 -> 221691 bytes
.../version-1.2.0/assets/tf-job-gpu-on-ui.png | Bin 0 -> 101211 bytes
.../version-1.2.0/assets/translationBuilding.png | Bin 0 -> 152552 bytes
.../version-1.2.0/assets/translationDemo.png | Bin 0 -> 4243960 bytes
versioned_docs/version-1.2.0/assets/ug_lookup.png | Bin 0 -> 106036 bytes
.../assets/yunikorn-gpu-time-slicing.png | Bin 0 -> 40653 bytes
.../version-1.2.0/design/architecture.md | 4 +-
.../version-1.2.0/design/cache_removal.md | 15 +-
versioned_docs/version-1.2.0/design/config_v2.md | 345 +++++++++
.../version-1.2.0/design/priority_scheduling.md | 408 ++++++++++
.../version-1.2.0/design/user_group_handling.md | 290 +++++++
.../version-1.2.0/design/user_group_manager.md | 410 ++++++++++
.../version-1.2.0/developer_guide/build.md | 18 +-
.../version-1.2.0/developer_guide/deployment.md | 4 +-
.../version-1.2.0/developer_guide/env_setup.md | 72 +-
.../version-1.2.0/developer_guide/translation.md | 126 +++
.../version-1.2.0/get_started/core_features.md | 22 +-
.../performance/performance_tutorial.md | 2 +-
.../version-1.2.0/user_guide/gang_scheduling.md | 34 +-
.../version-1.2.0/user_guide/priorities.md | 220 ++++++
.../version-1.2.0/user_guide/queue_config.md | 117 ++-
.../user_guide/resource_quota_mgmt.md | 44 +-
.../version-1.2.0/user_guide/service_config.md | 848 +++++++++++++++++++++
.../version-1.2.0/user_guide/sorting_policies.md | 30 +-
.../{trouble_shooting.md => troubleshooting.md} | 37 +-
.../user_guide/usergroup_resolution.md | 60 +-
.../user_guide/workloads/run_flink.md | 4 +-
.../version-1.2.0/user_guide/workloads/run_mpi.md | 112 +++
.../user_guide/workloads/run_nvidia.md | 346 +++++++++
.../user_guide/workloads/run_spark.md | 54 +-
.../user_guide/workloads/run_tensorflow.md | 132 ++++
.../user_guide/workloads/workload_overview.md | 2 +
versioned_sidebars/version-1.2.0-sidebars.json | 32 +-
versions.json | 1 +
69 files changed, 7168 insertions(+), 211 deletions(-)
diff --git a/docs/developer_guide/env_setup.md b/docs/developer_guide/env_setup.md
index dcd0eb25c..98327f0ee 100644
--- a/docs/developer_guide/env_setup.md
+++ b/docs/developer_guide/env_setup.md
@@ -41,18 +41,18 @@ Once Kubernetes is started in docker desktop, you should see something similar b
This means that:
1. Kubernetes is running.
-1. the command line tool `kubctl` is installed in the `/usr/local/bin` directory.
-1. the Kubernetes context is set to `docker-desktop`.
+2. the command line tool `kubctl` is installed in the `/usr/local/bin` directory.
+3. the Kubernetes context is set to `docker-desktop`.
### Deploy and access dashboard
After setting up the local Kubernetes you need to deploy the dashboard using the following steps:
1. follow the instructions in [Kubernetes dashboard doc](https://github.com/kubernetes/dashboard) to deploy the dashboard.
-1. start the Kubernetes proxy in the background from a terminal to get access on the dashboard on the local host:
+2. start the Kubernetes proxy in the background from a terminal to get access on the dashboard on the local host:
```shell script
kubectl proxy &
```
-1. access the dashboard at the following URL: [clickable link](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login)
+3. access the dashboard at the following URL: [clickable link](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login)
### Access local Kubernetes cluster
@@ -87,27 +87,27 @@ Check hypervisor Docker Desktop should have already installed HyperKit. In a ter
chmod +x minikube
sudo mv minikube /usr/local/bin
```
-1. install HyperKit driver (required), you can either use brew or directly via these steps:
+2. install HyperKit driver (required), you can either use brew or directly via these steps:
```shell script
curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-hyperkit
sudo install -o root -g wheel -m 4755 docker-machine-driver-hyperkit /usr/local/bin/
```
-1. update the minikube config to default to the HyperKit install `minikube config set vm-driver hyperkit`
-1. change docker desktop to use minikube for Kubernetes:<br/>
+3. update the minikube config to default to the HyperKit install `minikube config set vm-driver hyperkit`
+4. change docker desktop to use minikube for Kubernetes:<br/>
### Deploy and access the cluster
After the installation is done you can start a new cluster.
1. start the minikube cluster: `minikube start --kubernetes-version v1.24.7`
-1. start the minikube dashboard: `minikube dashboard &`
+2. start the minikube dashboard: `minikube dashboard &`
### Build impact
When you create images make sure that the build is run after pointing it to the right environment.
Without setting the enviromnent minikube might not find the docker images when deploying the scheduler.
1. make sure minikube is started
-1. in the terminal where you wll run the build execute: `eval $(minikube docker-env)`
-1. run the image build from the yunikorn-k8shim repository root: `make image`
-1. deploy the scheduler as per the normal instructions.
+2. in the terminal where you wll run the build execute: `eval $(minikube docker-env)`
+3. run the image build from the yunikorn-k8shim repository root: `make image`
+4. deploy the scheduler as per the normal instructions.
## Local Kubernetes Cluster with Kind
@@ -169,17 +169,17 @@ For a generic view on how to access a multiple cluster and integrate it follow t
Or follow these simplified steps:
1. get the Kubernetes `config` file from remote cluster, copy it to the local machine and give it a unique name i.e. `config-remote`
-1. save the `KUBECONFIG` environment variable (if set)
+2. save the `KUBECONFIG` environment variable (if set)
```shell script
export KUBECONFIG_SAVED=$KUBECONFIG
```
-1. add the new file to the environment variable
+3. add the new file to the environment variable
```shell script
export KUBECONFIG=$KUBECONFIG:config-remote
```
-1. run the command `kubectl config view` to check that both configs can be accessed
-1. switch context using `kubectl config use-context my-remote-cluster`
-1. confirm that the current context is now switched to the remote cluster config:
+4. run the command `kubectl config view` to check that both configs can be accessed
+5. switch context using `kubectl config use-context my-remote-cluster`
+6. confirm that the current context is now switched to the remote cluster config:
```text
kubectl config get-contexts
CURRENT NAME CLUSTER AUTHINFO NAMESPACE
diff --git a/docs/user_guide/gang_scheduling.md b/docs/user_guide/gang_scheduling.md
index af3d7017a..21739bce2 100644
--- a/docs/user_guide/gang_scheduling.md
+++ b/docs/user_guide/gang_scheduling.md
@@ -65,7 +65,7 @@ This in effect will overrule the gang reservation and cause slowdowns and excess
### App Configuration
On Kubernetes, YuniKorn discovers apps by loading metadata from individual pod, the first pod of the app
-is required to enclosed with a full copy of app metadata. If the app does not have any notion about the first or second pod,
+is required to include a full copy of app metadata. If the app does not have any notion about the first or second pod,
then all pods are required to carry the same taskGroups info. Gang scheduling requires taskGroups definition,
which can be specified via pod annotations. The required fields are:
diff --git a/docs/user_guide/sorting_policies.md b/docs/user_guide/sorting_policies.md
index 413d1f1c2..f806f9ef1 100644
--- a/docs/user_guide/sorting_policies.md
+++ b/docs/user_guide/sorting_policies.md
@@ -41,7 +41,7 @@ Even when applications are sorted using a first in first out policy multiple app
A _parent_ queue will always use the fair policy to sort the child queues.
The relative priority of child queues (in the case of _parent_ queue sorting)
-and applciations (in the case of _leaf_ queue sorting) will be considered first.
+and applications (in the case of _leaf_ queue sorting) will be considered first.
To ignore application and queue priorities when scheduling, set the queue
property `application.sort.priority` to `disabled`.
diff --git a/docs/user_guide/usergroup_resolution.md b/docs/user_guide/usergroup_resolution.md
index 34d4e65dd..2eae43511 100644
--- a/docs/user_guide/usergroup_resolution.md
+++ b/docs/user_guide/usergroup_resolution.md
@@ -36,9 +36,9 @@ A more reliable and robust mechanism is using the `yunikorn.apache.org/user.info
Since, Kubernetes has no pre-defined field or resource for user information and individual cluster deployments with unique user identification tools can vary, we have defined a standard way of identifying the user. Yunikorn requires a Kubernetes [Label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) added. Using the [recommendation](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/) provided here, the default label is defined as below:
-| Label | Value |
-|----------------------------------------------- |--------------------- |
-| yunikorn.apache.org/username | User name. It can have duplicate entries but only the first value will be used. The default user is `nobody` |
+| Label | Value |
+|--------------------------------|--------------------------------------------------------------------------------------------------------------|
+| yunikorn.apache.org/username | User name. It can have duplicate entries but only the first value will be used. The default user is `nobody` |
Example:
```yaml
@@ -107,13 +107,13 @@ Group resolution is no longer necessary inside the shim.
The admission controller can be configured with the `yunikorn-configs` configmap. All entries start with the prefix `admissionController.accessControl.`.
-|Variable|Default value|Description|
-|--|--|--|
-|`bypassAuth`|false|Allow any external user to create pods with user information set|
-|`trustControllers`|true|Allow Kubernetes controller users to create pods with user information set|
-|`systemUsers`|"^system:serviceaccount:kube-system:"|Regular expression for the allowed controller service account list|
-|`externalUsers`|""|Regular expression for the allowed external user list|
-|`externalGroups`|""|Regular expression for the allowed external group list|
+| Variable | Default value | Description |
+|--------------------|---------------------------------------|----------------------------------------------------------------------------|
+| `bypassAuth` | false | Allow any external user to create pods with user information set |
+| `trustControllers` | true | Allow Kubernetes controller users to create pods with user information set |
+| `systemUsers` | "^system:serviceaccount:kube-system:" | Regular expression for the allowed controller service account list |
+| `externalUsers` | "" | Regular expression for the allowed external user list |
+| `externalGroups` | "" | Regular expression for the allowed external group list |
If `bypassAuth` is set to true the admission controller will not add the annotation to a pod if the annotation is not present and the deprecated user labell is set. If the annotation is not set and the user label is not set the new annotation will be added. In the case that `bypassAuth` is false, the default, the admission controller will always add the new annotation, regardless of the existence of the deprecated label.
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/api/cluster.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/api/cluster.md
new file mode 100644
index 000000000..e81a12d94
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/api/cluster.md
@@ -0,0 +1,86 @@
+---
+id: cluster
+title: 集群
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## 集群
+
+将回传Yunikorn所管理的集群的信息。信息包括应用程序的数量（包含总数、失败、挂起、正在运行、已完成）和容器的数量，以及资源管理器的构建信息。
+
+**位置(URL)** : `/ws/v1/clusters`
+
+**方法(Method)** : `GET`
+
+**需求权限**：无
+
+### 成功时的响应
+
+**回传代码**：`200 OK`
+
+**示例**
+
+在本示例中，响应的内容来自含两个节点的群集，拥有三个应用程序和四个容器以及一个资源管理器。
+
+
+```json
+[
+ {
+ "startTime": 1649167576110754000,
+ "rmBuildInformation": [
+ {
+ "buildDate": "2022-02-21T19:09:16+0800",
+ "buildVersion": "latest",
+ "isPluginVersion": "false",
+ "rmId": "rm-123"
+ }
+ ],
+ "partition": "default",
+ "clusterName": "kubernetes",
+ "totalApplications": "3",
+ "failedApplications": "1",
+ "pendingApplications": "",
+ "runningApplications": "3",
+ "completedApplications": "",
+ "totalContainers": "4",
+ "failedContainers": "",
+ "pendingContainers": "",
+ "runningContainers": "4",
+ "activeNodes": "2",
+ "totalNodes": "2",
+ "failedNodes": ""
+ }
+]
+```
+
+### 出错时的响应
+
+**回传代码**：`500 Internal Server Error`
+
+**示例**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/assets b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/assets
new file mode 120000
index 000000000..778d0f8e4
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/assets
@@ -0,0 +1 @@
+../../../../docs/assets
\ No newline at end of file
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/get_started/core_features.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/get_started/core_features.md
new file mode 100644
index 000000000..d6d3c4979
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/get_started/core_features.md
@@ -0,0 +1,71 @@
+---
+id: core_features
+title: 特征
+keywords:
+ - 特征
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn的主要特点包括：
+
+## 应用感知调度
+
+YuniKorn的关键特性之一就是支持应用感知。在默认的K8s调度程序中，它只能根据Pod进行调度，而不能基于用户、作业或者队列进行更细粒度的调度。
+与之不同的是，YuniKorn可以识别用户、作业或者队列，并在做出调度决策时，考虑更多与它们相关的因素，如资源、排序等。
+这使我们能够对资源配额、资源公平性和优先级进行细粒度控制，这是多租户计算系统最重要的需求。
+
+## 层次资源队列
+
+层次队列提供了一种有效的机制来管理集群资源。
+队列的层次结构可以在逻辑上映射到组织结构。这为不同租户提供了对资源的细粒度控制。
+YuniKorn UI 提供了一个集中的视图来监视资源队列的使用情况，它可以帮助您了解不同租户是如何使用资源的。
+此外，用户可以利用设置最小/最大队列容量来为每个租户设定其弹性资源配额。
+
+## 作业排序和排队
+
+YuniKorn将每个资源队列中的队列进行排队，排序策略决定哪个应用程序可以首先获得资源。
+这个策略可以是多种多样的，例如简单的 `FIFO`、`Fair`、`StateAware` 或基于 `Priority` 的策略。
+队列可以维持应用的顺序，调度器根据不同的策略为作业分配相应的资源。这种行为更容易被理解和控制。
+
+此外，当配置队列最大容量时，作业和任务可以在资源队列中正确排队。
+如果剩余的容量不够，它们可以排队等待，直到释放一些资源。这就简化了客户端操作。
+而在默认调度程序中，资源由命名空间资源配额限制：如果命名空间没有足够的配额，Pod就不能被创建。这是由配额许可控制器强制执行的。
+客户端需要更复杂的逻辑来处理此类场景，例如按条件重试。
+
+## 资源公平性
+
+在多租户环境中，许多用户共享集群资源。
+为了避免租户争夺资源或者可能的资源不足，需要做到更细粒度的公平性需求，以此来实现跨用户以及跨团队/组织的公平性。
+考虑到权重或优先级，一些更重要的应用可以获得超过其配额的更多的需求资源。
+这往往与资源预算有关，更细粒度的公平模式可以进一步提高资源控制。
+
+## 资源预留
+
+YuniKorn会自动为未完成的请求进行资源预留。
+如果Pod无法分配，YuniKorn将尝试把它预留在一个满足条件的节点上，并在这个节点上暂时分配该 pod（在尝试其他节点之前）。
+这种机制可以避免这个 Pod 需要的资源被后来提交的更小的、更不挑剔的 Pod 所挤占。
+此功能在批处理工作负载场景中非常重要，因为当对集群提交大量异构 Pod 时，很有可能一些 Pod 会处于“饥饿”状态，即使它们提交得更早。
+
+## 吞吐量
+
+吞吐量是衡量调度器性能的关键标准。这对于一个大规模的分布式系统来说是至关重要的。
+如果吞吐量不好，应用程序可能会浪费时间等待调度，并进一步影响服务的 SLA（服务级别协议）。
+集群越大，对吞吐量的要求也越高。[基于Kube标记的运行评估](performance/evaluate_perf_function_with_kubemark.md) 章节显示了一些性能数据。
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/get_started/get_started.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/get_started/get_started.md
new file mode 100644
index 000000000..fcf009f0f
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/get_started/get_started.md
@@ -0,0 +1,76 @@
+---
+id: user_guide
+title: 开始
+slug: /
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+在阅读本指南之前，我们假设您有一个Kubernetes集群或本地 Kubernetes 开发环境，例如 MiniKube。
+还假定 `kubectl` 在您的环境路径内，并且配置正确。
+遵循此 [指南](developer_guide/env_setup.md) 来讲述如何使用 docker-desktop 设置本地Kubernetes集群。
+
+## 安装
+
+最简单的方法是使用我们的 Helm Chart 在现有的Kubernetes集群上部署YuniKorn。
+我们建议使用 Helm 3 或更高版本。
+
+```shell script
+helm repo add yunikorn https://apache.github.io/yunikorn-release
+helm repo update
+kubectl create namespace yunikorn
+helm install yunikorn yunikorn/yunikorn --namespace yunikorn
+```
+
+默认情况下，Helm Chart 将在集群中安装调度器、web服务器和 admission-controller。
+`admission-controller` 一旦安装，它将把所有集群流量路由到YuniKorn。
+这意味着资源调度会委托给YuniKorn。在Helm安装过程中，可以通过将 `embedAdmissionController` 标志设置为 `false` 来禁用它。
+通过将Helm的 `enableSchedulerPlugin` 标志设置为 `true`，YuniKorn调度器也可以以Kubernetes的调度器插件的方式进行部署。
+这种方式将会部署一个包含与默认调度器一起编译的YuniKorn备用Docker镜像。
+这种新模式借助默认的Kubernetes调度器提供了更好的兼容性，并且适合与将所有调度委托给YuniKorn的 admission-controller 协同使用。
+因为这个模式还是很新的，所以默认没有开启。
+
+如果您不确定应该使用哪种部署模式，请参阅我们 [并列比较](user_guide/deployment_modes) 章节的内容。
+
+如果你不想使用 Helm Chart，您可以找到我们的细节教程 [点击这里](developer_guide/deployment.md) 。
+
+## 卸载
+
+运行如下的命令卸载 YuniKorn:
+
+```shell script
+helm uninstall yunikorn --namespace yunikorn
+```
+
+## 访问 Web UI
+
+当部署调度程序时，Web UI 也会部署在容器中。
+我们可以通过以下方式在标准端口上打开 Web 界面的端口转发：
+
+```shell script
+kubectl port-forward svc/yunikorn-service 9889:9889 -n yunikorn
+```
+
+`9889` 是 Web UI 的默认端口。
+完成此操作后，web UI将在以下地址可用: http://localhost:9889 。
+
+
+
+YuniKorn UI 提供了集群资源容量、利用率和所有应用信息的集中视图。
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/evaluate_perf_function_with_kubemark.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/evaluate_perf_function_with_kubemark.md
new file mode 100644
index 000000000..44f4c67eb
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/evaluate_perf_function_with_kubemark.md
@@ -0,0 +1,120 @@
+---
+id: evaluate_perf_function_with_kubemark
+title: 使用 Kubemark 评估 YuniKorn 的性能
+keywords:
+ - 性能
+ - 吞吐量
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn 社区关注调度程序的性能，并继续在发布时对其进行优化。 社区已经开发了一些工具来反复测试和调整性能。
+
+## 环境设置
+
+我们利用[Kubemark](https://github.com/kubernetes/kubernetes/blob/release-1.3/docs/devel/kubemark-guide.md#starting-a-kubemark-cluster)评估调度器的性能。 Kubemark是一个模拟大规模集群的测试工具。 它创建空节点，运行空kubelet以假装原始kubelet行为。 这些空节点上的调度pod不会真正执行。它能够创建一个满足我们实验要求的大集群，揭示yunikorn调度器的性能。 请参阅有关如何设置环境的[详细步骤](performance/performance_tutorial.md)。
+
+## 调度程序吞吐量
+
+我们在模拟的大规模环境中设计了一些简单的基准测试场景，以评估调度器的性能。 我们的工具测量[吞吐量](https://en.wikipedia.org/wiki/Throughput)并使用这些关键指标来评估性能。 简而言之，调度程序吞吐量是处理pod从在集群上发现它们到将它们分配给节点的速率。
+
+在本实验中，我们使用 [Kubemark](https://github.com/kubernetes/kubernetes/blob/release-1.3/docs/devel/kubemark-guide.md#starting-a-kubemark-cluster) 设置了一个模拟的2000/4000节点集群。然后我们启动10个[部署](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/)，每个部署分别设置5000个副本。 这模拟了大规模工作负载同时提交到K8s集群。 我们的工具会定期监控和检查pod状态，随着时间的推移，根据 `podSpec.StartTime` 计算启动的pod数量。 作为对比，我们将相同的实验应用到相同环境中的默认调度程序。 我们看到了YuniKorn相对于默认调度程序的性能优势，如下图所示：
+
+
+<p align="center">图 1. Yunikorn 和默认调度器吞吐量 </p>
+
+图表记录了集群上所有 Pod 运行所花费的时间：
+
+| 节点数 | yunikorn | k8s 默认调度器 | 差异 |
+|------------------ |:--------------: |:---------------------: |:-----: |
+| 2000(节点) | 204(pods/秒) | 49(pods/秒) | 416% |
+| 4000(节点) | 115(pods/秒) | 48(pods/秒) | 240% |
+
+为了使结果标准化，我们已经运行了几轮测试。 如上所示，与默认调度程序相比，YuniKorn实现了`2 倍`~`4 倍`的性能提升。
+
+:::note
+
+与其他性能测试一样，结果因底层硬件而异，例如服务器CPU/内存、网络带宽、I/O速度等。为了获得适用于您的环境的准确结果，我们鼓励您运行这些测试在靠近生产环境的集群上。
+
+:::
+
+## 性能分析
+
+我们从实验中得到的结果是有希望的。 我们通过观察更多的YuniKorn内部指标进一步深入分析性能，我们能够找到影响性能的几个关键区域。
+
+### K8s 限制
+
+我们发现整体性能实际上受到了K8s主服务的限制，例如api-server、controller-manager和etcd，在我们所有的实验中都没有达到YuniKorn的限制。 如果您查看内部调度指标，您可以看到：
+
+
+<p align="center">图 2. 4k 节点中的 Yunikorn 指标 </p>
+
+图2是Prometheus的截图，它记录了YuniKorn中的[内部指标](performance/metrics.md) `containerAllocation`。 它们是调度程序分配的 pod 数量，但不一定绑定到节点。 完成调度50k pod大约需要122秒，即410 pod/秒。 实际吞吐量下降到115个 Pod/秒，额外的时间用于绑定不同节点上的Pod。 如果K8s方面能赶上来，我们会看到更好的结果。 实际上，当我们在大规模集群上调整性能时，我们要做的第一件事就是调整API-server、控制器管理器中的一些参数，以提高吞吐量。 在[性能教程文档](performance/performance_tutorial.md)中查看更多信息。
+
+### 节点排序
+
+当集群大小增加时，我们看到YuniKorn的性能明显下降。 这是因为在YuniKorn中，我们对集群节点进行了完整排序，以便为给定的pod找到 **“best-fit”** 节点。 这种策略使Pod分布更加优化，基于所使用的 [节点排序策略](./../user_guide/sorting_policies#node-sorting)。 但是，对节点进行排序很昂贵，在调度周期中这样做会产生很多开销。 为了克服这个问题，我们在 [YUNIKORN-807](https://issues.apache.org/jira/browse/YUNIKORN-807) 中改进了我们的节点排序机制，其背后的想法是使用 [B-Tree ](https://en.wikipedia.org/wiki/B-tree)来存储所有节点并在必要时应用增量更新。 这显着改善了延迟，根据我们的基准测试，这在500、1000、2000 和 5000个节点的集群上分别提高了 35 倍、42 倍、51 倍、74 倍。
+
+### 每个节点的前提条件检查
+
+在每个调度周期中，另一个耗时的部分是节点的“前提条件检查”。 在这个阶段，YuniKorn评估所有K8s标准断言(Predicates)，例如节点选择器、pod亲和性/反亲和性等，以确定pod是否适合节点。 这些评估成本很高。
+
+我们做了两个实验来比较启用和禁用断言评估的情况。 请参阅以下结果：
+
+
+<p align="center">图 3. Yunikorn 中的断言效果比较 </p>
+
+当断言评估被禁用时，吞吐量会提高很多。 我们进一步研究了整个调度周期的延迟分布和断言评估延迟。 并发现：
+
+
+<p align="center">图 4. 断言延迟 </p>
+
+
+<p align="center">图 5. 启用断言的调度时间 </p>
+
+
+<p align="center">图 6. 不启用断言的调度时间 </p>
+
+总体而言，YuniKorn 调度周期运行得非常快，每个周期的延迟下降在 **0.001s - 0.01s** 范围内。 并且大部分时间用于断言评估，10倍于调度周期中的其他部分。
+
+| | 调度延迟分布（秒） | 断言-评估延迟分布（秒） |
+|----------------------- |:---------------------: |:---------------------: |
+| 启用断言 | 0.01 - 0.1 | 0.01-0.1 |
+| 不启用断言 | 0.001 - 0.01 | 无 |
+
+## 为什么 YuniKorn 更快?
+
+默认调度器被创建为面向服务的调度器； 与YuniKorn相比，它在吞吐量方面的敏感性较低。 YuniKorn社区非常努力地保持出色的性能并不断改进。 YuniKorn可以比默认调度器运行得更快的原因是：
+
+* 短调度周期
+
+YuniKorn 保持调度周期短而高效。 YuniKorn 使用所有异步通信协议来确保所有关键路径都是非阻塞调用。 大多数地方只是在进行内存计算，这可能非常高效。 默认调度器利用 [调度框架](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/)，它为扩展调度器提供了很大的灵活性，但是，权衡是性能。 调度周期变成了一条很长的链，因为它需要访问所有这些插件。
+
+* 异步事件处理
+
+YuniKorn利用异步事件处理框架来处理内部状态。 这使得核心调度周期可以快速运行而不会被任何昂贵的调用阻塞。 例如，默认调度程序需要将状态更新、事件写入pod对象，这是在调度周期内完成的。 这涉及将数据持久化到etcd，这可能很慢。 YuniKorn将所有此类事件缓存在一个队列中，并以异步方式写回pod。
+
+* 更快的节点排序
+
+[YUNIKORN-807](https://issues.apache.org/jira/browse/YUNIKORN-807)之后，YuniKorn进行了高效的增量节点排序。 这是建立在所谓的基于“资源权重”的节点评分机制之上的，它也可以通过插件进行扩展。 所有这些一起减少了计算节点分数时的开销。 相比之下，默认调度器提供了一些计算节点分数的扩展点，例如`PreScore`、`Score`和`NormalizeScore`。 这些计算量很大，并且在每个调度周期中都会调用它们。 请参阅[代码行](https://github.com/kubernetes/kubernetes/blob/481459d12dc82ab88e413886e2130c2a5e4a8ec4/pkg/scheduler/framework/runtime/framework.go#L857)中的详细信息。
+
+## 概括
+
+在测试过程中，我们发现YuniKorn的性能非常好，尤其是与默认调度程序相比。 我们已经确定了YuniKorn中可以继续提高性能的主要因素，并解释了为什么YuniKorn的性能优于默认调度程序。 我们还意识到将Kubernetes扩展到数千个节点时的局限性，可以通过使用其他技术（例如联合）来缓解这些局限性。 因此，YuniKorn是一个高效、高吞吐量的调度程序，非常适合在Kubernetes上运行批处理/混合工作负载。
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/metrics.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/metrics.md
new file mode 100644
index 000000000..7d6fa73e5
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/metrics.md
@@ -0,0 +1,105 @@
+---
+id: metrics
+title: 调度程序指标
+keywords:
+ - 指标
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn利用[Prometheus](https://prometheus.io/) 记录指标。 度量系统不断跟踪调度程序的关键执行路径，以揭示潜在的性能瓶颈。 目前，这些指标分为三类：
+
+- 调度器：调度器的通用指标，例如分配延迟、应用程序数量等。
+- 队列：每个队列都有自己的指标子系统，跟踪队列状态。
+- 事件：记录YuniKorn中事件的各种变化。
+
+所有指标都在`yunikorn`命名空间中声明。
+### 调度程序指标
+
+| 指标名称 | 指标类型 | 描述 |
+| --------------------- | ------------ | ------------ |
+| containerAllocation | Counter | 尝试分配容器的总次数。 尝试状态包括`allocated`, `rejected`, `error`, `released`。 该指标只会增加。 |
+| applicationSubmission | Counter | 提交申请的总数。 尝试的状态包括 `accepted`和`rejected`。 该指标只会增加。 |
+| applicationStatus | Gauge | 申请状态总数。 应用程序的状态包括`running`和`completed`。 |
+| totalNodeActive | Gauge | 活动节点总数。 |
+| totalNodeFailed | Gauge | 失败节点的总数。 |
+| nodeResourceUsage | Gauge | 节点的总资源使用情况，按资源名称。 |
+| schedulingLatency | Histogram | 主调度例程的延迟，以秒为单位。 |
+| nodeSortingLatency | Histogram | 所有节点排序的延迟，以秒为单位。 |
+| appSortingLatency | Histogram | 所有应用程序排序的延迟，以秒为单位。 |
+| queueSortingLatency | Histogram | 所有队列排序的延迟，以秒为单位。 |
+| tryNodeLatency | Histogram | 节点条件检查容器分配的延迟，例如放置约束，以秒为单位。 |
+
+### 队列指标
+
+| 指标名称 | 指标类型 | 描述 |
+| ------------------------- | ------------- | ----------- |
+| appMetrics | Counter | 应用程序指标，记录申请总数。 应用程序的状态包括`accepted`、`rejected`和`Completed`。 |
+| usedResourceMetrics | Gauge | 排队使用的资源。 |
+| pendingResourceMetrics | Gauge | 排队等待的资源。 |
+| availableResourceMetrics | Gauge | 与队列等相关的已用资源指标。 |
+
+### 事件指标
+
+| 指标名称 | 指标类型 | 描述 |
+| ------------------------ | ------------ | ----------- |
+| totalEventsCreated | Gauge | 创建的事件总数。 |
+| totalEventsChanneled | Gauge | 引导的事件总数。 |
+| totalEventsNotChanneled | Gauge | 引导的事件总数。 |
+| totalEventsProcessed | Gauge | 处理的事件总数。 |
+| totalEventsStored | Gauge | 存储的事件总数。 |
+| totalEventsNotStored | Gauge | 未存储的事件总数。 |
+| totalEventsCollected | Gauge | 收集的事件总数。 |
+
+## 访问指标
+
+YuniKorn指标通过Prometheus客户端库收集，并通过调度程序RESTful服务公开。
+启动后，可以通过端点http://localhost:9080/ws/v1/metrics访问它们。
+
+## Prometheus 的聚合指标
+
+设置 Prometheus 服务器以定期获取 YuniKorn 指标很简单。 按着这些次序：
+
+- 设置Prometheus（从[Prometheus 文档](https://prometheus.io/docs/prometheus/latest/installation/)了解更多信息）
+
+- 配置Prometheus规则：示例配置
+
+```yaml
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['docker.for.mac.host.internal:9080']
+```
+
+- 启动 Prometheus
+
+```shell script
+docker pull prom/prometheus:latest
+docker run -p 9090:9090 -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus
+```
+
+如果您在Mac OS上的本地docker容器中运行Prometheus，请使用`docker.for.mac.host.internal`而不是`localhost`。 启动后，打开Prometheus网页界面：http://localhost:9090/graph。 您将看到来自YuniKorn调度程序的所有可用指标。
+
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/performance_tutorial.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/performance_tutorial.md
new file mode 100644
index 000000000..8180fa071
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/performance_tutorial.md
@@ -0,0 +1,451 @@
+---
+id: performance_tutorial
+title: 基准测试教程
+keywords:
+ - 性能
+ - 教程
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## 概述
+
+YuniKorn社区不断优化调度器的性能，确保YuniKorn满足大规模批处理工作负载的性能要求。 因此，社区为性能基准测试构建了一些有用的工具，可以跨版本重用。 本文档介绍了所有这些工具和运行它们的步骤。
+
+## 硬件
+
+请注意，性能结果因底层硬件而异。 文档中发布的所有结果只能作为参考。 我们鼓励每个人在自己的环境中运行类似的测试，以便根据您自己的硬件获得结果。 本文档仅用于演示目的。
+
+本次测试中使用的服务器列表是（非常感谢[国立台中教育大学](http://www.ntcu.edu.tw/newweb/index.htm), [Kuan-Chou Lai](http://www.ntcu.edu.tw/kclai/) 为运行测试提供这些服务器):
+
+| 机型 | CPU | 内存 | 下载/上传(Mbps) |
+| --------------------- | --- | ------ | --------------------- |
+| HP | 16 | 36G | 525.74/509.86 |
+| HP | 16 | 30G | 564.84/461.82 |
+| HP | 16 | 30G | 431.06/511.69 |
+| HP | 24 | 32G | 577.31/576.21 |
+| IBM blade H22 | 16 | 38G | 432.11/4.15 |
+| IBM blade H22 | 16 | 36G | 714.84/4.14 |
+| IBM blade H22 | 16 | 42G | 458.38/4.13 |
+| IBM blade H22 | 16 | 42G | 445.42/4.13 |
+| IBM blade H22 | 16 | 32G | 400.59/4.13 |
+| IBM blade H22 | 16 | 12G | 499.87/4.13 |
+| IBM blade H23 | 8 | 32G | 468.51/4.14 |
+| WS660T | 8 | 16G | 87.73/86.30 |
+| ASUSPRO D640MB_M640SA | 4 | 8G | 92.43/93.77 |
+| PRO E500 G6_WS720T | 16 | 8G | 90/87.18 |
+| WS E500 G6_WS720T | 8 | 40G | 92.61/89.78 |
+| E500 G5 | 8 | 8G | 91.34/85.84 |
+| WS E500 G5_WS690T | 12 | 16G | 92.2/93.76 |
+| WS E500 G5_WS690T | 8 | 32G | 91/89.41 |
+| WS E900 G4_SW980T | 80 | 512G | 89.24/87.97 |
+
+每个服务器都需要执行以下步骤，否则由于用户/进程/打开文件的数量有限，大规模测试可能会失败。
+
+### 1. 设置/etc/sysctl.conf
+```
+kernel.pid_max=400000
+fs.inotify.max_user_instances=50000
+fs.inotify.max_user_watches=52094
+```
+### 2. 设置/etc/security/limits.conf
+
+```
+* soft nproc 4000000
+* hard nproc 4000000
+root soft nproc 4000000
+root hard nproc 4000000
+* soft nofile 50000
+* hard nofile 50000
+root soft nofile 50000
+root hard nofile 50000
+```
+---
+
+## 部署工作流
+
+在进入细节之前，这里是我们测试中使用的一般步骤：
+
+- [步骤 1](#Kubernetes): 正确配置Kubernetes API服务器和控制器管理器，然后添加工作节点。
+- [步骤 2](#Setup-Kubemark): 部署空pod，将模拟工作节点，命名空节点。 在所有空节点都处于就绪状态后，我们需要封锁(cordon)所有本地节点，这些本地节点是集群中的物理存在，而不是模拟节点，以避免我们将测试工作负载 pod 分配给本地节点。
+- [步骤 3](#Deploy-YuniKorn): 在主节点上使用Helm chart部署YuniKorn，并将 Deployment 缩减为 0 副本，并在`prometheus.yml`中 [修改端口](#Setup-Prometheus) 以匹配服务的端口。
+- [步骤 4](#Run-tests): 部署50k Nginx pod进行测试，API服务器将创建它们。 但是由于YuniKorn调度程序Deployment已经被缩减到0个副本，所有的Nginx pod都将停留在等待状态。
+- [步骤 5](../user_guide/troubleshooting.md#restart-the-scheduler): 将YuniKorn部署扩展回1个副本，并封锁主节点以避免YuniKorn 在那里分配Nginx pod。 在这一步中，YuniKorn将开始收集指标。
+- [步骤 6](#Collect-and-Observe-YuniKorn-metrics): 观察Prometheus UI中公开的指标。
+---
+
+## 设置 Kubemark
+
+[Kubemark](https://github.com/kubernetes/kubernetes/tree/master/test/kubemark)是一个性能测试工具，允许用户在模拟集群上运行实验。 主要用例是可扩展性测试。 基本思想是在一个物理节点上运行数十或数百个假kubelet节点，以模拟大规模集群。 在我们的测试中，我们利用 Kubemark 在少于20个物理节点上模拟多达4K节点的集群。
+
+### 1. 构建镜像
+
+##### 克隆kubernetes仓库，并构建kubemark二进制文件
+
+```
+git clone https://github.com/kubernetes/kubernetes.git
+```
+```
+cd kubernetes
+```
+```
+KUBE_BUILD_PLATFORMS=linux/amd64 make kubemark GOFLAGS=-v GOGCFLAGS="-N -l"
+```
+
+##### 将kubemark二进制文件复制到镜像文件夹并构建kubemark docker镜像
+
+```
+cp _output/bin/kubemark cluster/images/kubemark
+```
+```
+IMAGE_TAG=v1.XX.X make build
+```
+完成此步骤后，您可以获得可以模拟集群节点的kubemark镜像。 您可以将其上传到Docker-Hub或仅在本地部署。
+
+### 2. 安装Kubermark
+
+##### 创建kubemark命名空间
+
+```
+kubectl create ns kubemark
+```
+
+##### 创建configmap
+
+```
+kubectl create configmap node-configmap -n kubemark --from-literal=content.type="test-cluster"
+```
+
+##### 创建secret
+
+```
+kubectl create secret generic kubeconfig --type=Opaque --namespace=kubemark --from-file=kubelet.kubeconfig={kubeconfig_file_path} --from-file=kubeproxy.kubeconfig={kubeconfig_file_path}
+```
+### 3. 标签节点
+
+我们需要给所有的原生节点打上标签，否则调度器可能会将空pod分配给其他模拟的空节点。 我们可以利用yaml中的节点选择器将空pod分配给本地节点。
+
+```
+kubectl label node {node name} tag=tagName
+```
+
+### 4. 部署Kubemark
+
+hollow-node.yaml如下所示，我们可以配置一些参数。
+
+```
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: hollow-node
+ namespace: kubemark
+spec:
+ replicas: 2000 # 要模拟的节点数
+ selector:
+ name: hollow-node
+ template:
+ metadata:
+ labels:
+ name: hollow-node
+ spec:
+ nodeSelector: # 利用标签分配给本地节点
+ tag: tagName
+ initContainers:
+ - name: init-inotify-limit
+ image: docker.io/busybox:latest
+ imagePullPolicy: IfNotPresent
+ command: ['sysctl', '-w', 'fs.inotify.max_user_instances=200'] # 设置为与实际节点中的max_user_instance相同
+ securityContext:
+ privileged: true
+ volumes:
+ - name: kubeconfig-volume
+ secret:
+ secretName: kubeconfig
+ - name: logs-volume
+ hostPath:
+ path: /var/log
+ containers:
+ - name: hollow-kubelet
+ image: 0yukali0/kubemark:1.20.10 # 您构建的kubemark映像
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 4194
+ - containerPort: 10250
+ - containerPort: 10255
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ command:
+ - /kubemark
+ args:
+ - --morph=kubelet
+ - --name=$(NODE_NAME)
+ - --kubeconfig=/kubeconfig/kubelet.kubeconfig
+ - --alsologtostderr
+ - --v=2
+ volumeMounts:
+ - name: kubeconfig-volume
+ mountPath: /kubeconfig
+ readOnly: true
+ - name: logs-volume
+ mountPath: /var/log
+ resources:
+ requests: # 空pod的资源，可以修改。
+ cpu: 20m
+ memory: 50M
+ securityContext:
+ privileged: true
+ - name: hollow-proxy
+ image: 0yukali0/kubemark:1.20.10 # 您构建的kubemark映像
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ command:
+ - /kubemark
+ args:
+ - --morph=proxy
+ - --name=$(NODE_NAME)
+ - --use-real-proxier=false
+ - --kubeconfig=/kubeconfig/kubeproxy.kubeconfig
+ - --alsologtostderr
+ - --v=2
+ volumeMounts:
+ - name: kubeconfig-volume
+ mountPath: /kubeconfig
+ readOnly: true
+ - name: logs-volume
+ mountPath: /var/log
+ resources: # 空pod的资源，可以修改。
+ requests:
+ cpu: 20m
+ memory: 50M
+ tolerations:
+ - effect: NoExecute
+ key: node.kubernetes.io/unreachable
+ operator: Exists
+ - effect: NoExecute
+ key: node.kubernetes.io/not-ready
+ operator: Exists
+```
+
+完成编辑后，将其应用于集群：
+
+```
+kubectl apply -f hollow-node.yaml
+```
+
+---
+
+## 部署 YuniKorn
+
+#### 使用helm安装YuniKorn
+
+我们可以用 Helm 安装 YuniKorn，请参考这个[文档](https://yunikorn.apache.org/docs/#install)。 我们需要根据默认配置调整一些参数。 我们建议克隆[发布仓库](https://github.com/apache/yunikorn-release)并修改`value.yaml`中的参数。
+
+```
+git clone https://github.com/apache/yunikorn-release.git
+cd helm-charts/yunikorn
+```
+
+#### 配置
+
+`value.yaml`中的修改是：
+
+- 增加调度程序 pod 的内存/cpu 资源
+- 禁用 admission controller
+- 将应用排序策略设置为 FAIR
+
+请参阅以下更改：
+
+```
+resources:
+ requests:
+ cpu: 14
+ memory: 16Gi
+ limits:
+ cpu: 14
+ memory: 16Gi
+```
+```
+embedAdmissionController: false
+```
+```
+configuration: |
+ partitions:
+ -
+ name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ queues:
+ -
+ name: sandbox
+ properties:
+ application.sort.policy: fair
+```
+
+#### 使用本地版本库安装YuniKorn
+
+```
+Helm install yunikorn . --namespace yunikorn
+```
+
+---
+
+## 设置Prometheus
+
+YuniKorn通过Prometheus公开其调度指标。 因此，我们需要设置一个Prometheus服务器来收集这些指标。
+
+### 1. 下载Prometheus版本
+
+```
+wget https://github.com/prometheus/prometheus/releases/download/v2.30.3/prometheus-2.30.3.linux-amd64.tar.gz
+```
+```
+tar xvfz prometheus-*.tar.gz
+cd prometheus-*
+```
+
+### 2. 配置prometheus.yml
+
+```
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['docker.for.mac.host.internal:9080']
+ # 9080为内部端口，需要端口转发或修改9080为服务端口
+```
+
+### 3. 启动Prometheus
+```
+./prometheus --config.file=prometheus.yml
+```
+
+---
+## 运行测试
+
+设置环境后，您就可以运行工作负载并收集结果了。 YuniKorn社区有一些有用的工具来运行工作负载和收集指标，更多详细信息将在此处发布。
+
+---
+
+## 收集和观察YuniKorn指标
+
+Prometheus 启动后，可以轻松收集 YuniKorn 指标。 这是 YuniKorn 指标的[文档](metrics.md)。 YuniKorn 跟踪一些关键调度指标，这些指标衡量一些关键调度路径的延迟。 这些指标包括：
+
+ - **scheduling_latency_seconds:** 主调度例程的延迟，以秒为单位。
+ - **app_sorting_latency_seconds**: 所有应用程序排序的延迟，以秒为单位。
+ - **node_sorting_latency_seconds**: 所有节点排序的延迟，以秒为单位。
+ - **queue_sorting_latency_seconds**: 所有队列排序的延迟，以秒为单位。
+ - **container_allocation_attempt_total**: 尝试分配容器的总次数。 尝试状态包括 `allocated`、`rejected`、`error`、`released`。 该指标仅增加。
+
+您可以在Prometheus UI上轻松选择和生成图形，例如：
+
+
+
+
+---
+
+## 性能调优
+
+### Kubernetes
+
+默认的 K8s 设置限制了并发请求，这限制了集群的整体吞吐量。 在本节中，我们介绍了一些需要调整的参数，以提高集群的整体吞吐量。
+
+#### kubeadm
+
+设置pod网络掩码
+
+```
+kubeadm init --pod-network-cidr=10.244.0.0/8
+```
+
+#### CNI
+
+修改CNI掩码和资源。
+
+```
+ net-conf.json: |
+ {
+ "Network": "10.244.0.0/8",
+ "Backend": {
+ "Type": "vxlan"
+ }
+ }
+```
+```
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "200Mi"
+ limits:
+ cpu: "100m"
+ memory: "200Mi"
+```
+
+
+#### Api-Server
+
+在 Kubernetes API 服务器中，我们需要修改两个参数：`max-mutating-requests-inflight`和`max-requests-inflight`。 这两个参数代表API请求带宽。 因为我们会产生大量的Pod请求，所以我们需要增加这两个参数。修改`/etc/kubernetes/manifest/kube-apiserver.yaml`：
+
+```
+--max-mutating-requests-inflight=3000
+--max-requests-inflight=3000
+```
+
+#### Controller-Manager
+
+在Kubernetes控制器管理器中，我们需要增加三个参数的值：`node-cidr-mask-size`、`kube-api-burst` `kube-api-qps`. `kube-api-burst`和`kube-api-qps`控制服务器端请求带宽。`node-cidr-mask-size`表示节点 CIDR。 为了扩展到数千个节点，它也需要增加。
+
+
+Modify `/etc/kubernetes/manifest/kube-controller-manager.yaml`:
+
+```
+--node-cidr-mask-size=21 //log2(集群中的最大pod数)
+--kube-api-burst=3000
+--kube-api-qps=3000
+```
+
+#### kubelet
+
+在单个工作节点中，我们可以默认运行110个pod。 但是为了获得更高的节点资源利用率，我们需要在Kubelet启动命令中添加一些参数，然后重启它。
+
+修改`/etc/systemd/system/kubelet.service.d/10-kubeadm.conf`中的起始参数，在起始参数后面添加`--max-Pods=300`并重启。
+
+```
+systemctl daemon-reload
+systemctl restart kubelet
+```
+
+---
+
+## 概括
+
+借助Kubemark和Prometheus，我们可以轻松运行基准测试、收集YuniKorn指标并分析性能。 这有助于我们识别调度程序中的性能瓶颈并进一步消除它们。 YuniKorn社区未来将继续改进这些工具，并继续获得更多的性能改进。
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/profiling.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/profiling.md
new file mode 100644
index 000000000..eb2ae7442
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/performance/profiling.md
@@ -0,0 +1,115 @@
+---
+id: profiling
+title: 分析
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+使用[pprof](https://github.com/google/pprof)做CPU，Memory profiling可以帮助你了解YuniKorn调度器的运行状态。YuniKorn REST服务中添加了分析工具，我们可以轻松地从HTTP端点检索和分析它们。
+
+## CPU 分析
+
+在这一步，确保你已经运行了YuniKorn，它可以通过`make run`命令从本地运行，也可以部署为在K8s内运行的pod。 然后运行
+
+```
+go tool pprof http://localhost:9080/debug/pprof/profile
+```
+
+配置文件数据将保存在本地文件系统中，一旦完成，它就会进入交互模式。 现在您可以运行分析命令，例如
+
+```
+(pprof) top
+Showing nodes accounting for 14380ms, 44.85% of 32060ms total
+Dropped 145 nodes (cum <= 160.30ms)
+Showing top 10 nodes out of 106
+ flat flat% sum% cum cum%
+ 2130ms 6.64% 6.64% 2130ms 6.64% __tsan_read
+ 1950ms 6.08% 12.73% 1950ms 6.08% __tsan::MetaMap::FreeRange
+ 1920ms 5.99% 18.71% 1920ms 5.99% __tsan::MetaMap::GetAndLock
+ 1900ms 5.93% 24.64% 1900ms 5.93% racecall
+ 1290ms 4.02% 28.67% 1290ms 4.02% __tsan_write
+ 1090ms 3.40% 32.06% 3270ms 10.20% runtime.mallocgc
+ 1080ms 3.37% 35.43% 1080ms 3.37% __tsan_func_enter
+ 1020ms 3.18% 38.62% 1120ms 3.49% runtime.scanobject
+ 1010ms 3.15% 41.77% 1010ms 3.15% runtime.nanotime
+ 990ms 3.09% 44.85% 990ms 3.09% __tsan::DenseSlabAlloc::Refill
+```
+
+您可以键入诸如`web`或`gif`之类的命令来获得可以更好地帮助您的图表
+了解关键代码路径的整体性能。 你可以得到一些东西
+如下所示：
+
+
+
+注意，要使用这些选项，您需要先安装虚拟化工具`graphviz`，如果您使用的是 Mac，只需运行`brew install graphviz`，更多信息请参考[这里](https://graphviz. gitlab.io/）。
+
+## 内存分析
+
+同样，您可以运行
+
+```
+go tool pprof http://localhost:9080/debug/pprof/heap
+```
+
+这将返回当前堆的快照，允许我们检查内存使用情况。 进入交互模式后，您可以运行一些有用的命令。 比如top可以列出top内存消耗的对象。
+```
+(pprof) top
+Showing nodes accounting for 83.58MB, 98.82% of 84.58MB total
+Showing top 10 nodes out of 86
+ flat flat% sum% cum cum%
+ 32MB 37.84% 37.84% 32MB 37.84% github.com/apache/yunikorn-core/pkg/cache.NewClusterInfo
+ 16MB 18.92% 56.75% 16MB 18.92% github.com/apache/yunikorn-core/pkg/rmproxy.NewRMProxy
+ 16MB 18.92% 75.67% 16MB 18.92% github.com/apache/yunikorn-core/pkg/scheduler.NewScheduler
+ 16MB 18.92% 94.59% 16MB 18.92% github.com/apache/yunikorn-k8shim/pkg/dispatcher.init.0.func1
+ 1.04MB 1.23% 95.81% 1.04MB 1.23% k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypeWithName
+ 0.52MB 0.61% 96.43% 0.52MB 0.61% github.com/gogo/protobuf/proto.RegisterType
+ 0.51MB 0.61% 97.04% 0.51MB 0.61% sync.(*Map).Store
+ 0.50MB 0.6% 97.63% 0.50MB 0.6% regexp.onePassCopy
+ 0.50MB 0.59% 98.23% 0.50MB 0.59% github.com/json-iterator/go.(*Iterator).ReadString
+ 0.50MB 0.59% 98.82% 0.50MB 0.59% text/template/parse.(*Tree).newText
+```
+
+您还可以运行 `web`、`pdf` 或 `gif` 命令来获取堆图形。
+
+## 下载分析样本并在本地进行分析
+
+我们在调度程序docker映像中包含了基本的go/go-tool二进制文件，您应该能够进行一些基本的分析
+docker容器内的分析。 但是，如果您想深入研究一些问题，最好进行分析
+本地。 然后您需要先将示例文件复制到本地环境。 复制文件的命令如下：
+
+```
+kubectl cp ${SCHEDULER_POD_NAME}:${SAMPLE_PATH_IN_DOCKER_CONTAINER} ${LOCAL_COPY_PATH}
+```
+
+例如
+
+```
+kubectl cp yunikorn-scheduler-cf8f8dd8-6szh5:/root/pprof/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+在本地环境中获取文件后，您可以运行“pprof”命令进行分析。
+
+```
+go tool pprof /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+## 资源
+
+* pprof 文档 https://github.com/google/pprof/tree/master/doc。
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/deployment_modes.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/deployment_modes.md
new file mode 100644
index 000000000..b33e1502a
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/deployment_modes.md
@@ -0,0 +1,48 @@
+---
+id: deployment_modes
+title: 部署模式
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## YuniKorn部署模式
+
+YuniKorn有两种部署模式: `标准模式`与`插件模式`。
+在标准模式下，YuniKorn作为自定义模式运行Kubernetes调度程序。
+在插件模式下，YuniKorn被实现为默认Kubernetes调度框架之上的一组插件。
+
+在这两种情况下，建议同时运行准入控制器(admin controller)，这会确保只有一个调度程序在您的Kubernetes集群中处于活动状态。
+当准入控制器运行时，所有pod将会绕过Kubernetes的默认调度器，除了YuniKorn本身的pod。
+
+## 应该使用哪个版本？
+
+### 标准模式(Standard)
+
+目前的默认模式是标准模式。 标准模式提供稳定、高效、良好性能。
+当pod利用YuniKorn的队列功能，此模式非常适合大多数此类部署。
+
+### 插件模式(Plugin)
+
+插件模式是一种新的部署模型，调度器是在默认的Kubernetes调度逻辑之上实现的，可以更好地兼容默认的Kubernetes调度器。
+它非常适合混合工作负载，如传统的Kubernetes以及排队的应用程序。
+
+插件模式目前非常新，因此还没有达到标准模式的成熟度。
+
+要在使用 Helm 部署时激活插件模式，请将变量`enableSchedulerPlugin`设置为`true`。
diff --git a/docs/user_guide/gang_scheduling.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/gang_scheduling.md
similarity index 98%
copy from docs/user_guide/gang_scheduling.md
copy to i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/gang_scheduling.md
index af3d7017a..8a27522b5 100644
--- a/docs/user_guide/gang_scheduling.md
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/gang_scheduling.md
@@ -47,7 +47,7 @@ treated as the same kind in the scheduler.
### Prerequisite
For the queues which runs gang scheduling enabled applications, the queue sorting policy should be set to `FIFO`.
-To configure queue sorting policy, please refer to doc: [app sorting policies](sorting_policies.md#application-sorting).
+To configure queue sorting policy, please refer to doc: [app sorting policies](user_guide/sorting_policies.md#application-sorting).
#### Why the `FIFO` sorting policy
@@ -106,7 +106,7 @@ This parameter defines the reservation timeout for how long the scheduler should
The timeout timer starts to tick when the scheduler *allocates the first placeholder pod*. This ensures if the scheduler
could not schedule all the placeholder pods, it will eventually give up after a certain amount of time. So that the resources can be
freed up and used by other apps. If non of the placeholders can be allocated, this timeout won't kick-in. To avoid the placeholder
-pods stuck forever, please refer to [troubleshooting](troubleshooting.md#gang-scheduling) for solutions.
+pods stuck forever, please refer to [troubleshooting](troubleshooting.md#成组调度) for solutions.
` gangSchedulingStyle`
@@ -293,4 +293,4 @@ Check field including: namespace, pod resources, node-selector, toleration and a
## Troubleshooting
-Please see the troubleshooting doc when gang scheduling is enabled [here](troubleshooting.md#gang-scheduling).
+Please see the troubleshooting doc when gang scheduling is enabled [here](troubleshooting.md#成组调度).
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/placement_rules.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/placement_rules.md
new file mode 100644
index 000000000..17ccd279c
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/placement_rules.md
@@ -0,0 +1,296 @@
+---
+id: placement_rules
+title: App 放置规则
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+基础的放置规则(placement rules)配置[调度弃配置设计文档](../design/scheduler_configuration#placement-rules-definition)中有相关描述。
+
+我们可以将多个规则链结再一起形成一个放置策略，并通过[存取控制列表(Access control lists)](user_guide/acls.md)和规则过滤来定义和执行规则。
+
+本篇文章将通过例子来解释如何建立策略以及规则的使用，而该放置策略为调度器的一部分。
+
+## 配置
+放置规则作为调度程序队列配置的一部分，可以在各分区(partition)中被定义。规则定义的顺序就是他们会被执行的顺序。如果一个规则匹配，策略会停止执行剩余的规则。
+
+匹配的放置规则会产生一个完全合格的队列名称。同时这个回传的队列名称会从根队列(root)开始。而在队列层次结构中，可以生成的层次数量没有限制。
+
+当要执行一条规则时，已经被执行过的规则不会被列入考虑；这条规则也不会对往后要执行的规则有所影响：规则不能影响其他的规则，除非它们被配置为父规则([parent](#parent-参数))。
+
+当放置策略没有生成队列名称且没有更多的规则时，应用程式将会被拒绝。
+
+在配置中，定义放置规则的基本结构如下：
+```yaml
+placementrules:
+ - name: <第一条放置规则的名称>
+ - name: <第二条放置规则的名称>
+```
+每个[规则](#规则)都可以在配置中采取一套预定义的参数。可以被使用的规则名称在规则描述中提及。规则名称在配置中不区分大小写，且必须遵从以下的命名惯例：
+* 从字母开始：a-z 或 A-Z
+* 后面跟着0或多个字元 a-z, A-Z, 0-9 或 _
+
+当规则名称没有对应到任何已定义的规则时，表示规则处于未知的状态，会造成放置管理器初始化错误。如果参数不正确，会抛出解析错误或在初始化时发生错误。一个有错误的规则集永远不可能被激活。
+
+如果放置管理器中含有一个被激活的规则集，就代表他已经初始化了。重新载入时，若配置了一个新的规则集时，就会将新的规则集激活，并取代旧有的。如果新规则集包含一个错误，放置管理器就会忽视这个新规则集，同时也持续处于「使用不完全的规则集」的状态。如果放置管理器持续使用已激活的规则集，忽视新的且有错误的规则集时，则会记录一条关于坏掉且被忽视的配置信息。
+
+规则结果中的「点 "."」，会被文字的「"\_dot\_"」所替代。而「点」会被取代是因为在完全限定的队列名称中，他是被用来分隔队列层次的。而替代过程是发生在建构完整队列层次，且结果合格之前。这代表我们允许用「点」来表达用户名或标籤值，但不影响到队列层次。举例来说，当配置中的队列对应到「含有点的用户名」时，你必须要使用以下方法来指定它们：如果一个用户名称为`user.name`且使用`user`规则，则会产生队列名称为`root.user_dot_name`的队列。而如果该用户队列必须事先被加到配置中，则也该使用`user_dot_name`这个名字。
+
+### Create 参数
+create 参数是一个布林值，会定义当一个队列不存在时，是否可以在规则的定义下创建队列。不能保证队列一定会被创建，因为现有的队列可能会阻止创建。但如果规则定义下想创建的队列不存在，且`create: false`，那结果将会是创建失败。
+
+带有`create`参数的简易yaml范例：
+```yaml
+placementrules:
+ - name: <放置规则的名称>
+ create: true
+```
+预设值为`false`。允许的值为`true`或`false`，其他值均会导致显示错误。
+
+### Parent 参数
+`parent` 参数允许指定一个规则，并为他产生一个父队列。父规则可以被嵌套，一个父规则可以包含另一个父规则，此外对于父规则的嵌套也没有强制限制。
+
+父规则被视为顶层的规则，因此具有与放置定义中任何其他规则相同的参数和需求。例外的情况是，在「已经生成完全合格队列」的规则上再使用父规则，被认为是一个配置错误。这个错误只能发生在`fixed`型的规则上，查看规范可以得到更多细节。
+
+注意：规则的执行会沿着父规则的列表向下遍历，并首先执行列表中的最后一条。代表最后一条父规则将会直接产生在根部，详见示例。
+
+带有`parent`参数的简易yaml范例：
+```yaml
+placementrules:
+ - name: <放置规则的名称>
+ parent:
+ name: <父规则的名称>
+```
+预设值中没有parent参数。
+
+### Filter 参数
+通过`filter`参数的定义，能过滤掉适用的用户来进行配置。过滤器是一个复杂的配置組件。
+
+用户和群组可以被配置为两种类别：
+* 正则表达式
+* 用户或群组的列表
+
+如果用户或群组在yaml中的条目超过一个，则就会被视为一个用户列表或群组列表。列表中重复的条目会被忽略，不会引起错误。依据其他列表元素来使用正则表达式是不被允许的。
+
+用户和群组名称遵循标准的Linux 用户和群组惯例，对于用户名来说：
+* 从一个字母作为开头：a-z 或 A-Z
+* 后面跟着0或更多个字符：a-z, A-Z, 0-9 或 _ . @ -
+* 最后一个字符可以式$
+
+对于群组名来说：
+* 从一个字母作为开头：a-z 或 A-Z
+* 后面跟着0或更多个字符：a-z, A-Z, 0-9 或 _ -
+
+如果列表中正好有一个条目，他可以是单个用户、群组或正则表达式。当一个条目包含一个在用户或群组中不允许的字符时，该条目会被认为是一个正则表达式。正则表达式必须按照规定进行编译。一个不能编译的正则表达式将会被忽略。
+
+此外针对群组的正则表达式中：每次都是针对一个组进行匹配，而不是针对组的列表。
+
+带有 `filter`参数的简易yaml范例：
+```yaml
+filter:
+ type: deny
+ users:
+ - <user name or regexp>
+ - <user name>
+ groups:
+ - <group name or regexp>
+ - <group name>
+```
+预设值中没有filter参数。
+
+### Value 参数
+这是一个通用值，用来传递给规则以实现或改变其行为。该值被用来与[fixed](#fixed-rule)和[tag](#tag-rule)型的规则一起使用。此外`value`是以字串型式存在的单一值，也不会被系统解释或干扰。
+
+带有 `value`参数的简易yaml范例：
+```yaml
+placementrules:
+ - name: <放置规则的名称>
+ value: "any string"
+```
+预设值中没有value参数。
+
+## 准入控制列表
+准入控制列表没有在规则中定义，但它们影响放置策略的结果。有两种准入控制列表可以被定义到队列上：
+1. 提交ACL: `submitacl`
+2. 管理ACL: `adminacl`
+
+如果队列的ACL允许任何ACL提交存取，则放置规则才能匹配。管理队列ACL还提供「提交准入」的功能。如果队列不存在或没有配置ACL，则将检查父队列的ACL。这种递迴的检查会重复执行，直到有ACL提供存取或根目录的ACL被检查过后。
+
+关于ACL语法的更多细节，请查阅[ACL文档](user_guide/acls.md)。
+
+## 规则
+### Provided Rule
+在配置中使用的名称：*provided*
+
+提供在提交应用时所指定的队列。本规则的行为是─如果队列不完全合格，则完全限定由「应用程序提供的队列」作为放置的队列。如果设置了一个父规则，并且在应用程序提交中提供的队列是完全合格的，那麽父规则就不会被执行。
+
+支持的参数：
+* create
+* parent
+* filter
+
+举例：如果用户名下不存在队列，则用户传入的队列会被创建。
+```yaml
+placementrules:
+ - name: provided
+ create: true
+ parent:
+ name: user
+ create: true
+```
+当用户 `developer` 向应用程式提交请求，且应用程式要求的队列名称为：`my_special_queue`。<br/>
+结果：`root.developer.my_special_queue`(父规则将设置用户名称)。
+
+当用户 `developer` 向应用程式提交请求，且应用程式要求的队列名称为：`root.dev_queue`。<br/>
+结果：`root.dev_queue`(父规则将被忽视)。
+
+### User Name Rule
+在配置中使用的名称：*user*
+
+提供一个基于用户名的队列，而该用户名为所提交的应用程序的一部分。
+
+支持的参数：
+* create
+* parent
+* filter
+
+举例：提交一个基于用户名的队列，如果队列不存在，不要创建队列。
+```yaml
+placementrules:
+ - name: user
+ create: false
+```
+
+当用户`finance.test`提交了一个应用程式，此外对应的队列也存在。<br/>
+结果： `root.finance_dot_test`(注意，「点」被替换了)。
+
+当用户`developer`提交了一个应用程式，但对映的队列不存在。<br/>
+结果：失败，执行下一条规则。
+
+### Fixed Rule
+在配置中使用的名称：*fixed*
+
+回传一个队列，其依据规则参数的值作为名称，配置的值必须是一个合法的队列名称或层次结构。该队列名称不一定需要完全合格。名称中的层次结构使用「一个点(.)」作为层次结构中不同级别的分隔符。只有当配置的队列不存在且`create`标籤未设置时，`fixed`规则才会失败，因为他始终新增一个配置的队列。
+
+支持的参数：
+* value(reuqired)
+* create
+* parent
+* filter
+
+举例：因为总是会回传一个队列，若没有设置`create`标籤，则要在配置中将队列定义为子队列。
+```yaml
+placementrules:
+ - name: fixed
+ value: last_resort
+```
+
+当用户 `developer`提交请求，且在应用程式中设定的队列为：`my_special_queue` 。<br/>
+结果：`root.last_resort`。
+
+### Tag Rule
+在配置中使用的名称：*tag*
+
+从应用程式的标籤中检索队列名称。检查`tag`的值(value)，并在规则中使用该值进行配置。若`tag`规则没有包含任何值，则代表配置错误，但是一个应用程式不一定要设置该值。
+
+如果应用程序上没有设置对应的`tag`，则规则失败。如果从应用程序回传的`tag` value是一个完全限定的队列名称，则父规则(如果已配置)将不会执行。
+
+支持的参数：
+* value(reuqired)
+* create
+* parent
+* filter
+
+举例：根据kubernetes命名空间放置一个应用程序，该命名空间在提交时自动在应用程序中设置。
+```yaml
+placementrules:
+ - name: tag
+ value: namespace
+ create: true
+```
+
+用户`developer`在命名空间`default`，提交基于kubernetes的应用程序请求，且在应用程是中设定的队列为：`my_pecial_queue`。<br/>
+结果：`root.default`。
+
+用户`developer`在命名空间`testing`，提交基于kubernetes的应用程序请求，且在应用程是中设定的队列为：`my_pecial_queue`。<br/>
+结果：`root.testing`。
+
+用户提交非基于kubernetes的应用程序请求。<br/>
+结果：失败，执行下一条规则。
+
+## 复杂的例子
+在这个复杂的例子中，我们串连了三个规则：
+
+1. 一个`user`规则，其父规则标籤使用kubernetes命名空间，且仅供`dev`群组内的用户。
+2. 一个`tag`规则，使用kubernetes命名空间，其`parent`规则固定使用已存在的队列─ `root.namespaces`。
+3. 一个`fixed`规则，将所有到达这个点的应用程序都放在`root.default`队列中。
+
+举例：
+```yaml
+placementrules:
+ - name: user
+ create: true
+ filter:
+ type: allow
+ groups:
+ - dev*
+ parent:
+ - name: tag
+ value: namespace
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ - name: fixed
+ value: root.namespaces
+ filter:
+ type: allow
+ users:
+ - john
+ - name: fixed
+ value: root.default
+```
+用户`john`在命名空间testing中提交基于kubernetes应用程序的请求。<br/>
+结果：`root.namespaces.testing`(在规则2中匹配)。
+
+用户`sarah`在命名空间`newapp`中提交基于kubernetes应用程序的请求，组员为`sarah`, `test_app`, `dev_app`。<br/>
+结果：`root.newapp.sarah`(在规则1中匹配)。
+
+用户bob在命名空间`testapp`中提交基于kubernetes应用程序的请求，组别成员为`bob`。<br/>
+结果：`root.default`(在规则3中匹配)。
+<br/>
+在第二个例子中，我们使用了两条规则。
+
+1. 一个`fixed`规则，将所有东西放在`root.production`队列中。
+2. 一个`user`规则，以及设置 `create` 标籤。
+
+但在这个例子中，我们在`root.production`的队列上设置了ACL，只允许两个特定的用户使用这个队列。因此，即使规则匹配，除非是`john`或`bob`，否则应用程式将不会被放在`production`队列中。所有其他用户将匹配第二条规则并使用它们自己的队列，如果队列不存在，就会创建。
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: production
+ submitacl: john,bob
+ placementrules:
+ - name: fixed
+ value: root.production
+ - name: user
+ create: true
+```
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/queue_config.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/queue_config.md
new file mode 100644
index 000000000..c8a140eba
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/queue_config.md
@@ -0,0 +1,382 @@
+---
+id: queue_config
+title: 分区和队列配置
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+队列配置基础在[配置设计文档](design/scheduler_configuration.md)中有描述。
+
+本文档提供通用队列配置。
+队列配置引用了[访问控制列表](user_guide/acls.md)和[放置规则](user_guide/placement_rules.md)文档。
+
+本文档通过示例说明如何为调度程序创建分区和队列配置。
+
+调度程序依靠shim可靠地提供用户信息作为应用程序提交的一部分。
+当前shim使用[用户和组解析](usergroup_resolution)中提供的方法识别用户和用户所属的组。
+
+## 配置
+此处描述的调度器的配置文件仅提供分区和队列的配置。
+
+默认情况下，我们在部署中使用`queues.yaml`文件。
+文件名可以通过命令行更改调度器的`policyGroup`标志。
+更改文件名后必须对部署细节进行相应的更改，可以是`configmap`或包含在docker容器中的文件。
+配置的示例文件位于yunikorn-core的[queues.yaml](https://github.com/apache/yunikorn-core/blob/master/config/queues.yaml).
+
+## 分区
+分区(partitions)是调度器配置的最高级别。
+在配置中可以定义多个分区。
+
+配置中分区定义的基本结构如下：
+```yaml
+partitions:
+ - name: <第一个分区的名称>
+ - name: <第二个分区的名称>
+```
+分区的默认名称是`default`。
+分区定义包含特定shim相应调度器的完整配置。
+每个shim都使用自己独特的分区。
+
+分区必须至少需要定义以下键：
+* name
+* [queues](#队列)
+
+队列配置会在分区结束后介绍。
+
+可以选择为分区定义以下键：
+* [placementrules](#放置规则)
+* [statedumpfilepath](#状态转储文件路径)
+* [limits](#限制)
+* nodesortpolicy
+* preemption
+
+放置规则和限制在各自的章节中解释。
+
+`nodesortpolicy`定义节点为分区排序的方式。
+有关可以使用的节点排序策略值的详细信息，请参阅[排序策略](user_guide/sorting_policies.md#node-sorting)文档。
+
+抢占键目前只能有一个子键： _enabled_。
+这个布林值定义了整个分区的抢占行为。
+
+_enabled_的默认值为_false_。
+_enabled_的允许值：_true_或_false_，任何其他值都会导致解析错误。
+
+下方的示例中，`partition` yaml条目是带有_preemption_标志集和_fair_的`nodesortpolicy`。
+```yaml
+partitions:
+ - name: <分区名称>
+ nodesortpolicy: fair
+ preemption:
+ enabled: true
+```
+备注：
+目前，Kubernetes独特的shim不支持除`default`分区之外的任何其他分区。
+这已被记录为shim的[jira](https://issues.apache.org/jira/browse/YUNIKORN-22)。
+
+### 队列
+
+YuniKorn通过利用资源队列来管理资源。
+资源队列(queues)具有以下特征：
+- 队列可以有**层次**结构
+- 每个队列都可以预设**最小/最大容量**，其中最小容量定义保证资源，最大容量定义资源限制(所谓的资源配额)
+- 任务必须在某个`leaf`队列下运行
+- 队列可以是**静态**(从配置文件加载)或**动态**(由YuniKorn内部管理)
+- 队列级别的**资源公平**是由调度器强制执行
+- 作业只能在特定队列下运行
+
+:::info
+YuniKorn队列与[Kubernetes命名空间](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)的区别：
+Kubernetes命名空间提供了Kubernetes资源的范围，包括安全环境(即谁可以访问对象)，以及定义[资源配额](https://kubernetes.io/docs/concepts/policy/resource-quotas/)时的资源边界(即对象可以使用多少资源)。
+另一方面，YuniKorn队列仅使用一组作业可以使用多少资源，以及以何种顺序使用。
+YuniKorn队列在考虑资源公平性、作业排序等方面提供了更细粒度的多租户资源共享控制。
+在大多数情况下，YuniKorn队列可用于替代命名空间资源配额，以提供更多的调度特性。
+:::
+
+_queues_条目是主要的配置元素，它为队列定义了层次结构。
+
+它可以定义一个`root`队列，但它不是必需的元素。
+如果未定义`root`队列，配置解析将插入root队列以保持一致性。
+以下因素会触发root队列的插入：
+* 如果配置在顶层定义了多个队列，则会插入一个根队列。
+* 如果在顶层只定义了一个队列并且它不被称为`root`，则插入一个root队列。
+
+定义的一个或多个队列将成为插入的`root`队列的子队列。
+
+带有子队列的基本`queue` yaml条目：
+```yaml
+queues:
+- name: <队列名称>
+ queues:
+ - name: <队列名称>
+```
+
+队列支持的参数：
+* name
+* parent
+* queues
+* maxapplications
+* properties
+* adminacl
+* submitacl
+* [resources](#资源)
+* [limits](#限制)
+
+每个队列都必须有一个_name_并且队列的名称在定义队列的同一级别上必须是唯一的。
+由于队列结构是完全层次化的，层次结构中不同点的队列可能具有相同的名称。
+例如：队列结构`root.testqueue`和`root.parent.testqueue`是一个有效的结构。
+需注意的是，队列不能包含点“.”字符，因为该字符用于分隔层次结构中的队列。
+如果名称对于配置中的队列不是唯一的，或者名称包含一个点，则会生成解析错误并拒绝配置。
+
+依队列的类型是否于队列有子模板队列和子队列的事实，结构中的队列将自动获得分配的类型。
+队列类型有两种：
+* parent
+* leaf
+
+应用只能分配给_leaf_队列。
+在配置中，队列具有子模板队列或子队列将自动成为_parent_队列。
+如果队列在配置中没有子队列，则该队列是_leaf_队列，除非该队列`parent`参数设置为_true_。
+尝试覆盖配置中的_parent_队列类型将导致配置解析错误。
+
+parent队列的子队列在`queues`条目下定义。
+`queues`条目是队列级别的递归条目，它使用完全相同的参数集。
+_maxapplications_属性是一个大于 1 的整数值，它允许您限制队列中正在运行的应用的数量。
+不允许为_maxapplications_指定零，因为它会阻止队列中应用的任何分配。
+_子_队列的_maxapplications_值必须小于或等于_parent_队列的值。
+`properties`参数是一个简单的键值对列表，并为队列提供一组简单的属性。其中的键或值没有限制，任何东西都是允许的。
+目前，属性列表仅在调度器中用于定义leaf队列的[排序顺序](user_guide/sorting_policies.md#application-sorting)。
+在未来的扩展中，添加比如打开或关闭队列抢占或其他排序策略的选项，让使用相同的属性构造而无需更改配置。
+
+通过`adminacl`设置对队列的访问权限以进行管理操作，并通过`submitacl`条目提交应用。
+访问控制列表(ACLs)的描述可见[访问控制列表(ACLs)](user_guide/acls.md)文档。
+
+队列资源限制是通过`resources`参数设置。
+用户和群组的限制是通过`limits`参数设置。
+由于这两个条目都是复杂的配置条目，因此它们在下面的[resources](#资源)和[limits](#限制)中有相应解释。
+
+以下示例配置是`root.namespaces`队列作为具有限制的_parent_队列：
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: namespaces
+ parent: true
+ maxapplications: 12
+ resources:
+ guaranteed:
+ {memory: 1G, vcore: 10}
+ max:
+ {memory: 10G, vcore: 100}
+ queues:
+ - name: level1
+ maxapplications: 8
+ resources:
+ guaranteed:
+ {memory: 0.5G, vcore: 5}
+ max:
+ {memory: 5G, vcore: 50}
+```
+
+### 放置规则
+
+放置规则(placement rules)在[放置规则](user_guide/placement_rules.md)文档中有相关定义和记录。
+
+每个分区只能定义一组放置规则。
+如果没有定义规则，则放置管理器不会启动。
+此外，在提交应用时，*必须*为每个应用设置一个队列。
+
+### 状态转储文件路径
+
+状态转储文件路径(Statedump filepath)定义YuniKorn状态转储的输出文件并且它可以在分区级别上设置。
+如果设置转储文件路径，该字段的值可以是相对路径或绝对路径(此处相对路径是基于工作目录)。
+如果YuniKorn调度器没有足够的权限在指定路径创建状态转储文件，它将无法启动状态转储的功能。
+
+```yaml
+statedumpfilepath: <path/to/statedump/file>
+```
+如果上面的键没有在分区配置中指定，它的默认值为`yunikorn-state.txt`。
+需注意的是，如果键在多个分区中指定，则其第一次出现的值优先。
+
+状态转储文件也有一个固定的循环策略。
+目前，每个状态转储文件的容量为10MB，最多可以有10个这样的文件。
+当前正在写入的状态转储文件将始终是上面配置的值或默认的`yunikorn-state.txt`。
+当达到文件大小限制时，日志旋转器(`lumberjack`)将通过在文件前加上时间戳来修改文件，并创建一个具有相同的无前缀名称的新文件来写入状态转储。
+如果达到状态转储文件的最大数量，轮换策略将删除根据标记时间戳的最旧文件。
+
+### 限制
+限制(limits)为分区或队列定义一组限制对象，以及它可以在分区或任何级别的队列上设置。
+```yaml
+limits:
+ - limit: <描述>
+ - limit: <描述>
+```
+
+限制对象是一个复杂的配置对象，它为一组用户和/或群组定义一个限制。
+多个独立的限制可以设置为队列或分区上一个`limits`条目的一部分。
+不属于限制设置的用户和/或群组将不受限制。
+
+limits条目的示例：
+```yaml
+limits:
+ - limit: <描述>
+ users:
+ - <用户名或"*">
+ - <用户名>
+ groups:
+ - <群组名称或"*">
+ - <群组名称>
+ maxapplications: <1..最大值>
+ maxresources:
+ <资源名称1>: <0..最大值>[后缀]
+ <资源名称2>: <0..最大值>[后缀]
+```
+
+队列限制的情况下，应用递归限制。
+这意味着对`root`队列的限制是集群中用户或群组的总体限制。
+因此，`root`队列限制也等同于`partition`限制。
+
+limits参数：
+* limit
+* users
+* groups
+* maxapplications
+* maxresources
+
+_limit_参数是limits条目的可选描述。
+除了使配置易于理解和可读之外，它不用于任何其他用途。
+
+可以配置的_users_和_groups_可以是以下两种类型之一：
+* 一个星号字符 "*"
+* users或groups的列表。
+
+如果users或groups的条目包含超过1个条目，则它始终被视为users或groups的列表。
+星号字符“*”为通配符，匹配所有用户或群组。
+不允许在其他列表元素旁边指定星号字符。
+列表中的重复条目将被忽略，并不会导致解析错误。
+
+_maxapplications_是一个无符号整数值。
+当_maxapplications_大于1，它允许您限制为配置的用户或群组运行的应用的数量。
+不允许指定_maxapplications_为0，因为_maxapplications_为0时，隐含拒绝任何访问。
+拒绝访问的规范应交由ACL条目处理。
+
+_maxresources_参数可用于指定一个或多个资源的限制。
+_maxresources_使用与队列的[resources](#资源)参数相同的语法。
+未在列表中指定的资源不受限制。
+资源限制可以设置为 0，这可以防止用户或群组请求指定的资源，即使队列或分区具有可用的特定资源也是如此。
+不允许将总体资源限制指定为零，换言之，这意味着限制中指定的至少一个资源必须大于零。
+如果资源在队列上不可用，则应使用队列定义上的最大资源。
+指定一个实际上为零的限制，_maxapplications_ 为零并且所有资源限制为零，这是不允许的，并且会导致解析错误。
+
+每个用户或群组都有一个限制，它*不是*所有用户或群组的组合限制。
+
+举个例子：
+```yaml
+limit: "example entry"
+maxapplications: 10
+users:
+- sue
+- bob
+```
+在这种情况下，用户`sue`和`bob`都被允许运行10个应用。
+
+### 资源
+队列的resources条目可以为队列设置_guaranteed_和/或_maximum_资源，yunikorn会递归地检查资源限制。
+leaf队列的资源使用量是为该队列分配的所有资源的总和。
+parent队列的资源使用量是该parent队列下面所有队列，leaf和parent队列的资源使用量的总和。
+
+root队列没有_guaranteed_的资源设置。
+root队列的_max_资源限制自动等于集群大小。
+如果root队列设置了任何限制，则会发生解析错误。
+leaf队列在定义时不能设置任何资源限制。
+
+配置后的_max_资源对可以在任何时间点分配给队列的所有分配的大小进行了硬性限制。
+_max_资源可以设置为0，这使得资源对队列不可用。
+_guaranteed_资源用于计算队列份额和分配期间，它用作决定将分配分配给哪个队列的输入之一。
+抢占使用队列的_guaranteed_资源作为队列不能低于的基础。
+
+基本的`resources` yaml条目：
+```yaml
+resources:
+ guaranteed:
+ <资源名称1>: <0..最大值>[后缀]
+ <资源名称2>: <0..最大值>[后缀]
+ max:
+ <资源名称1>: <0..最大值>[后缀]
+ <资源名称2>: <0..最大值>[后缀]
+```
+列表中未指定的资源不受限制，对于最大(max)资源，或在保证(guaranteed)资源的情况下保证。
+
+可以为资源数量指定一个可选的后缀。
+有效的国际单位制后缀是`k`、`M`、`G`、`T`、`P` 和 `E`，用于10的幂，以及`Ki`、`Mi`、`Gi`、`Ti`、 `Pi`和`Ei`表示2的幂。
+此外，`vcore`类型的资源可能带有后缀`m`以表示millicores。 例如，`500m`是vcore的50%。
+默认情况下，`memory`类型的单位以byte为单位进行解释。
+所有其他资源类型都没有指定的基本单位。
+
+注意，以上单位行为从yunikorn 1.0开始有效
+以前的版本将`memory`解释为1000000(百万)bytes的单位，将`vcore`解释为millicores。
+
+### 子模板
+
+子模板(child template)可以在父类型队列的队列层次结构中的任何级别来定义。
+parent队列可以提供一个模板来定义它下面的动态leaf队列的行为。
+如果parent队列定义了子模板，则没有子模板的parent队列会从其parent队列继承子模板。
+
+模板中支持的配置如下所示。
+1. application sort policy
+2. max resources
+3. guaranteed resources
+4. max applications
+
+举个例子：
+```yaml
+ partitions:
+ - name: default
+ placementrules:
+ - name: provided
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ childtemplate:
+ maxapplications: 10
+ properties:
+ application.sort.policy: stateaware
+ resources:
+ guaranteed:
+ vcore: 1
+ memory: 1G
+ max:
+ vcore: 20
+ memory: 600G
+ queues:
+ - name: parent
+ parent: true
+ childtemplate:
+ resources:
+ max:
+ vcore: 21
+ memory: 610G
+ - name: notemplate
+ parent: true
+```
+在这种情况下，`root.parent.sales`将直接使用parent队列`root.parent`的子模板。
+相比之下，`root.notemplate.sales`将使用在队列`root`上设置的子模板，因为其parent队列 `root.notemplate` 从队列`root`继承了子模板。
+
+[已弃用] 如果您的集群依赖于动态叶队列可以从父级继承`application.sort.policy`的旧行为(由[YUNIKORN-195](https://issues.apache.org/jira/browse/YUNIKORN-195)引入)，请迁移到模板。
+旧的行为将在未来的版本中被删除。
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/troubleshooting.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/troubleshooting.md
new file mode 100644
index 000000000..4b47d6b0a
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/troubleshooting.md
@@ -0,0 +1,189 @@
+---
+id: troubleshooting
+title: 故障排除
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## 调度日志(Scheduler logs)
+
+### 检索调度日志
+
+调度器会将日志写入stdout/stderr，docker容器就会将这些日志重新导向到节点的本地位置，你可以从[这里](https://docs.docker.com/config/containers/logging/configure/)读到更多的文档，这些日志可以通过[kuberctl logs](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#logs)检索。如：
+
+```shell script
+//获取调度的pod
+kubectl get pod -l component=yunikorn-scheduler -n yunikorn
+NAME READY STATUS RESTARTS AGE
+yunikorn-scheduler-766d7d6cdd-44b82 2/2 Running 0 33h
+
+//检索日志
+kubectl logs yunikorn-scheduler-766d7d6cdd-44b82 yunikorn-scheduler-k8s -n yunikorn
+```
+
+在大多数的情况下，这个命令没有办法获取所有的日志，因为调度程序的日志数量庞大，您需要设置[集群级别的日志收集](https://kubernetes.io/docs/concepts/cluster-administration/logging/#cluster-level-logging-architectures)。推荐的设置方式是利用[fluentd](https://www.fluentd.org/)在外部储存(例如s3)上持久的收集日志。
+
+### 设定日志级别
+
+###
+:::note
+我们建议通过REST API来调整日志级别，如此以来我们不需要每次修改级别时重新启动调动程序的pod。但是透过编辑部署配置来设定日志级别时，需要重新启用调度程序的pod，因此强烈不建议这么做。
+:::
+
+停止调度器：
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=0
+```
+
+使用vim编辑部署配置：
+```shell script
+kubectl edit deployment yunikorn-scheduler -n yunikorn
+```
+
+在容器模板的`env`字段中加入`LOG_LEVEL`。例如将`LOG_LEVEL`设置为0会将日志纪录的级别设置为`INFO`。
+
+```yaml
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+…
+spec:
+template:
+…
+spec:
+containers:
+- env:
+- name: LOG_LEVEL
+value: '0'
+```
+
+启用调度器：
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=1
+```
+
+可使用的日志级别：
+
+| 值 | 日志级别 |
+|:-----: |:-------------: |
+| -1 | DEBUG |
+| 0 | INFO |
+| 1 | WARN |
+| 2 | ERROR |
+| 3 | DPanic |
+| 4 | Panic |
+| 5 | Fatal |
+
+## Pods卡在`Pending`状态
+
+如果Pod卡在Pending状态，则意味着调度程序找不到分配Pod的节点。造成这种情况有以下几种可能：
+
+### 1.没有节点满足pod的放置要求
+
+可以在Pod中配置一些放置限制，例如[节点选择器(node-selector)](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)、[亲合/反亲合性(affinity/anti-affinity)](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity)、对节点的[污点(taints)](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)没有一定的容忍度等。若要修正此类问题，你可以通过以下方法观察pod：
+
+```shell script
+kubectl describe pod <pod名称> -n <命名空间>
+```
+
+pod事件中包含预测失败，而这解释了为什么节点不符合分配条件
+
+### 2.队列的可用资源不足
+
+如果队列的可用资源不足，Pod将等待队列资源。检查队列是否还有空间可以给Pending pod的方法有以下几种：
+
+1 ) 从Yunikorn UI检查队列使用情况
+
+如果你不知道如何访问UI，可以参考[这里](../get_started/get_started.md#访问-web-ui)的文档。在`Queues`页面中，寻找Pod对应到的队列。你将能够看到队列中剩馀的可用资源。
+
+2 ) 检查pod事件
+
+运行`kubectl describe pod`以获取pod事件。如果你看到类似以下的事件：`Application <appID> does not fit into <队列路径> queue`。则代表pod无法分配，因为队列的资源用完了。
+
+当队列中的其他Pod完成工作或被删除时，代表目前Pending的pod能得到分配，如果Pod依旧在有足够的剩馀资源下，保持pending状态，则可能是因为他正在等待集群扩展。
+
+## 获取完整的状态
+
+Yunikorn状态储存中，包含了对每个进程中每个对象的状态。透过端点来检索，我们可以有很多有用的信息，举一个故障排除的例子：分区列表、应用程序列表(包括正在运行的、已完成的以及历史应用程序的详细信息)、节点数量、节点利用率、通用集群信息、集群利用率的详细信息、容器历史纪录和队列信息。
+
+状态是Yunikorn提供的用于故障排除的宝贵资源。
+
+有几种方法可以获得完整的状态：
+### 1.调度器URL
+
+步骤：
+*在浏览器中打开Yunikorn UI，且在URL中编辑：
+*将`/#/dashboard`取代为`/ws/v1/fullstatedump`，(例如，`http://localhost:9889/ws/v1/fullstatedump`)
+*按下回车键。
+
+透过这个简单的方法来观看即时且完整的状态。
+
+### 2.调度器的REST API
+
+使用以下的调度器REST API，能够让我们看到Yunikorn的完整状态。
+
+`curl -X 'GET'http://localhost:9889/ws/v1/fullstatedump-H 'accept: application/json'`
+
+有关储存状态的更多信息，可以参阅[检索完整状态](api/scheduler.md#retrieve-full-state-dump)的文档。
+
+## 重启调度器
+:::note
+最好的故障排除方法是─把「重启调度器」当作完全没有其他方法之下的最后一步，他不应该在搜集所有日志和状态之前使用。
+:::
+
+Yunikorn可以在重启之后恢复其状态。Yunikorn调度器的pod作为deployment部署。我们可以透过`scale`来增加和减少副本数量来重启Yunikorn调度器，方法如下：
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=0
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=1
+```
+
+## 成组调度
+### 1.没有占位符被创建，且app处于pending状态
+*原因*：这通常是因为应用程序被调度气拒绝，因此没有一个pod被调度。
+
+导致拒绝的常见原因有：
+
+1)任务群组(taskGroups)定义无效。调度程序在应用程序被提交时会进行健全性检查，以确保正确定义所有任务群组，如果这些信息的格式不正确，调度程序将拒绝该应用程序
+
+2)任务群组中定义的总最小资源量大于队列的最大资源量，因此调度程序拒绝该应用程序，因为队列的资源量无法满足它。可以通过检查pod事件中的相关消息，或调度器日志来找到更多详细的错误信息。
+
+*解决方案*：更正任务群组的定义并重新提交应用程序。
+
+### 2.有些占位符没有被分配
+*原因*：占位符也会消耗资源，如果不能全部分配，通常是队列或者集群没有足够的资源分配给它们。在这种情况下，占位符将在一定时间后被清理，该时间由调度策略参数─`placeholderTimeoutInSeconds`所定义。
+
+*解决方案*：如果占位符超时了，目前的app将会转为failed状态，无法再被调度。如果您愿意等待更长的时间，可以增加占位符超时的值。将来可能会添加倒退策略以提供重试，而不是使应用程序失败。
+
+### 3.有些占位符没有被交换
+*原因*：这通常代表应用程序的pod少于任务群组中定义的最小成员数(`minMembers`)
+
+*解决方案*：检查任务群组字段中的`minMember`并确保其设置正确。`minMember`可以小于实际的pod数，设置大于实际pod数是无效的。
+
+### 4.应用程序终止时不会清除占位符
+*原因*：所有占位符都会设置[ownerReference](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#owners-and-dependents)到应用程序的第一个真实的pod，或控制器参考。如果无法清理占位符，则意味着垃圾回收(garbage collector)的机制不正常。
+
+*解决方案*：检查占位符的`ownerReference`和Kubernetes中的垃圾收集器。
+
+## 仍然遇到问题？
+没问题！Apache Yunikorn社区将很乐意提供帮助。您可以通过以下选项联系社区：
+
+1. 将您的问题发布到dev@yunikorn.apache.org。
+2. 加入[YuniKorn slack](https://join.slack.com/t/yunikornworkspace/shared_invite/enQtNzAzMjY0OTI4MjYzLTBmMDdkYTAwNDMwNTE3NWVjZWE1OTczMWE4NDI2Yzg3MmEyZjUyYTZlMDE5M2U4ZjZhNmYyNGFmYjY4ZGYyMGE)并将您的问题发布到`#yunikorn-user`频道。
+3. 加入[社区会议](http://yunikorn.apache.org/community/get_involved#community-meetings)并直接与社区成员交谈。
\ No newline at end of file
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/usergroup_resolution.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/usergroup_resolution.md
new file mode 100644
index 000000000..7702498cd
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/usergroup_resolution.md
@@ -0,0 +1,121 @@
+---
+id: usergroup_resolution
+title: 解析用户和群组
+---
+
+<!--
+Licensed to the Apache Software Foundation(ASF)under one
+or more contributor license agreements.See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.The ASF licenses this file
+to you under the Apache License，Version 2.0(the
+"License")；you may not use this file except in compliance
+with the License.You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing，
+software distributed under the License is distributed on an
+"AS IS"BASIS，WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND，either express or implied.See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## 用户解析 (User resolution)
+
+用户信息是调度周期的重要概念，他是決定「是否将作业提交到队列」的关键指标之一。Yunikorn调度器依赖k8s Shim来提供用户信息，但在Kubernetes的世界中，没有任何对象可以识别实际用户，这是基于设计使然，可以从这个[页面](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#users-in-kubernetes)中得到更多资讯。
+
+在Yunikorn中有两种处理用户和群组的方法。第一种是传统的方式，使用标签`yunikorn.apache.org/usernname`。如果此标签设置在pod上，则该值会自动提取到shim中并被使用。群组解析也在shim中完成，但预设是不启用的。这种方法的问题有两个方面：首先，用户限制可以很容易地被绕过，因为提交者可以自由地将标签设置为任何值，因此这只能在受信任的环境中使用。其次，在shim中识别组别太慢了─将用户分配到多个群组是不可行的，因为根据身份验证机制(如X509，tokens，LDP，etc)可能很难查找用户属于哪一个群组。
+
+由于这些限制很大，为了向后兼容才保留了此方法，且将来可能会被删除。
+
+一种更可靠和健全的机制是使用`yunikorn.apache.org/user.info`，其中用户信息中的「允许的用户或群组列表」可以由从外部设置，或者准入控制(admission controller)可以自动将其附加到每个工作负载(workflow)。
+
+## 传统的用户处理
+
+### 使用`yunikorn.apache.org/username`标签
+
+由于kubernetes没有预定义的字段和资源能用于用户信息，且个别集群使用的用户识别工具可能有所不同，于是我们定义了标准的方法来识别用户。Yunikorn需要添加一个kubernetes[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)。这里提供了[推荐的使用方法](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/)，预设的标签定义如下：
+
+|标签|值|
+|----------------------------------------------- |--------------------- |
+| yunikorn.apache.org/username |用户名。可以重复该条目，但只会使用第一个值。默认的用户是`nobody` |
+
+举例：
+```yaml
+metadata:
+ labels:
+ yunikorn.apache.org/username:"john"
+```
+:::tip
+为了让该字段(field)成为唯一识别用户的指标，建议集群管理员使用用户识别工具，将这个标签设为不可变的字段。集群管理员或用户可以自由的使用任何方法或工具来增加这个字段和值。其中包含在提交时手动添加。
+:::
+
+:::note 假设
+假设：
+Yunikorn假设一个应用程序的所有pod都属于同一个用户。我们建议将用户标签添加到应用程序的每个pod。这是为了确保没有任何意外。
+:::
+
+在[K8s的部署](https://github.com/apache/yunikorn-release/blob/master/helm-charts/yunikorn/templates/deployment.yaml)时，可以通过覆盖`USER_LABEL_KEY`来改变标签`yunikorn.apache.org/username`的默认值，这在已经添加用户标签，或出于某些安全因素下必须修改标签的情况下特别有用。
+
+```yaml
+ env:
+ - name: USER_LABEL_KEY
+ value:"custom_user_label"
+```
+
+### 群组解析 (Group resolution)
+
+这里包含群组成员解析的定义，此外该功能是可以选择加入或移除的。群组不必是用户或群组对象所提供的一部分。当对象被加到缓存中时，群组将自动根据配置来解析。可以为每个分区(partition)设置连接到缓存的解析器(resolver)。
+
+预设的群组解析器是"no resolver"。此解析器仅回传用户明和与用户同名的主要群组。
+
+其他解析器：
+* 操作系统解析器
+* 测试解析器
+
+## 推荐的处理用户的新方式
+
+从Yunikorn 1.2开始，可以使用更复杂的用户/群组解析方式。
+
+在这种模式下，Yunikorn不再依赖标签`yunikorn.apache.org/username`，而是将`yunikorn.apache.org/user.info`附加到工作负载。通过简单的JSON值，定义用户名和群组：
+
+```yaml
+metadata:
+ annotations:
+ yunikorn.apache.org/user.info:"
+ {
+ username: \"yunikorn\"，
+ groups: [
+ \"developers\"，
+ \"devops\"
+ ]
+ }"
+```
+
+然而，为了加强安全性，在准入控制器中强制执行了以下操作：
+* 不是集群中的每个用户都可以附加此注解(annotation)，只有在配置允许时才可以使用。
+* 如果缺少注解，则准入控制器将自动添加。
+* 若尝试修改此注解将会被拒绝。
+
+我们不只在pods上这样做，而是在Deployments，ReplicaSet，DeamonSet，StatefulSet，Jobs和CronJobs上都这样做。
+
+此外，群组解析器不再需要放在k8s-shim中。
+
+### 配置准入控制器
+
+准入控制可以在`yunikorn-configs`的configmap中被配置。所有条目都以前缀`admissionController.accessControl.`开头。
+
+|变量|默认值|描述|
+|--|--|--|
+|`bypassAuth`|false|允许任何外部用户使用用户信息集创建pod|
+|`trustControllers`|true|允许Kubernetes控制器的用户创建带有用户信息集的pod|
+|`systemUsers`|"^system:serviceaccount:kube-system:"|允许控制器符物帐号列表使用正则表达式|
+|`externalUsers`|""|在「允许的外部用户列表」上使用正则表达式|
+|`externalGroups`|""|在「允许的外部群组列表」上使用正则表达式|
+
+如果`bypassAuth`设置为`true`，但注解不存在且设定了已遗弃的用户标签，则准入控制器将不会把注解加到pod中。如果未设置注解也未设置用户标签，则新的注解将会被加入。
+在`bypassAuth`为`false`(默认值)的情况下，准入控制器将始终添加新的注解，而不管是否存在已弃用的标签。
+
+在某些情况下，必须将用户和群组在提交时提供给Yunikorn，因为用户和群组管理由外部系统提供，查找机制并不简单。因此，在这些情况下，可以将`externalUsers`和`externalGroups`配置为正则表达式。其中，匹配的用户或群组就可以将`yunikorn.apache.org/user.info`设置为任意值。但这会影响Yunikorn内部的调度，因此必须仔细设置这些属性。
\ No newline at end of file
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/run_flink.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_flink.md
similarity index 59%
copy from versioned_docs/version-1.2.0/user_guide/workloads/run_flink.md
copy to i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_flink.md
index d20e8158a..40eb05b19 100644
--- a/versioned_docs/version-1.2.0/user_guide/workloads/run_flink.md
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_flink.md
@@ -1,7 +1,7 @@
---
id: run_flink
-title: Run Flink Jobs
-description: How to run Flink jobs with YuniKorn
+title: 运行Flink作业
+description: 如何与YuniKorn一起运行Flink作业
image: https://svn.apache.org/repos/asf/flink/site/img/logo/png/100/flink_squirrel_100_color.png
keywords:
- spark
@@ -26,21 +26,21 @@ specific language governing permissions and limitations
under the License.
-->
-It's very easy to run [Apache Flink](https://flink.apache.org/) on Kubernetes with YuniKorn. Depending on which mode is
-used to run Flink on Kubernetes, the configuration is slight different.
+使用 YuniKorn 在 Kubernetes 上运行 [Apache Flink](https://flink.apache.org/) 非常容易。
+根据在 Kubernetes 上运行 Flink 的模式不同，配置会略有不同。
-## Standalone mode
+## Standalone（独立）模式
-Please follow [Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html) to get details and examples of standalone deploy mode.
-In this mode, we can directly add required labels (applicationId and queue) in Deployment/Job spec to run flink application with YuniKorn scheduler, as well as [Run workloads with YuniKorn Scheduler](#run-workloads-with-yunikorn-scheduler).
+请关注 [Kubernetes 设置](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html) 以获取 standalone 部署模式的细节和示例。
+在这种模式下，我们可以直接在 Deployment/Job spec 中添加需要的标签（applicationId 和 queue）来使用 YuniKorn 调度器运行 flink 应用程序，以及 [使用 YuniKorn 调度器运行 workloads](#run-workloads-with-yunikorn-scheduler) .
-## Native mode
+## Native（原生）模式
-Please follow [Native Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/native_kubernetes.html) to get details and examples of native deploy mode.
-Running flink application with YuniKorn scheduler in native mode is only supported for flink 1.11 or above, we can leverage two flink configurations `kubernetes.jobmanager.labels` and `kubernetes.taskmanager.labels` to set the required labels.
-Examples:
+请关注 [原生 Kubernetes 设置](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/native_kubernetes.html) 以获取原生部署模式的细节和示例。
+只有 Flink 1.11 或更高版本才支持在 native 模式下使用 YuniKorn 调度程序运行 Flink 应用程序，我们可以利用两个 Flink 配置 `kubernetes.jobmanager.labels` 和 `kubernetes.taskmanager.labels` 来设置所需的标签。
+例子：
-* Start a flink session
+* 启动一个 Flink session
```
./bin/kubernetes-session.sh \
-Dkubernetes.cluster-id=<ClusterId> \
@@ -52,7 +52,7 @@ Examples:
-Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox
```
-* Start a flink application
+* 启动一个 Flink application
```
./bin/flink run-application -p 8 -t kubernetes-application \
-Dkubernetes.cluster-id=<ClusterId> \
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_mpi.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_mpi.md
new file mode 100644
index 000000000..374c5ca1e
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_mpi.md
@@ -0,0 +1,107 @@
+---
+id: run_mpi
+title: 运行MPI作业
+description: 在Yunikorn中运行MPI作业
+keywords:
+ - mpi
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+本指南介绍将介绍如何设置[MPI Operator](https://github.com/kubeflow/mpi-operator)，以及如何使用YuniKorn调度程序运行MPIJob。
+
+## 安装MPI操作器
+您可以使用以下命令安装MPI操作器。如果您在安装时遇到问题，请参阅[此文档](https://github.com/kubeflow/mpi-operator)了解详细信息。
+```
+kubectl create -f https://raw.githubusercontent.com/kubeflow/mpi-operator/master/deploy/v2beta1/mpi-operator.yaml
+```
+
+## 运行MPI作业
+此示例显示如何运行MPI应用程序。
+
+此程序将印出一些关于workers的基础信息，然后计算圆周率的近似值。
+
+这是一个计算圆周率的[YAML示例](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/examples/mpioperator/Pi/pi.yaml)：
+
+```yaml
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+ name: pi
+spec:
+ slotsPerWorker: 1
+ runPolicy:
+ cleanPodPolicy: Running
+ ttlSecondsAfterFinished: 60
+ sshAuthMountPath: /home/mpiuser/.ssh
+ mpiReplicaSpecs:
+ Launcher:
+ replicas: 1
+ template:
+ labels:
+ applicationId: "mpi_job_pi"
+ queue: root.mpi
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - image: mpioperator/mpi-pi
+ name: mpi-launcher
+ securityContext:
+ runAsUser: 1000
+ command:
+ - mpirun
+ args:
+ - -n
+ - "2"
+ - /home/mpiuser/pi
+ resources:
+ limits:
+ cpu: 1
+ memory: 1Gi
+ Worker:
+ replicas: 2
+ template:
+ labels:
+ applicationId: "mpi_job_pi"
+ queue: root.mpi
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - image: mpioperator/mpi-pi
+ name: mpi-worker
+ securityContext:
+ runAsUser: 1000
+ command:
+ - /usr/sbin/sshd
+ args:
+ - -De
+ - -f
+ - /home/mpiuser/.sshd_config
+ resources:
+ limits:
+ cpu: 1
+ memory: 1Gi
+```
+创建一个MPIJob。
+```
+kubectl create -f deployments/examples/mpioperator/Pi/pi.yaml
+```
+
+我们在圆周率示例中添加了Yunikorn标签，以演示如何使用yunikorn调度程序。
\ No newline at end of file
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_spark.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_spark.md
new file mode 100644
index 000000000..45791a184
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_spark.md
@@ -0,0 +1,159 @@
+---
+id: run_spark
+title: 运行Spark作业
+description: 如何使用YuniKorn运行Spark作业
+keywords:
+ - spark
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+:::note 注意
+本文档假设您已安装YuniKorn及其准入控制器。请参阅 [开始](../../get_started/get_started.md) 查看如何操作。
+:::
+
+## 为Spark准备docker镜像
+
+要在Kubernetes上运行Spark，您需要Spark的docker镜像。您可以
+1）使用Spark团队提供的docker镜像
+2）从头开始构建一个镜像。如果你想建立自己的Spark的docker镜像，您可以找到 [完整说明](https://spark.apache.org/docs/latest/building-spark.html)
+在 Spark 文档中。简化步骤:
+* 下载一个支持Kubernetes的Spark版本，URL: https://github.com/apache/spark
+* 构建支持Kubernetes的Spark版本:
+```shell script
+./buid/mvn -Pkubernetes -DskipTests clean package
+```
+建议使用[dockerhub](https://hub.docker.com/r/apache/spark/tags)中不同spark版本的官方镜像
+
+## 为Spark作业创建一个命名空间
+
+创建一个命名空间：
+
+```shell script
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: spark-test
+EOF
+```
+
+## 创建服务帐号和角色绑定
+
+在 `spark-test` 命名空间下创建 service account 和 role bindings :
+
+```shell script
+cat <<EOF | kubectl apply -n spark-test -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: spark
+ namespace: spark-test
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: spark-role
+ namespace: spark-test
+rules:
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get", "watch", "list", "create", "delete"]
+- apiGroups: [""]
+ resources: ["configmaps"]
+ verbs: ["get", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: spark-role-binding
+ namespace: spark-test
+subjects:
+- kind: ServiceAccount
+ name: spark
+ namespace: spark-test
+roleRef:
+ kind: Role
+ name: spark-role
+ apiGroup: rbac.authorization.k8s.io
+EOF
+```
+
+:::注意
+不可以在生产环境使用 `ClusterRole` 和 `ClusterRoleBinding` 去运行一个Spark作业！
+请为运行Spark作业配置更细粒度的安全上下文。有关如何配置正确的RBAC规则的详细信息，请参见[链接](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)。
+:::
+
+## 提交一个Spark作业
+
+如果这是从本地计算机运行的，您需要启动代理才能与api服务器通信。
+```shell script
+kubectl proxy
+```
+
+[dockerhub](https://hub.docker.com/r/apache/spark/tags)中有不同spark版本的官方镜像
+运行一个简单的 SparkPi 作业，假设 Spark 二进制文件本地安装在 `/usr/local` 目录中。
+```shell script
+export SPARK_HOME=/usr/local/spark/
+${SPARK_HOME}/bin/spark-submit --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --class org.apache.spark.examples.SparkPi \
+ --conf spark.executor.instances=1 \
+ --conf spark.kubernetes.namespace=spark-test \
+ --conf spark.kubernetes.executor.request.cores=1 \
+ --conf spark.kubernetes.container.image=docker.io/apache/spark:v3.3.0 \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-test:spark \
+ local:///opt/spark/examples/jars/spark-examples_2.12-3.3.0.jar
+```
+:::note
+在 [spark](https://spark.apache.org/docs/latest/running-on-kubernetes.html#configuration) 中有更多设置驱动程序和执行程序的选项。
+可以分配 applicationId 和队列路径。
+```
+--conf spark.kubernetes.executor.label.applicationId=application-spark-0001
+--conf spark.kubernetes.driver.label.applicationId=application-spark-0001
+--conf spark.kubernetes.executor.label.queue=root.default.sandbox
+--conf spark.kubernetes.driver.label.queue=root.default.sandbox
+```
+:::
+
+您可以看见Spark的driver和executors在Kubernetes上创建:
+
+
+
+spark-pi结果在 driver pod中。
+
+
+
+## 幕后发生了什么？
+
+当Spark作业提交到集群时，该作业将提交到 `spark-test` 命名空间。Spark的driver的pod将首先在此名称空间下创建。
+由于该集群已启用YuniKorn准入控制器，当driver的pod创建后，准入控制器会修改pod的规范并注入 `schedulerName=yunikorn`，
+通过这样做默认K8s调度程序将跳过此pod，而由YuniKorn调度。请查看这里[在Kubernetes配置其他调度器](https://kubernetes.io/docs/tasks/extend-kubernetes/configure-multiple-schedulers/)来了解是如何完成的.
+
+默认配置已启用放置规则，该规则会自动将 `spark-test` 命名空间映射到YuniKorn的队列 `root.spark test`。
+提交到此命名空间的所有Spark作业将首先自动提交到该队列。
+要了解有关放置规则如何工作的更多信息，请参阅文档[应用放置规则](user_guide/placement_rules.md)。
+到目前为止，名称空间定义pod的安全上下文，队列考虑到作业顺序、队列资源公平性等因素会确定如何调度作业和pod。
+注意，这是最简单的设置，不强制执行队列容量。该队列被视为具有无限容量。
+
+YuniKor在标签 `spark-app-selector` 中重复设置Spark的应用程序ID，并提交此作业去YuniKorn，同时被视为一份作业。
+当集群中有足够的资源时，作业被调度并运行。
+YuniKorn将driver的pod分配给一个节点，绑定pod并启动所有容器。
+一旦driver的pod启动，它会请求一堆executor的pod来运行它的任务。这些pod也将在相同的名称空间中创建，并且被YuniKorn所调度。
diff --git a/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_tensorflow.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_tensorflow.md
new file mode 100644
index 000000000..a6007028b
--- /dev/null
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/run_tensorflow.md
@@ -0,0 +1,229 @@
+---
+id: run_tf
+title: 运行TensorFlow作业
+description: 如何使用 YuniKorn 运行 TensorFlow 作业
+keywords:
+ - tensorflow
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+本章节概述了如何设置 [training-operator](https://github.com/kubeflow/training-operator) 以及如何使用 YuniKorn 调度器运行 Tensorflow 作业。
+training-operator 是由 Kubeflow 维护的一体化集成的训练 operator。它不仅支持 TensorFlow，还支持 PyTorch、XGboots 等。
+
+## 安装 training-operator
+您可以使用以下命令在 kubeflow 命名空间中默认安装 training operator。如果安装有问题，
+请参阅 [此文档](https://github.com/kubeflow/training-operator#installation) 来查找相关的详细信息。
+```
+kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone?ref=v1.3.0"
+```
+
+## 准备 docker 镜像
+在开始于 Kubernetes 上运行 TensorFlow 作业之前，您需要构建 docker 镜像。
+1. 从 [deployment/examples/tfjob](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/tfjob) 上下载文件
+2. 使用以下命令构建这个 docker 镜像
+
+```
+docker build -f Dockerfile -t kubeflow/tf-dist-mnist-test:1.0 .
+```
+
+## 运行一个 TensorFlow 作业
+以下是一个使用 MNIST [样例](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/examples/tfjob/tf-job-mnist.yaml) 的 TFJob yaml.
+
+```yaml
+apiVersion: kubeflow.org/v1
+kind: TFJob
+metadata:
+ name: dist-mnist-for-e2e-test
+ namespace: kubeflow
+spec:
+ tfReplicaSpecs:
+ PS:
+ replicas: 2
+ restartPolicy: Never
+ template:
+ metadata:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ queue: root.sandbox
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - name: tensorflow
+ image: kubeflow/tf-dist-mnist-test:1.0
+ Worker:
+ replicas: 4
+ restartPolicy: Never
+ template:
+ metadata:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ queue: root.sandbox
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - name: tensorflow
+ image: kubeflow/tf-dist-mnist-test:1.0
+```
+创建 TFJob
+```
+kubectl create -f deployments/examples/tfjob/tf-job-mnist.yaml
+```
+
+您可以从 YuniKorn UI 中查看作业信息。 如果您不知道如何访问 YuniKorn UI，
+请阅读此 [文档](../../get_started/get_started.md#访问-web-ui)。
+
+
+
+## 使用GPU Time-slicing
+### 前提
+要使用 Time-slicing GPU，您需要先设定丛集以让GPU和Time-slicing GPU能被使用。
+- 节点上必须连接GPU
+- Kubernetes版本为1.24
+- 丛集中需要安装 GPU drivers
+- 透过 [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/getting-started.html) 自动化的建置与管理节点中的 NVIDIA 软体组件
+- 在Kubernetes中设定 [Time-Slicing GPUs in Kubernetes](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/gpu-sharing.html)
+
+
+在安装完 GPU Operator 及 Time-slicing GPU 以后，确认pods的状态以确保所有的containers正在运行或完成：
+```shell script
+kubectl get pod -n gpu-operator
+```
+```shell script
+NAME READY STATUS RESTARTS AGE
+gpu-feature-discovery-fd5x4 2/2 Running 0 5d2h
+gpu-operator-569d9c8cb-kbn7s 1/1 Running 14 (39h ago) 5d2h
+gpu-operator-node-feature-discovery-master-84c7c7c6cf-f4sxz 1/1 Running 0 5d2h
+gpu-operator-node-feature-discovery-worker-p5plv 1/1 Running 8 (39h ago) 5d2h
+nvidia-container-toolkit-daemonset-zq766 1/1 Running 0 5d2h
+nvidia-cuda-validator-5tldf 0/1 Completed 0 5d2h
+nvidia-dcgm-exporter-95vm8 1/1 Running 0 5d2h
+nvidia-device-plugin-daemonset-7nzvf 2/2 Running 0 5d2h
+nvidia-device-plugin-validator-gj7nn 0/1 Completed 0 5d2h
+nvidia-operator-validator-nz84d 1/1 Running 0 5d2h
+```
+确认时间片设定是否被成功的使用：
+```shell script
+kubectl describe node
+```
+
+```shell script
+Capacity:
+ nvidia.com/gpu: 16
+...
+Allocatable:
+ nvidia.com/gpu: 16
+...
+```
+### 使用GPU测试TensorFlow job
+在这个段落中会在 Time-slicing GPU 的支援下，测试及验证TFJob的运行
+
+1. 新建一个workload的测试档案tf-gpu.yaml：
+ ```shell script
+ vim tf-gpu.yaml
+ ```
+ ```yaml
+ apiVersion: "kubeflow.org/v1"
+ kind: "TFJob"
+ metadata:
+ name: "tf-smoke-gpu"
+ namespace: kubeflow
+ spec:
+ tfReplicaSpecs:
+ PS:
+ replicas: 1
+ template:
+ metadata:
+ creationTimestamp:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - args:
+ - python
+ - tf_cnn_benchmarks.py
+ - --batch_size=32
+ - --model=resnet50
+ - --variable_update=parameter_server
+ - --flush_stdout=true
+ - --num_gpus=1
+ - --local_parameter_device=cpu
+ - --device=cpu
+ - --data_format=NHWC
+ image: docker.io/kubeflow/tf-benchmarks-cpu:v20171202-bdab599-dirty-284af3
+ name: tensorflow
+ ports:
+ - containerPort: 2222
+ name: tfjob-port
+ workingDir: /opt/tf-benchmarks/scripts/tf_cnn_benchmarks
+ restartPolicy: OnFailure
+ Worker:
+ replicas: 1
+ template:
+ metadata:
+ creationTimestamp: null
+ labels:
+ applicationId: "tf_job_20200521_001"
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - args:
+ - python
+ - tf_cnn_benchmarks.py
+ - --batch_size=32
+ - --model=resnet50
+ - --variable_update=parameter_server
+ - --flush_stdout=true
+ - --num_gpus=1
+ - --local_parameter_device=cpu
+ - --device=gpu
+ - --data_format=NHWC
+ image: docker.io/kubeflow/tf-benchmarks-gpu:v20171202-bdab599-dirty-284af3
+ name: tensorflow
+ ports:
+ - containerPort: 2222
+ name: tfjob-port
+ resources:
+ limits:
+ nvidia.com/gpu: 2
+ workingDir: /opt/tf-benchmarks/scripts/tf_cnn_benchmarks
+ restartPolicy: OnFailure
+ ```
+2. 创建TFJob
+ ```shell script
+ kubectl apply -f tf-gpu.yaml
+ ```
+3. 在Yunikorn中验证TFJob是否运行
+ 
+ 察看pod的日志:
+ ```shell script
+ kubectl logs logs po/tf-smoke-gpu-worker-0 -n kubeflow
+ ```
+ ```
+ .......
+ ..Found device 0 with properties:
+ ..name: NVIDIA GeForce RTX 3080 major: 8 minor: 6 memoryClockRate(GHz): 1.71
+
+ .......
+ ..Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: NVIDIA GeForce RTX 3080, pci bus id: 0000:01:00.0, compute capability: 8.6)
+ .......
+ ```
+ 
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/workload_overview.md b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/workload_overview.md
similarity index 66%
copy from versioned_docs/version-1.2.0/user_guide/workloads/workload_overview.md
copy to i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/workload_overview.md
index cae7392bf..c7b46a4c3 100644
--- a/versioned_docs/version-1.2.0/user_guide/workloads/workload_overview.md
+++ b/i18n/zh-cn/docusaurus-plugin-content-docs/version-1.2.0/user_guide/workloads/workload_overview.md
@@ -1,6 +1,6 @@
---
id: workload_overview
-title: Overview
+title: 概述
---
<!--
@@ -22,9 +22,8 @@ specific language governing permissions and limitations
under the License.
-->
-The YuniKorn scheduler is able to run any Kubernetes workload. All that is required is to ensure
-that the `schedulerName` field of a Pod specification is set to `yunikorn` and an `applicationId`
-label is set to a unique value per application:
+YuniKorn调度器支持任何的Kubernetes工作负载。所需要的只是确保将Pod规范的
+`schedulerName`字段设置为`yunikorn`并将`applicationId`标签设置为每个应用程序的唯一值：
```yaml
apiVersion: v1
@@ -46,13 +45,12 @@ spec:
memory: "100M"
```
-Additionally, if the YuniKorn admission controller is present, the `schedulerName` field may be
-omitted as it will be set automatically on newly created pods.
+此外，如果存在YuniKorn准入控制器，则可以省略“schedulerName”字段，因为它将在新创建的pod上自动设置。
-## Advanced Examples
+## 进阶例子
-Examples of more advanced use cases can be found here:
+可以在此处找到更进阶的例子：
-* [Run Spark Jobs](run_spark)
-* [Run Flink Jobs](run_flink)
-* [Run TensorFlow Jobs](run_tf)
+* [运行Spark作业](run_spark)
+* [运行Flink作业](run_flink)
+* [运行TensorFlow作业](run_tf)
diff --git a/versioned_docs/version-1.2.0/api/scheduler.md b/versioned_docs/version-1.2.0/api/scheduler.md
index ef292d1d7..873ca04cb 100644
--- a/versioned_docs/version-1.2.0/api/scheduler.md
+++ b/versioned_docs/version-1.2.0/api/scheduler.md
@@ -179,6 +179,10 @@ For the default queue hierarchy (only `root.default` leaf queue exists) a simila
"memory": 54000000,
"vcore": 80
},
+ "pendingResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
"isLeaf": "false",
"isManaged": "false",
"properties": {
@@ -215,6 +219,10 @@ For the default queue hierarchy (only `root.default` leaf queue exists) a simila
"memory": 54000000,
"vcore": 80
},
+ "pendingResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
"isLeaf": "true",
"isManaged": "false",
"properties": {
@@ -226,7 +234,13 @@ For the default queue hierarchy (only `root.default` leaf queue exists) a simila
"absUsedCapacity": {
"memory": 1,
"vcore": 0
- }
+ },
+ "maxRunningApps": 12,
+ "runningApps": 4,
+ "allocatingAcceptedApps": [
+ "app-1",
+ "app-2"
+ ]
}
],
"absUsedCapacity": {
@@ -283,6 +297,10 @@ In the example below there are three allocations belonging to two applications,
"memory": 4000000000,
"vcore": 4000
},
+ "pendingResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
"partition": "default",
"queueName": "root.default",
"submissionTime": 1648754032076020293,
@@ -382,7 +400,9 @@ In the example below there are three allocations belonging to two applications,
"replaced": 1,
"timedout": 1
}
- ]
+ ],
+ "hasReserved": false,
+ "reservations": []
},
{
"applicationID": "application-0002",
@@ -394,6 +414,10 @@ In the example below there are three allocations belonging to two applications,
"memory": 4000000000,
"vcore": 4000
},
+ "pendingResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
"partition": "default",
"queueName": "root.default",
"submissionTime": 1648754032076020293,
@@ -465,7 +489,9 @@ In the example below there are three allocations belonging to two applications,
"applicationState": "Running"
}
],
- "placeholderData": []
+ "placeholderData": [],
+ "hasReserved": false,
+ "reservations": []
}
]
```
@@ -513,6 +539,10 @@ Fetch an Application given a Partition, Queue and Application ID and displays ge
"memory": 4000000000,
"vcore": 4000
},
+ "pendingResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
"partition": "default",
"queueName": "root.default",
"submissionTime": 1648754032076020293,
@@ -612,7 +642,9 @@ Fetch an Application given a Partition, Queue and Application ID and displays ge
"replaced": 1,
"timedout": 1
}
- ]
+ ],
+ "hasReserved": false,
+ "reservations": []
}
```
@@ -795,7 +827,9 @@ Here you can see an example response from a 2-node cluster having 3 allocations.
"placeholderUsed": false
}
],
- "schedulable": true
+ "schedulable": true,
+ "isReserved": false,
+ "reservations": []
}
]
```
@@ -1448,9 +1482,9 @@ The output of this REST query can be rather large and it is a combination of tho
## Enable or disable periodic state dump
-Endpoint to enable a state dump to be written periodically. By default, it is 60 seconds. The output goes to a file called `yunikorn-state.txt`. In the current version, the file is located in the current working directory of Yunikorn and it is not configurable.
+Endpoint to enable a state dump to be written periodically.
-Trying to enable or disable this feature more than once in a row results in an error.
+**Status** : Deprecated and ignored since v1.2.0, no replacement.
**URL** : `/ws/v1/periodicstatedump/{switch}/{periodSeconds}`
@@ -1458,22 +1492,10 @@ Trying to enable or disable this feature more than once in a row results in an e
**Auth required** : NO
-The value `{switch}` can be either `disable` or `enable`. The `{periodSeconds}` defines how often state snapshots should be taken. It is expected to be a positive integer and only interpreted in case of `enable`.
-
### Success response
-**Code** : `200 OK`
+None
### Error response
**Code**: `400 Bad Request`
-
-**Content examples**
-
-```json
-{
- "status_code": 400,
- "message": "required parameter enabled/disabled is missing",
- "description": "required parameter enabled/disabled is missing"
-}
-```
diff --git a/versioned_docs/version-1.2.0/archived_design/cross_queue_preemption.md b/versioned_docs/version-1.2.0/archived_design/cross_queue_preemption.md
new file mode 100644
index 000000000..51c803308
--- /dev/null
+++ b/versioned_docs/version-1.2.0/archived_design/cross_queue_preemption.md
@@ -0,0 +1,126 @@
+---
+id: cross_queue_preemption
+title: Cross Queue Preemption
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Problems:
+
+According to lessons we learned from YARN Scheduler preemption.
+
+**Here're top bad things:**
+
+- Preemption is a shotgun instead of a sniper, when a preemption decision is made, nobody knows if preempted resources will go to demanding queue/app/user or not.
+- Preemption logic and allocation is different, we have to implement (and mimic) what we have done in scheduler allocation logic.
+
+**Here're top good things:**
+
+- Preemption is fast (thanks to the shotgun), reclaiming thousands of containers only takes ~ 1 sec.
+- We have understand how painful it is to handle DRF, multiple preemption policies (inter/intra-queue, shotgun/surgical preemption, etc.) And we have developed some good logic
+to make sure better modularization and plug-ability
+
+## Answer some questions for design/implementation choices
+
+**1\. Do we really want preemption-delay? (Or we just want to control pace)**
+
+In CS, we have preemption-delay, which select victims in preemption candidates, wait for a certain time before killing it.
+
+The purposes of preemption delay are: a. give heads-up time to apps so
+they can prepare bad things happen (unfortunately no app do anything for these heads up, at least from what I knew). b. control preemption pace.
+
+And in practice, I found it causes a lot of issues, for example when a
+cluster state keep changing, it is very hard to ensure accurate preemption.
+
+**Proposal:**
+
+Remove the preemption-delay, keep the logics of controlling preemption pace. (such as ```yarn.resourcemanager.monitor.capacity.preemption
+.total_preemption_per_round```). And we can do allocation together with preemption.
+This don't mean containers will be stopped immediately after preemption issued. Instead, RM can control delays between signal a container and kill a container. Such as grace
+termination of POD in K8s: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
+
+**2\. Do we want to do preemption for every scheduling logic, or we can do periodically?**
+
+In CS, we have preemption logic runs periodically, like every 1 sec or 3 sec.
+
+Since preemption logic involves some heavy logics, like calculating shares of queues/apps. And when doing accurate preemption, we may need to scan nodes for preemption candidate.
+Considering this, I propose to have preemption runs periodically. But it is important to note that, we need to try to use as much code as possible for
+allocation-inside-preemption, otherwise there will be too much duplicated logic and very hard to be maintained in the future.
+
+**3\. Preemption cost and function**
+
+We found it is helpful to add cost for preemption, such as container live time, priority, type of container. It could be a cost function (Which returns a numeric value) or it
+could be a comparator (which compare two allocations for preemption ask).
+
+## Pseudo code
+
+Logic of allocation (invoked every allocation cycle)
+
+```
+input:
+ - nAlloc, allocate N allocations for this allocation cycle.
+
+for partition:
+ askCandidates := findAskCandidates(nAlloc, preemption=false)
+
+ allocated, failed_to_allocated := tryAllocate(askCandidates);
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ nAlloc -= len(allocated)
+```
+
+Logic of preemption (invoked every preemption cycle)
+
+```
+// It has to be done for every preemption-policy because calculation is different.
+for preemption-policy:
+ preempt_results := policy.preempt()
+ for preempt_results:
+ send-preempt-result-to-cache-to-commit;
+ updated-missed-opportunity (allocated)
+```
+
+Inside preemption policy
+
+```
+inter-queue-preempt-policy:
+ calculate-preemption-quotas;
+
+ for partitions:
+ total_preempted := resource(0);
+
+ while total_preempted < partition-limited:
+ // queues will be sorted by allocating - preempting
+ // And ignore any key in preemption_mask
+ askCandidates := findAskCandidates(N, preemption=true)
+
+ preempt_results := tryAllocate(askCandidates, preemption=true);
+
+ total_preempted += sigma(preempt_result.allocResource)
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ update-preemption-mask(askCandidates.allocKeys - preempt_results.allocKeys)
+```
\ No newline at end of file
diff --git a/versioned_docs/version-1.2.0/archived_design/k8shim.md b/versioned_docs/version-1.2.0/archived_design/k8shim.md
new file mode 100644
index 000000000..0f9a848ee
--- /dev/null
+++ b/versioned_docs/version-1.2.0/archived_design/k8shim.md
@@ -0,0 +1,74 @@
+---
+id: k8shim
+title: Kubernetes Shim Design
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Github repo: https://github.com/apache/yunikorn-k8shim
+
+Please read the [architecture](../design/architecture.md) doc before reading this one, you will need to understand
+the 3 layer design of YuniKorn before getting to understand what is the Kubernetes shim.
+
+## The Kubernetes shim
+
+The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is responsible for translating the Kubernetes
+cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
+And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
+between the shim and the scheduler core is through the scheduler-interface.
+
+## The admission controller
+
+The admission controller runs in a separate pod, it runs a
+[mutation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook)
+and a [validation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook), where:
+
+1. The `mutation webhook` mutates pod spec by:
+ - Adding `schedulerName: yunikorn`
+ - By explicitly specifying the scheduler name, the pod will be scheduled by YuniKorn scheduler.
+ - Adding `applicationId` label
+ - When a label `applicationId` exists, reuse the given applicationId.
+ - When a label `spark-app-selector` exists, reuse the given spark app ID.
+ - Otherwise, assign a generated application ID for this pod, using convention: `yunikorn-<namespace>-autogen`. This is unique per namespace.
+ - Adding `queue` label
+ - When a label `queue` exists, reuse the given queue name. Note, if placement rule is enabled, values set in the label is ignored.
+ - Otherwise, adds `queue: root.default`
+ - Adding `disableStateAware` label
+ - If pod was assigned a generated applicationId by the admission controller, also set `disableStateAware: true`. This causes the generated application
+ to immediately transition from the `Starting` to `Running` state so that it will not block other applications.
+2. The `validation webhook` validates the configuration set in the configmap
+ - This is used to prevent writing malformed configuration into the configmap.
+ - The validation webhook calls scheduler [validation REST API](api/scheduler.md#configuration-validation) to validate configmap updates.
+
+### Admission controller deployment
+
+By default, the admission controller is deployed as part of the YuniKorn Helm chart installation. This can be disabled if necessary (though not recommended) by setting the Helm parameter `embedAdmissionController` to `false`.
+
+On startup, the admission controller performs a series of tasks to ensure that it is properly registered with Kubernetes:
+1. Loads a Kubernetes secret called `admission-controller-secrets`. This secret stores a pair of CA certificates which are used to sign the TLS server certificate used by the admission controller.
+2. If the secret cannot be found or either CA certificate is within 90 days of expiration, generates new certificate(s). If a certificate is expiring, a new one is generated with an expiration of 12 months in the future. If both certificates are missing or expiring, the second certificate is generated with an expiration of 6 months in the future. This ensures that both certificates do not expire at the same time, and that there is an overlap of trusted certificates.
+3. If the CA certificates were created or updated, writes the secrets back to Kubernetes.
+4. Generates an ephemeral TLS server certificate signed by the CA certificate with the latest expiration date.
+5. Validates, and if necessary, creates or updates the Kubernetes webhook configurations named `yunikorn-admission-controller-validations` and `yunikorn-admission-controller-mutations`. If the CA certificates have changed, the webhooks will also be updated. These webhooks allow the Kubernetes API server to connect to the admission controller service to perform configmap validations and pod mutations.
+6. Starts up the admission controller HTTPS server.
+
+Additionally, the admission controller also starts a background task to wait for CA certificates to expire. Once either certificate is expiring within the next 30 days, new CA and server certificates are generated, the webhook configurations are updated, and the HTTPS server is quickly restarted. This ensures that certificates rotate properly without downtime.
+
+In production clusters, it is recommended to deploy the admission controller with two replicas by setting the Helm parameter `admissionController.replicaCount` to `2`. This will ensure that at least one admission controller webhook is reachable by the Kubernetes API server at all times. In this configuration, the CA certificates and webhook configurations are shared between the instances.
diff --git a/versioned_docs/version-1.2.0/archived_design/namespace_resource_quota.md b/versioned_docs/version-1.2.0/archived_design/namespace_resource_quota.md
new file mode 100644
index 000000000..90830b6a1
--- /dev/null
+++ b/versioned_docs/version-1.2.0/archived_design/namespace_resource_quota.md
@@ -0,0 +1,183 @@
+---
+id: namespace_resource_quota
+title: Namespace Resource Quota
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+In K8s, user can setup namespace with [resource quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) to limit aggregated resource consumption in this namespace. The validation of namespace resource quotas is handled in api-server directly, therefore YuniKorn simply honors the quotas like the default scheduler.
+
+## Best practice
+
+It is not mandatory to setup YuniKorn queues with respect of namespaces.
+However, in practice, it makes more sense to do so.
+Namespace is often used to set a cap for resource consumptions per user-group/team,
+YuniKorn queue is also meant to divide cluster resource into multiple groups.
+Let's go through an example.
+
+### 1. Setup namespace
+
+Namespace: `advertisement`:
+```
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+ name: advertisement
+spec:
+ hard:
+ requests.cpu: "200m"
+ requests.memory: 2000Mi
+ limits.cpu: "200m"
+ limits.memory: 4000Mi
+```
+Create the namespace
+```
+kubectl create namespace advertisement
+kubectl create -f ./advertisement.yaml --namespace=advertisement
+kubectl get quota --namespace=advertisement
+kubectl describe quota advertisement --namespace=advertisement
+
+// output
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 0 200m
+limits.memory 0 4000Mi
+requests.cpu 0 200m
+requests.memory 0 2000Mi
+```
+
+### 2. Setup YuniKorn queues
+
+Queue: `advertisement`:
+```
+name: advertisement
+resources:
+ guaranteed:
+ vcore: 100
+ memory: 1000
+ max:
+ vcore: 200
+ memory: 2000
+```
+
+ensure `QueueMaxResource <= NamespaceResourceQuotaRequests`
+
+### 3. Mapping applications to queues & namespace
+
+In a pod spec
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00001"
+ queue: "root.advertisement"
+ name: task0
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "800M"
+ limits:
+ cpu: "100m"
+ memory: "1000M"
+```
+
+Check Quota
+
+```
+kubectl describe quota advertisement --namespace=advertisement
+
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 100m 200m
+limits.memory 1G 4000Mi
+requests.cpu 50m 200m
+requests.memory 800M 2000Mi
+```
+
+Now submit another application,
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00002"
+ queue: "root.advertisement"
+ name: task1
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "200m"
+ memory: "800M"
+ limits:
+ cpu: "200m"
+ memory: "1000M"
+```
+
+pod will not be able to submitted to api-server, because the requested cpu `200m` + used cpu `100m` = `300m` which exceeds the resource quota.
+
+```
+kubectl create -f pod_ns_adv_task1.yaml
+Error from server (Forbidden): error when creating "pod_ns_adv_task1.yaml": pods "task1" is forbidden: exceeded quota: advertisement, requested: limits.cpu=200m,requests.cpu=200m, used: limits.cpu=100m,requests.cpu=50m, limited: limits.cpu=200m,requests.cpu=200m
+```
+
+## Future Work
+
+For compatibility, we should respect namespaces and resource quotas.
+Resource quota is overlapped with queue configuration in many ways,
+for example the `requests` quota is just like queue's max resource. However,
+there are still a few features resource quota can do but queue cannot, such as
+
+1. Resource `limits`. The aggregated resource from all pods in a namespace cannot exceed this limit.
+2. Storage Resource Quota, e.g storage size, PVC number, etc.
+3. Object Count Quotas, e.g count of PVCs, services, configmaps, etc.
+4. Resource Quota can map to priority class.
+
+Probably we can build something similar to cover (3) in this list.
+But it would be hard to completely support all these cases.
+
+But currently, setting applications mapping to a queue as well as a corresponding namespace is over complex.
+Some future improvements might be:
+
+1. Automatically detects namespaces in k8s-shim and map them to queues. Behind the scenes, we automatically generates queue configuration based on namespace definition. Generated queues are attached under root queue.
+2. When new namespace added/updated/removed, similarly to (1), we automatically update queues.
+3. User can add more configuration to queues, e.g add queue ACL, add child queues on the generated queues.
+4. Applications submitted to namespaces are transparently submitted to corresponding queues.
\ No newline at end of file
diff --git a/versioned_docs/version-1.2.0/archived_design/predicates.md b/versioned_docs/version-1.2.0/archived_design/predicates.md
new file mode 100644
index 000000000..9233a25f3
--- /dev/null
+++ b/versioned_docs/version-1.2.0/archived_design/predicates.md
@@ -0,0 +1,80 @@
+---
+id: predicates
+title: Support K8s Predicates
+---
+
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+## Design
+
+Predicates are a set of pre-registered functions in K8s, the scheduler invokes these functions to check if a pod
+is eligible to be allocated onto a node. Common predicates are: node-selector, pod affinity/anti-affinity etc. To support
+these predicates in YuniKorn, we don't intend to re-implement everything on our own, but to re-use the core predicates
+code as much as possible.
+
+YuniKorn-core is agnostic about underneath RMs, so the predicates functions are implemented in K8s-shim as a `SchedulerPlugin`.
+SchedulerPlugin is a way to plug/extend scheduler capabilities. Shim can implement such plugin and register itself to
+yunikorn-core, so plugged function can be invoked in the scheduler core. Find all supported plugins in
+[types](https://github.com/apache/yunikorn-core/blob/master/pkg/plugins/types.go).
+
+## Workflow
+
+First, RM needs to register itself to yunikorn-core, it advertises what scheduler plugin interfaces are supported.
+E.g a RM could implement `PredicatePlugin` interface and register itself to yunikorn-core. Then yunikorn-core will
+call PredicatePlugin API to run predicates before making allocation decisions.
+
+
+Following workflow demonstrates how allocation looks like when predicates are involved.
+
+```
+pending pods: A, B
+shim sends requests to core, including A, B
+core starts to schedule A, B
+ partition -> queue -> app -> request
+ schedule A (1)
+ run predicates (3)
+ generate predicates metadata (4)
+ run predicate functions one by one with the metadata
+ success
+ proposal: A->N
+ schedule B (2)
+ run predicates (calling shim API)
+ generate predicates metadata
+ run predicate functions one by one with the metadata
+ success
+ proposal: B->N
+commit the allocation proposal for A and notify k8s-shim
+commit the allocation proposal for B and notify k8s-shim
+shim binds pod A to N
+shim binds pod B to N
+```
+
+(1) and (2) are running in parallel.
+
+(3) yunikorn-core calls a `schedulerPlugin` API to run predicates, this API is implemented on k8s-shim side.
+
+(4) K8s-shim generates metadata based on current scheduler cache, the metadata includes some intermittent states about nodes and pods.
+
+## Predicates White-list
+
+Intentionally, we only support a white-list of predicates. Majorly due to 2 reasons,
+* Predicate functions are time-consuming, it has negative impact on scheduler performance. To support predicates that are only necessary can minimize the impact. This will be configurable via CLI options;
+* The implementation depends heavily on K8s default scheduler code, though we reused some unit tests, the coverage is still a problem. We'll continue to improve the coverage when adding new predicates.
+
+The white-list currently is defined in [PredicateManager](https://github.com/apache/yunikorn-k8shim/blob/master/pkg/plugin/predicates/predicate_manager.go).
diff --git a/versioned_docs/version-1.2.0/archived_design/scheduler_core_design.md b/versioned_docs/version-1.2.0/archived_design/scheduler_core_design.md
new file mode 100644
index 000000000..c3da548c0
--- /dev/null
+++ b/versioned_docs/version-1.2.0/archived_design/scheduler_core_design.md
@@ -0,0 +1,401 @@
+---
+id: scheduler_core_design
+title: Scheduler Core Design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+:::caution
+The scheduler core design has changed. [YUNIKORN-317](https://issues.apache.org/jira/browse/YUNIKORN-317) was committed and has removed the scheduler cache.
+This document will not be maintained and is just for historical reference.
+See [scheduler cache removal design](../design/cache_removal.md)
+:::
+
+Github repo: https://github.com/apache/yunikorn-core
+
+Scheduler core encapsulates all scheduling algorithms, it collects resources from underneath resource management
+platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
+best spot for each request and then sends response allocations to the resource management platform.
+Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/yunikorn-scheduler-interface).
+
+## Components:
+
+```
+
+ +---------------+ +--------------+
+ |K8s Shim | |YARN Shim |
+ +---------------+ +--------------+
+
+ +--------------+ +------------+
+ Scheduler- | GRPC Protocol| |Go API |
+ Interface: +--------------+ +------------+
+
++---------------------------------------------------------------------------+
+ +--------------------+
+ |Scheduler API Server|
+ +-------------+ +---------+----------+
+ |AdminService | |
+ +-------------+ |Write Ops +----------------+
+ +-------------+ V ++Scheduler |
+ |Configurator | +-------------------+ Allocate || And |
+ +-------------+ |Cache Event Handler+<-----------------| |
+ +----------> +-------------------+ Preempt ++Preemptor |
+ Update Cfg Handled by policies +----------------+
+ + (Stateless)
+ +------v--------+
+ |Scheduler Cache|
+ +---------------+
+ +---------------------------------------------+
+ |--------+ +------+ +----------+ +----------+ |
+ ||Node | |Queue | |Allocation| |Requests | |
+ |--------+ +------+ +----------+ +----------+ |
+ +---------------------------------------------+
+```
+
+### Scheduler API Server (RMProxy)
+
+Responsible for communication between RM and Scheduler, which implements scheduler-interface GRPC protocol,
+or just APIs. (For intra-process communication w/o Serde).
+
+### Scheduler Cache
+
+Caches all data related to scheduler state, such as used resources of each queues, nodes, allocations.
+Relationship between allocations and nodes, etc. Should not include in-flight data for resource allocation.
+For example to-be-preempted allocation candidates. Fair share resource of queues, etc.
+
+### Scheduler Cache Event Handler
+
+Handles all events which needs to update scheduler internal state. So all the write operations will be carefully handled.
+
+### Admin Service
+
+Handles request from Admin, which can also load configurations from storage and update scheduler policies.
+
+### Scheduler and Preemptor
+
+Handles Scheduler's internal state. (Which is not belong to scheduelr cache), such as internal reservations, etc.
+Scheduler and preemptor will work together, make scheduling or preemption decisions. All allocate/preempt request
+will be handled by event handler.
+
+## Scheduler's responsibility
+
+- According to resource usages between queues, sort queues, applications, and figure out order of application allocation. (This will be used by preemption as well).
+- It is possible that we cannot satisfy some of the allocation request, we need to skip them and find next request.
+- It is possible that some allocation request cannot be satisfied because of resource fragmentation. We need to reserve room for such requests.
+- Different nodes may belong to different disjoint partitions, we can make independent scheduler runs
+- Be able to config and change ordering policies for apps, queues.
+- Application can choose their own way to manage sort of nodes.
+
+## Preemption
+
+- It is important to know "who wants the resource", so we can do preemption based on allocation orders.
+- When do preemption, it is also efficient to trigger allocation op. Think about how to do it.
+- Preemption needs to take care about queue resource balancing.
+
+## Communication between Shim and Core
+
+YuniKorn-Shim (like https://github.com/apache/yunikorn-k8shim) communicates with core by
+using scheduler-interface (https://github.com/apache/yunikorn-scheduler-interface).
+Scheduler interface has Go API or GRPC. Currently, yunikorn-k8shim is using Go API to communicate with yunikorn-core
+to avoid extra overhead introduced by GRPC.
+
+**Shim (like K8shim) first need to register with core:**
+
+```go
+func (m *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerRequest, callback api.ResourceManagerCallback) (*si.RegisterResourceManagerResponse, error)
+```
+
+Which indicate ResourceManager's name, a callback function for updateResponse. The design of core is be able to do scheduling for multiple clusters (such as multiple K8s cluster) just with one core instance.
+
+**Shim interacts with core by invoking RMProxy's Update API frequently, which updates new allocation request, allocation to kill, node updates, etc.**
+
+```go
+func (m *RMProxy) Update(request *si.UpdateRequest) error
+```
+
+Response of update (such as new allocated container) will be received by registered callback.
+
+## Configurations & Semantics
+
+Example of configuration:
+
+- Partition is name space.
+- Same queues can under different partitions, but enforced to have same hierarchy.
+
+ Good:
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b c
+ ```
+
+ Good (c in partition y acl=""):
+
+ ```
+ partition=x partition=y
+ a a
+ / \ /
+ b c b
+ ```
+
+ Bad (c in different hierarchy)
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b d
+ /
+ c
+ ```
+
+ Bad (Duplicated c)
+
+ ```
+ partition=x
+ a
+ / \
+ b c
+ /
+ c
+
+ ```
+
+- Different hierarchies can be added
+
+ ```scheduler-conf.yaml
+ partitions:
+ - name: default
+ queues:
+ root:
+ configs:
+ acls:
+ childrens:
+ - a
+ - b
+ - c
+ - ...
+ a:
+ configs:
+ acls:
+ capacity: (capacity is not allowed to set for root)
+ max-capacity: ...
+ mapping-policies:
+ ...
+ - name: partition_a:
+ queues:
+ root:...
+ ```
+
+## How scheduler do allocation
+
+Scheduler runs a separate goroutine to look at asks and available resources, and do resource allocation. Here's allocation logic in pseudo code:
+
+Entry point of scheduler allocation is `scheduler.go: func (s *Scheduler) schedule()`
+
+```
+# First of all, YuniKorn has partition concept, a logical resource pool can consists
+# of one of multiple physical dis-joint partitions. It is similar to YARN's node
+# partition concept.
+
+for partition : partitions:
+ # YuniKorn can reserve allocations for picky asks (such as large request, etc.)
+ # Before doing regular allocation, YuniKorn look at reservedAllocations first.
+ for reservedAllocation : partition.reservedAllocations:
+ reservedAllocation.tryAllocate(..)
+
+ # After tried all reserved allocation, YuniKorn will go to regular allocation
+ partition.tryAllocate(..)
+
+ # If there's any allocation created, scheduler will create an AllocationProposal
+ # and send to Cache to "commit" the AllocationProposal
+```
+
+**Allocation by hierchical of queues**
+
+Inside `partition.tryAllocate`
+
+It recursively traverse from root queue and down to lower level, for each level, logic is inside `pkg/scheduler/scheduling_queue.go func (sq *SchedulingQueue) tryAllocate`
+
+Remember YuniKorn natively supports hierarchical of queues. For ParentQueue (which has sub queues under the parent queue), it uses queue's own sorting policy to sort subqueues and try to allocate from most preferred queue to least-preferred queue.
+
+For LeafQueue (which has applications inside the queue), it uses queue's own sorting policy to sort applications belong to the queue and allocate based on the sorted order.
+
+(All sorting policies can be configured differently at each level.)
+
+**Allocation by application**
+
+When it goes to Application, see (`scheduler_application.go: func (sa *SchedulingApplication) tryAllocate`), It first sort the pending resource requests belong to the application (based on requests' priority). And based on the selected request, and configured node-sorting policy, it sorts nodes belong to the partition and try to allocate resources on the sorted nodes.
+
+When application trying to allocate resources on nodes, it will invokes PredicatePlugin to make sure Shim can confirm the node is good. (For example K8shim runs predicates check for allocation pre-check).
+
+**Allocation completed by scheduler**
+
+Once allocation is done, scheduler will create an AllocationProposal and send to Cache to do further check, we will cover details in the upcoming section.
+
+## Flow of events
+
+Like mentioned before, all communications between components like RMProxy/Cache/Schedulers are done by using async event handler.
+
+RMProxy/Cache/Scheduler include local event queues and event handlers. RMProxy and Scheduler have only one queue (For example: `pkg/scheduler/scheduler.go: handleSchedulerEvent`), and Cache has two queues (One for events from RMProxy, and one for events from Scheduler, which is designed for better performance).
+
+We will talk about how events flowed between components:
+
+**Events for ResourceManager registration and updates:**
+
+```
+Update from ResourceManager -> RMProxy -> RMUpdateRequestEvent Send to Cache
+New ResourceManager registration -> RMProxy -> RegisterRMEvent Send to Cache
+```
+
+**Cache Handles RM Updates**
+
+There're many fields inside RM Update event (`RMUpdateRequestEvent`), among them, we have following categories:
+
+```
+1) Update for Application-related updates
+2) Update for New allocation ask and release.
+3) Node (Such as kubelet) update (New node, remove node, node resource change, etc.)
+```
+
+More details can be found at:
+
+```
+func (m *ClusterInfo) processRMUpdateEvent(event *cacheevent.RMUpdateRequestEvent)
+
+inside cluster_info.go
+```
+
+**Cache send RM updates to Scheduler**
+
+For most cases, Cache propagate updates from RM to scheduler directly (including Application, Node, Asks, etc.). And it is possible that some updates from RM is not valid (such as adding an application to a non-existed queue), for such cases, cache can send an event back to RMProxy and notify the ResourceManager. (See `RMApplicationUpdateEvent.RejectedApplications`)
+
+**Cache handles scheduler config**
+
+Cache also handles scheduler's config changes, see
+
+```go
+func (m *ClusterInfo) processRMConfigUpdateEvent(event *commonevents.ConfigUpdateRMEvent)
+```
+
+Similar to other RM updates, it propages news to scheduelr.
+
+**Scheduler do allocation**
+
+Once an AllocationProposal created by scheduler, scheduler sends `AllocationProposalBundleEvent` to Cache to commit.
+
+Cache look at AllocationProposal under lock, and commit these proposals. The reason to do proposal/commit is Scheduler can run in multi-threads which could cause conflict for resource allocation. This approach is inspired by Borg/Omega/YARN Global Scheduling.
+
+Cache checks more states such as queue resources, node resources (we cannot allocate more resource than nodes' available), etc. Once check is done, Cache updates internal data strcture and send confirmation to Scheduler to update the same, and scheduler sends allocated Allocation to RMProxy so Shim can do further options. For example, K8shim will `bind` an allocation (POD) to kubelet.
+
+```
+Job Add:
+--------
+RM -> Cache -> Scheduler (Implemented)
+
+Job Remove:
+-----------
+RM -> Scheduler -> Cache (Implemented)
+Released allocations: (Same as normal release) (Implemented)
+Note: Make sure remove from scheduler first to avoid new allocated created.
+
+Scheduling Request Add:
+-----------------------
+RM -> Cache -> Scheduler (Implemented)
+Note: Will check if requested job exists, queue exists, etc.
+When any request invalid:
+ Cache -> RM (Implemented)
+ Scheduler -> RM (Implemented)
+
+Scheduling Request remove:
+-------------------------
+RM -> Scheduler -> Cache (Implemented)
+Note: Make sure removed from scheduler first to avoid new container allocated
+
+Allocation remove (Preemption)
+-----------------
+Scheduler -> Cache -> RM (TODO)
+ (confirmation)
+
+Allocation remove (RM voluntarilly ask)
+---------------------------------------
+RM -> Scheduler -> Cache -> RM. (Implemented)
+ (confirmation)
+
+Node Add:
+---------
+RM -> Cache -> Scheduler (Implemented)
+Note: Inside Cache, update allocated resources.
+Error handling: Reject Node to RM (Implemented)
+
+Node Remove:
+------------
+Implemented in cache side
+RM -> Scheduler -> Cache (TODO)
+
+Allocation Proposal:
+--------------------
+Scheduler -> Cache -> RM
+When rejected/accepted:
+ Cache -> Scheduler
+
+Initial: (TODO)
+--------
+1. Admin configured partitions
+2. Cache initializes
+3. Scheduler copies configurations
+
+Relations between Entities
+-------------------------
+1. RM includes one or multiple:
+ - Partitions
+ - Jobs
+ - Nodes
+ - Queues
+
+2. One queue:
+ - Under one partition
+ - Under one RM.
+
+3. One job:
+ - Under one queue (Job with same name can under different partitions)
+ - Under one partition
+
+RM registration: (TODO)
+----------------
+1. RM send registration
+2. If RM already registered, remove old one, including everything belong to RM.
+
+RM termination (TODO)
+--------------
+Just remove the old one.
+
+Update of queues (TODO)
+------------------------
+Admin Service -> Cache
+
+About partition (TODO)
+-----------------------
+Internal partition need to be normalized, for example, RM specify node with partition = xyz.
+Scheduler internally need to normalize it to <rm-id>_xyz
+This need to be done by RMProxy
+
+```
diff --git a/versioned_docs/version-1.2.0/assets/RunningSparkOnK8s.png b/versioned_docs/version-1.2.0/assets/RunningSparkOnK8s.png
new file mode 100644
index 000000000..7594a0758
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/RunningSparkOnK8s.png differ
diff --git a/versioned_docs/version-1.2.0/assets/external_userinfo.png b/versioned_docs/version-1.2.0/assets/external_userinfo.png
new file mode 100644
index 000000000..1598351b8
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/external_userinfo.png differ
diff --git a/versioned_docs/version-1.2.0/assets/gang_scheduling_intro.png b/versioned_docs/version-1.2.0/assets/gang_scheduling_intro.png
new file mode 100644
index 000000000..b3be207ea
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/gang_scheduling_intro.png differ
diff --git a/versioned_docs/version-1.2.0/assets/priority-flows.png b/versioned_docs/version-1.2.0/assets/priority-flows.png
new file mode 100644
index 000000000..19b82ec06
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/priority-flows.png differ
diff --git a/versioned_docs/version-1.2.0/assets/priority-tree.png b/versioned_docs/version-1.2.0/assets/priority-tree.png
new file mode 100644
index 000000000..7288db5ea
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/priority-tree.png differ
diff --git a/versioned_docs/version-1.2.0/assets/sparkResult.png b/versioned_docs/version-1.2.0/assets/sparkResult.png
new file mode 100644
index 000000000..8a11e9b72
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/sparkResult.png differ
diff --git a/versioned_docs/version-1.2.0/assets/tf-job-gpu-on-logs.png b/versioned_docs/version-1.2.0/assets/tf-job-gpu-on-logs.png
new file mode 100644
index 000000000..db2a6b693
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/tf-job-gpu-on-logs.png differ
diff --git a/versioned_docs/version-1.2.0/assets/tf-job-gpu-on-ui.png b/versioned_docs/version-1.2.0/assets/tf-job-gpu-on-ui.png
new file mode 100644
index 000000000..b599dca7b
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/tf-job-gpu-on-ui.png differ
diff --git a/versioned_docs/version-1.2.0/assets/translationBuilding.png b/versioned_docs/version-1.2.0/assets/translationBuilding.png
new file mode 100644
index 000000000..0ba725051
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/translationBuilding.png differ
diff --git a/versioned_docs/version-1.2.0/assets/translationDemo.png b/versioned_docs/version-1.2.0/assets/translationDemo.png
new file mode 100644
index 000000000..4125dc676
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/translationDemo.png differ
diff --git a/versioned_docs/version-1.2.0/assets/ug_lookup.png b/versioned_docs/version-1.2.0/assets/ug_lookup.png
new file mode 100644
index 000000000..d86ae6a1a
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/ug_lookup.png differ
diff --git a/versioned_docs/version-1.2.0/assets/yunikorn-gpu-time-slicing.png b/versioned_docs/version-1.2.0/assets/yunikorn-gpu-time-slicing.png
new file mode 100644
index 000000000..8b3d734a4
Binary files /dev/null and b/versioned_docs/version-1.2.0/assets/yunikorn-gpu-time-slicing.png differ
diff --git a/versioned_docs/version-1.2.0/design/architecture.md b/versioned_docs/version-1.2.0/design/architecture.md
index 40db1bc00..5163d4c9f 100644
--- a/versioned_docs/version-1.2.0/design/architecture.md
+++ b/versioned_docs/version-1.2.0/design/architecture.md
@@ -50,7 +50,7 @@ Scheduler core encapsulates all scheduling algorithms, it collects resources fro
platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
best spot for each request and then sends response allocations to the resource management platform.
Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/yunikorn-scheduler-interface).
-Please read more about the design of schedule core [here](scheduler_core_design.md).
+Please read more about the design of schedule core [here](../archived_design/scheduler_core_design.md).
### Kubernetes shim
@@ -58,5 +58,5 @@ The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is res
cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
between the shim and the scheduler core is through the [scheduler interface](https://github.com/apache/yunikorn-scheduler-interface).
-Please read more about the design of the Kubernetes shim [here](k8shim.md).
+Please read more about the design of the Kubernetes shim [here](../archived_design/k8shim.md).
diff --git a/versioned_docs/version-1.2.0/design/cache_removal.md b/versioned_docs/version-1.2.0/design/cache_removal.md
index f78ba07c4..5c46eccfd 100644
--- a/versioned_docs/version-1.2.0/design/cache_removal.md
+++ b/versioned_docs/version-1.2.0/design/cache_removal.md
@@ -32,10 +32,14 @@ It describes the changes planned based on the analysis that was done of the curr
## Goals
The goal is to provide the same functionality before and after the change.
-- unit tests before and after the merge must all pass.
-- Smoke tests defined in the core should all pass without major changes <sup id="s1">[definition](#f1)</sup>.
+- Unit tests before and after the merge must all pass.
+- Smoke tests defined in the core should all pass without major changes.
- End-to-end tests that are part of the shim code must all pass without changes.
+:::info
+ Major changes for smoke tests are defined as changes to the tests that alter use case and thus test flows. Some changes will be needed as checks made could rely on cache objects which have been removed.
+:::
+
## Background
The current Scheduler Core is build up around two major components to store the data: the cache and scheduler objects.
The cache objects form the base for most data to be tracked.
@@ -47,10 +51,9 @@ This short period can have an impact on the scheduling decisions.
One of which is logged as [YUNIKORN-169](https://issues.apache.org/jira/browse/YUNIKORN-169).
A further point is the complexity that the two structure brings to the code.
-A distinct set of messages to communicate between the scheduler and the cache.
-A one on one mapping between the scheduler and cache objects shows that the distinction is probably more artificial than required.
----
-<b id="f1"></b>definition: Major changes for smoke tests are defined as changes to the tests that alter use case and thus test flows. Some changes will be needed as checks made could rely on cache objects which have been removed. [↩](#s1)
+A distinct set of messages to communicate between the scheduler and the cache. A one-on-one mapping between the scheduler and cache objects shows that the distinction is probably more artificial than required.
+
+
## Structure analysis
### Objects
The existing objects as per the code analysis.
diff --git a/versioned_docs/version-1.2.0/design/config_v2.md b/versioned_docs/version-1.2.0/design/config_v2.md
new file mode 100644
index 000000000..ae190bd4b
--- /dev/null
+++ b/versioned_docs/version-1.2.0/design/config_v2.md
@@ -0,0 +1,345 @@
+---
+id: config_v2
+title: Configuration V2
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Over time the YuniKorn configuration has increased in complexity. The current
+configuration for YuniKorn is split over a number of different options, file
+and command line, and formats. There is a requirement now to define an overall
+configuration approach that can be used for all components.
+
+### Goals
+
+- Define a configuration format
+- Define an update and modification strategy
+- Provide consistency between configuration of YuniKorn components
+- Legacy migration strategy
+
+### Non-Goals
+
+- Redefine the queue configuration
+- Migration timeline for the old configuration options
+
+### References
+
+#### Implementation JIRA
+
+[YUNIKORN-1221](https://issues.apache.org/jira/browse/YUNIKORN-1221) - [Umbrella] Configuration V2
+
+#### Other relevant JIRAs
+
+- [YUNIKORN-455](https://issues.apache.org/jira/browse/YUNIKORN-455) - Make the core configurable
+- [YUNIKORN-1213](https://issues.apache.org/jira/browse/YUNIKORN-1213) - Background health checker needs to be configurable
+- [YUNIKORN-1335](https://issues.apache.org/jira/browse/YUNIKORN-1335) - Write yunikorn-web error logs to stderr
+
+### Services to cover
+
+- Scheduler (Standard) - K8Shim + Core
+- Scheduler (Plugin) - K8Shim + Core
+- Admission controller
+
+### Services excluded
+
+ - Web UI (not needed after [YUNIKORN-1335](https://issues.apache.org/jira/browse/YUNIKORN-1335))
+
+## Configuration format
+
+Currently only Kubernetes is a supported deployment pattern. Future shims may
+need an alternative configuration syntax. To enable this, we will update the
+scheduler interface to support passing the core configuration (the content of
+the `queues.yaml` ConfigMap entry) to the core on startup and refresh. The
+existing config watcher functionality in the core will be removed and the shim
+will gain the responsibility for watching for ConfigMap changes. The shim
+actually already has this ability now, it is just not used.
+
+We will use two ConfigMaps to configure YuniKorn:
+ - `yunikorn-defaults`
+ - This will be a new ConfigMap written by Helm; it will contain only values
+ explicitly configured by an administrator via Helm variables.
+ - `yunikorn-configs`
+ - This ConfigMap will contain overrides for yunikorn-defaults. It was
+ previously a Helm hook in previous YuniKorn releases; it will no longer
+ be managed by Helm and may be updated freely by third-party tools.
+
+The effective configuration will be the combination of both `yunikorn-defaults`
+ and `yunikorn-configs`, with `yunikorn-configs` taking precedence.
+
+Neither ConfigMap (even the existing one) is required for proper functionality.
+In the case where a ConfigMap does not exist, we will use compiled-in defaults
+for all options. This enables bootstrapping in a variety of environments,
+including local development mode, as the ConfigMap need not exist to bring the
+scheduler components up. Future configuration options should be set via the
+ConfigMap exclusively.
+
+### Mutability of configuration values
+
+We should strive for hot-reload of all configuration values where possible.
+Where hot-reload is not implemented, this should be clearly documented. For the
+initial release, the core config and admission controller will be
+hot-reloadable but many items in the shim will likely not be. The shim was not
+designed to hot-reload anything so this will be a large future task. Some items
+may never be reloadable due to architectural limitations.
+
+### Update handling
+
+Given the security issues involved with updates, the default Kubernetes access
+control alone should be used for ConfigMap updates. We can still syntax-check
+the core side of the configuration in the admission controller as before.
+Partial updates are allowed as Kubernetes supports patching of ConfigMaps.
+
+### Legacy configuration handling
+
+Helm will be updated to render all existing legacy options into the
+`yunikorn-defaults` ConfigMap for easier transition to the new configuration
+format. Consequently, it is not necessary for YuniKorn components to parse the
+old environment and command-line arguments.
+
+### Environment variables
+
+All environment variables for all services will be removed, with the exception
+of:
+
+ - `NAMESPACE`
+ - Default value: `default`
+ - Will be auto-populated from Helm and resolve to the pod's running
+ namespace
+ - Needed so that YuniKorn can determine what namespace it was launched in
+ - Should only be provided explicitly at development time
+ - `KUBECONFIG`
+ - Default value: `$HOME/.kube/config`
+ - Will be ignored if running inside Kubernetes; in-cluster configuration
+ will be used in that case
+ - Needed so that we can bootstrap to K8s and load the YuniKorn ConfigMap at
+ runtime
+ - Should only be provided explicitly at development time
+
+### Helm considerations
+
+All configuration options which have moved into the ConfigMap will be marked as
+deprecated in the Helm chart documentation. Their replacement is the more
+generic `yunikornDefaults` dictionary. If the legacy values are present but
+the associated `yunikornDefaults` entry is missing, the legacy configuration
+value will be used. This will allow users some time to migrate to the new
+configuration style. The legacy attributes will be removed in a future release.
+
+### Core Configuration
+
+The core scheduler configuration is loaded from the `queues.yaml` section of
+the ConfigMap, assuming `service.policyGroup` is set to the default value
+`queues`. If the policy group has been customized, core scheduler configuration
+will be read from `{service.policyGroup}.yaml`. This section is special as its
+contents are passed directly to the core for evaluation and update and is not
+read directly by the shim. The content of this section is unchanged from prior
+releases.
+
+## Appendix
+
+### Default ConfigMap
+
+If not present, the default ConfigMap will resolve to these defaults.
+Additionally, individual options will resolve to those listed here if not
+specified. Note that these values are subject to change in future releases.
+
+ apiVersion: v1
+ kind: ConfigMap
+ metadata:
+ name: yunikorn-configs
+ data:
+ service.clusterId: "mycluster"
+ service.policyGroup: "queues"
+ service.schedulingInterval: "1s"
+ service.volumeBindTimeout: "10s"
+ service.eventChannelCapacity: "1048576"
+ service.dispatchTimeout: "5m"
+ service.operatorPlugins: "general"
+ service.disableGangScheduling: "false"
+ service.enableConfigHotRefresh: "true"
+ service.placeholderImage: "registry.k8s.io/pause:3.7"
+ health.checkInterval: "30s"
+ log.level: "0"
+ kubernetes.qps: "1000"
+ kubernetes.burst: "1000"
+ admissionController.webHook.amServiceName: "yunikorn-admission-controller-service"
+ admissionController.webHook.schedulerServiceAddress: "yunikorn-service:9080"
+ admissionController.filtering.processNamespaces: ""
+ admissionController.filtering.bypassNamespaces: "^kube-system$"
+ admissionController.filtering.labelNamespaces: ""
+ admissionController.filtering.noLabelNamespaces: ""
+ admissionController.accessControl.bypassAuth: "false"
+ admissionController.accessControl.trustControllers: "true"
+ admissionController.accessControl.systemUsers: "^system:serviceaccount:kube-system:"
+ admissionController.accessControl.externalUsers: ""
+ admissionController.accessControl.externalGroups: ""
+ queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'`
+
+### Admission Controller Configuration
+
+#### Removed Environment Variables
+ - `POLICY_GROUP`
+ - Use ConfigMap: `service.policyGroup`
+ - `ENABLE_CONFIG_HOT_REFRESH`
+ - Use ConfigMap: `service.enableConfigHotRefresh`
+ - `SCHEDULER_SERVICE_ADDRESS`
+ - Use ConfigMap: `admissionController.webHook.schedulerServiceAddress`
+ - `ADMISSION_CONTROLLER_SERVICE`
+ - Use ConfigMap: `admissionController.webHook.amServiceName`
+ - `ADMISSION_CONTROLLER_NAMESPACE`
+ - Use Environment variable: `NAMESPACE`
+ - `ADMISSION_CONTROLLER_PROCESS_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.processNamespaces`
+ - `ADMISSION_CONTROLLER_BYPASS_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.bypassNamespaces`
+ - `ADMISSON_CONTROLLER_LABEL_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.labelNamespaces`
+ - `ADMISSION_CONTROLLER_NO_LABEL_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.noLabelNamespaces`
+
+#### Removed CLI Options
+None
+
+### Scheduler (Standard) Configuration
+
+#### Removed Environment Variables
+ - `USER_LABEL_KEY`
+ - No replacement
+ - `OPERATOR_PLUGINS`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `PLACEHOLDER_IMAGE`
+ - Use ConfigMap: `service.placeholderImage`
+
+#### Removed CLI Options
+ - `-clusterId`
+ - Use ConfigMap: `service.clusterId`
+ - `-clusterVersion`
+ - No replacement
+ - `-policyGroup`
+ - Use ConfigMap: `service.policyGroup`
+ - `-interval`
+ - Use ConfigMap: `service.schedulingInterval`
+ - `-logLevel`
+ - Use ConfigMap: `log.level`
+ - `-logEncoding`
+ - No replacement
+ - `-logFile`
+ - No replacement
+ - `-volumeBindTimeout`
+ - Use ConfigMap: `service.volumeBindTimeout`
+ - `-eventChannelCapacity`
+ - Use ConfigMap: `service.eventChannelCapacity`
+ - `-dispatchTimeout`
+ - Use ConfigMap: `service.dispatchTimeout`
+ - `-kubeQPS`
+ - Use ConfigMap: `kubernetes.qps`
+ - `-kubeBurst`
+ - Use ConfigMap: `kubernetes.burst`
+ - `-operatorPlugins`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `-enableConfigHotRefresh`
+ - Use ConfigMap: `service.enableConfigHotRefresh`
+ - `-disableGangScheduling`
+ - Use ConfigMap: `service.disableGangScheduling`
+ - `-userLabelKey`
+ - No replacement
+ - `-placeHolderImage`
+ Use ConfigMap: `service.placeholderImage`
+
+### Scheduler (Plugin) Configuration
+
+#### Removed Environment Variables
+ - `USER_LABEL_KEY`
+ - No replacement
+ - `OPERATOR_PLUGINS`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `PLACEHOLDER_IMAGE`
+ - Use ConfigMap: `service.placeholderImage`
+
+#### Removed CLI Options
+ - `--yk-scheduler-name`
+ - No replacement
+ - `--yk-cluster-id`
+ - Use ConfigMap: `service.clusterId`
+ - `--yk-cluster-version`
+ - No replacement
+ - `--yk-policy-group`
+ - Use ConfigMap: `service.policyGroup`
+ - `--yk-scheduling-interval
+ - Use ConfigMap: `service.schedulingInterval`
+ - `--yk-kube-config`
+ - Use environment: `KUBECONFIG`
+ - `--yk-log-level`
+ - Use ConfigMap: `log.level`
+ - `--yk-log-encoding`
+ - No replacement
+ - `--yk-log-file`
+ - No replacement
+ - `--yk-event-channel-capacity`
+ - Use ConfigMap: `service.eventChannelCapacity`
+ - `--yk-dispatcher-timeout`
+ - Use ConfigMap: `service.dispatchTimeout`
+ - `--yk-kube-qps`
+ - Use ConfigMap: `kubernetes.qps`
+ - `--yk-kube-burst`
+ - Use ConfigMap: `kubernetes.burst`
+ - `--yk-operator-plugins`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `--yk-enable-config-hot-refresh`
+ - Use ConfigMap: `service.enableConfigHotRefresh`
+ - `--yk-disable-gang-scheduling`
+ - Use ConfigMap: `service.disableGangScheduling`
+ - `--yk-user-label-key`
+ - No replacement
+ - `--yk-placeholder-image`
+ - Use ConfigMap: `service.placeholderImage`
+
+### Helm Configuration
+
+#### Removed Options
+ - `userLabelKey`
+ - No replacement
+
+#### Deprecated Options
+ - `operatorPlugins`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `placeHolderImage`
+ - Use ConfigMap: `service.placeholderImage`
+ - `admissionController.processNamespaces`
+ - Use ConfigMap: `admissionController.filtering.processNamespaces`
+ - `admissionController.bypassNamespaces`
+ - Use ConfigMap: `admissionController.filtering.bypassNamespaces`
+ - `admissionController.labelNamespaces`
+ - Use ConfigMap: `admissionController.filtering.labelNamespaces`
+ - `admissionController.noLabelNamespaces`
+ - Use ConfigMap: `admissionController.filtering.noLabelNamespaces`
+ - `configuration`
+ - Use ConfigMap: `queues.yaml`
+
diff --git a/versioned_docs/version-1.2.0/design/priority_scheduling.md b/versioned_docs/version-1.2.0/design/priority_scheduling.md
new file mode 100644
index 000000000..84f9ad734
--- /dev/null
+++ b/versioned_docs/version-1.2.0/design/priority_scheduling.md
@@ -0,0 +1,408 @@
+---
+id: priority_scheduling
+title: Priority Scheduling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Currently, YuniKorn can sort at two levels: the applications in a queue and
+the resource requests within the application. The queue sorting policy is
+configurable and supports a _Fair_ or _FIFO_ policy. _StateAware_ is
+considered a filtered _FIFO_ policy. Applications cannot be sorted based
+on priority.
+
+Within an application, requests are sorted based on priority of the requests.
+For requests that have the same priority the submission time is used as a
+secondary sort key. Requests are sorted in _FIFO_ order if they have the
+same priority. Note that the priority of requests was not propagated from
+the shim to the core until
+[YUNIKORN-1235](https://issues.apache.org/jira/browse/YUNIKORN-1235) was
+implemented in v1.0.0. Until that implementation all requests were created
+with the _default_ priority.
+
+Additionally, Kubernetes uses priority as a sort when performing preemption.
+YuniKorn currently only does preemption of requests if a node-specific task
+needs to be scheduled (i.e. a DaemonSet pod,
+[YUNIKORN-1085](https://issues.apache.org/jira/browse/YUNIKORN-1085)).
+
+## Goals
+
+- Attempt to be compatible with Kubernetes standard priority handling
+ wherever possible
+- Priority should be orthogonal to application sorting policies (i.e.
+ _Fair_, _FIFO_ and _StateAware_ policies should support prioritization)
+- It should be possible to limit the scope of priority handling
+- Design of priority scheduling should consider the impact on preemption
+ design
+
+## Non Goals
+
+- Design of preemption
+- Change AllocationAsk sorting within an application
+
+## Definitions
+
+The following terms are defined as follows and will be used throughout this
+document.
+
+### Priority
+
+Priority is a numeric value associated with an Ask. Higher-priority asks have
+higher numeric values. To be compatible with Kubernetes, all 32-bit signed
+integers are considered valid priorities. An ask without a specific defined
+priority is assumed to have the default priority.
+
+### Minimum Priority
+
+Minimum priority is defined as the lowest possible priority, or
+`-2,147,483,648` (- 2<sup>31</sup>).
+
+### Maximum Priority
+
+Maximum priority is defined as the highest possible priority, or
+`2,147,483,647` (2<sup>31</sup> - 1). Note that Kubernetes does not permit
+custom priority classes (see below) to define priorities greater than
+`1,000,000,000` (1 billion). Values higher are reserved for system-level
+priorities.
+
+### Default Priority
+
+Default priority is defined as the value `0`.
+
+### PriorityClass
+
+A [PriorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass)
+is a Kubernetes object which maps between a priority name and an integer
+priority value. The name cannot start with `system-` as that prefix is
+reserved for Kubernetes internal usage.
+
+Additionally, a `preemptionPolicy` may be set on a PriorityClass. Official
+GA support was added in Kubernetes 1.24. The preemption policy is only
+used while scheduling the pod itself. It is not taken into account after
+the pod is scheduled.
+
+The policy defaults to `Never`, implying that pods of this priority will
+never preempt other pods. `PreemptLowerPriority` is also possible, which
+allows the scheduler to preempt pods with lower priority values.
+
+When a pod is submitted with its `priorityClassName` set to a valid priority
+class, a Kubernetes-provided admission controller will automatically update
+the `priority` and `preemptionPolicy` fields of the pod to match those of the
+referenced PriorityClass. Pods that specify a priority as part of the
+specification are rejected if the integer value set on the pod does not match
+the PriorityClass value.
+
+#### Pre-defined PriorityClass objects
+- system-cluster-critical: `2,000,000,000`
+- system-node-critical: `2,000,000,100`
+
+## Current state
+
+The framework for priority is minimally implemented within YuniKorn. The
+_AllocationAsk_ structure within the scheduler interface already contains a
+_priority_ field which is populated by the shim based on the _priority_ field
+on a Pod.
+
+One priority can be set on creation of the _AllocationAsk_. Each
+_AllocationAsk_ within an application may have its own priority.
+_AllocationAsk_ sorting is based on this priority. This means that within the
+application asks or pods are sorted based on priority. High-priority asks will
+be scheduled before low-priority asks.
+
+Priorities are currently also used for the minimal DaemonSet preemption process
+implemented in [YUNIKORN-1085](https://issues.apache.org/jira/browse/YUNIKORN-1085).
+Possible victims are sorted based on their priority among other things. The
+priority of the _Allocation_ is the same as the _AllocationAsk_ they
+represent.
+
+One use of the capability to set a priority for each _AllocationAsk_ could be to
+enable dynamic allocations such as those utilized by Apache Spark. An
+application can submit a set of asks at high priority that will be scheduled
+as soon as possible, and a set of lower-priority asks that can be used for
+additional capacity if available.
+
+## Proposal
+
+### Priority fencing
+
+By default, priority applies across queues globally. In other words, a
+higher-priority ask will be attempted to be satisfied before a lower-priority
+ask regardless of which queue the asks reside in.
+
+However, it is often useful to limit the scope of priority handling. For
+example, it may be desirable to limit the ability of asks in one area of the
+queue hierarchy from jumping ahead of asks in another area. To support this,
+we add the concept of a _Priority Fence_. A queue (either _leaf_ or _parent_)
+may be configured as priority-fenced, which prevents priorities from
+propagating upward.
+
+If a queue is fenced, asks within that queue (and subqueues) will be
+prioritized relative to each other, but not relative to asks in other portions
+of the queue hierarchy. A priority fence is a YuniKorn queue subtree boundary
+above which priorities are no longer considered. A priority fenced queue does
+not expose priority levels of its children to its parent queue, thereby
+ensuring that priorities are only considered within that queue and its
+descendants.
+
+Priority fences may be nested; for example a fenced queue `tenant 1` may have
+child queues `queue A` (fenced) and `queue B` (unfenced). In this case tasks
+in `queue A` will show as the default priority and will not be prioritized
+above `queue B`, but tasks in `queue B` may be prioritized over those in
+`queue A`. In no case will tasks in either queue be prioritized over tasks
+outside the scope of queue `tenant 1`, as they show as the default priority
+(shown in the diagram below with yellow arrows).
+
+A similar setup is created for the `tenant 2` queue. However in this case the
+priorities from `queue 1` to `queue 2` and vice-versa are fully visible
+(shown in the diagram below with blue arrows).
+
+None of the tasks running in the queues below `tenant 1` or `tenant 2` can be
+prioritized above the `system` queue. Tasks in the `system` queue are not
+fenced. That means that a task in the `system` queue can be prioritized above
+any queue in the hierarchy (shown in the diagram below with red arrows).
+
+
+
+### Priority offset
+
+Priorities need to be pre-defined at the cluster level via the PriorityClass
+objects. At the Kubernetes level there is no way to limit which priorities can
+be used in which part of the system. The only exception to that rule are the
+pre-defined system priorities from the Kubernetes system itself. This does not
+give an administrator a lot of control.
+
+Prioritizing a specific queue above another queue based on the priorities of
+the tasks inside the queue is thus difficult. Any user can start a task with
+any priority anywhere in the cluster. To allow an administrator more control
+we need to be able to steer, or augment, the priority of a queue. To
+accomplish this, we introduce the `priority.offset` property of a queue.
+
+The `priority.offset` property on a queue allows the administrator to increase
+(or decrease) a queue's priority. By using the offset a queue can be boosted
+to become a high-priority queue. It also can be used to decrease the priority
+compared to its siblings.
+
+As a general rule: the priority of a _leaf_ queue is equal to the maximum
+priority of the AllocationAsks in that queue, plus the `priority.offset`
+value. The priority of a _parent_ queue is the maximum of the priority of its
+child queues plus the `priority.offset` value. This means that priority
+offsets are additive when traversing up the queue hierarchy. A priority offset
+may be any valid signed `int32` value. The offset is applied after the dynamic
+priority for the queue is calculated.
+
+As a note of caution: a side effect of a very large offset value for a queue
+could result in all pods in that queue being clamped at the maximum priority
+level or exceeding the priority of Kubernetes system priority classes. This
+could then adversely impact the scheduling of node and or cluster critical
+pods in sibling queues. This effect needs to be clearly documented. Logging a
+warning about a configured value that high should also be considered.
+
+A second option that was considered is limiting the offset value to
+`999,999,999`. This would prevent the offset priority from exceeding the
+pre-defined system priorities. This would be a safe option but does not take
+into account the fact that we can have negative priorities that we want to
+offset. The current choice is to go with the first option to document and log.
+
+Fencing affects the priority calculation for the queue. When a queue is
+fenced (i.e. the `priority.policy` is set to `fence`), queue priority
+calculations are disabled for that queue. The result of application priority
+or child queue calculations are ignored for a fenced queue. The priority of
+the fenced queue is always set to the default priority or to the priority
+offset, if specified.
+
+### Extended application sorting
+
+Applications are currently sorted based on one of the defined application
+sorting policies. All application sorting policies only take into account
+applications that have pending requests.
+
+Currently defined are _Fair_, _FIFO_, and _StateAware_:
+
+- _Fair_ sorting sorts based on relative usage within a queue
+- _FIFO_ sorts applications by creation time in a first-in, first-out manner
+- _StateAware_ is similar to _FIFO_ except that only a single "new"
+ application is allowed at once (this is an over-simplification, but will
+ suffice here)
+
+To support priority scheduling, the policies _Fair_ and _FIFO_ will get an
+equivalent policy that considers the priority of an application first. For
+applications that have the same priority, sorting will then fall back to the
+secondary sort criteria (_Fair_ or _FIFO_).
+
+_StateAware_ as a special form of _FIFO_ will however always filter
+applications first and then sort. A priority based version of _StateAware_
+must change the way it filters. The single "new" application that is
+considered can no longer be the oldest application. That would be a major
+behavioral change from the current policy.
+
+The existing policies will not be changed to maintain backwards
+compatibility. None of the existing policies have a tiebreak built in. In
+the case the comparison returns equality the order of the elements is not
+changed.
+
+Instead of introducing new policies to add priority to the existing
+behavior a flag will be added that allows switching priorities on or off.
+The property will be added to the queue.
+
+New property: `appication.sort.priority`
+
+### Extended queue sorting
+
+Queue sorting currently only supports one policy: _Fair_. Queues cannot
+be sorted using a _FIFO_ mechanism. _FIFO_ sorting of queues is not in
+scope of this design. However the design has allowed for a simple addition
+of a new policy.
+
+The _Fair_ policy works similarly to application _Fair_ sorting, except
+that it sorts based on usage ratios between two queues. Allocations are
+compared to guaranteed resources. This results in the queue furthest below
+its guaranteed resource to be first in the sorted queue list as it is
+considered "more starved". For queues that have the same usage ratio,
+the queue with the highest pending requests is considered more starved.
+
+To add priority support, a new property will be added to consider the
+current queue priority as the main sorting factor. To break the tie between
+queues with the same priority, starvation like in the _Fair_ policy will be
+used. Again breaking the tie for queues with the same _Fair_ usage using the
+pending requests.
+
+The existing policy will not be changed to maintain backwards compatibility.
+The same property as used for application sorting will be reused to control
+priority behavior.
+
+New property: `application.sort.priority`
+
+### Priority Configuration
+
+#### Queue Configuration
+
+Three new attributes will be added to the queue configuration object to allow
+specifying the priority fence type and priority offset and priority sorting.
+All properties `priority.policy`, `priority.offset`, and
+`application.sort.priority`, can be set on any queue. The attributes are part
+of the queue properties as follows:
+
+```yaml
+queues:
+ - name: fencedqueue
+ properties:
+ priority.policy: fence
+ priority.offset: 100
+ application.sort.priority: enabled
+```
+
+The `priority.policy` and `priority.offset` fields are not inherited from the
+parent queue to a child queue; they apply only to the queue they are specified
+on.
+
+Note that the `application.sort.priority` field is only inherited if the value
+is set to `disabled` (the `enabled` value is already the default). This allows
+priorities to be ignored for an entire queue subtree by setting the property
+on the parent queue.
+
+Properties are fully supported in child templates. Support for setting these
+properties for dynamic queues is included as part of the existing
+functionality.
+
+The supported, not case sensitive, values for the `application.sort.priority`
+are:
+
+ - `enabled`
+ - `disabled`
+
+The proposal is to default to `enabled` for the `application.sort.priority`
+property. Given that the Kubernetes default scheduler and preemption support
+are all based on priorities that choice seems more logical than `disabled`.
+
+The `application.sort.priority` when set on a _parent_ queue affects the way
+the queues are sorted. It will change queue sorting from the standard _Fair_
+sorting to priority-based sorting. To break the tie for equal priority
+queues it will use the _Fair_ comparison. When set on a _leaf_ queue it will
+change the application sorting itself. All three existing policies will be
+supported with `application.sort.priority` set to `enabled` or `disabled`.
+For the `disabled` case nothing will change and the old behavior is
+maintained. For the `enabled` case the primary sort will be priority and the
+configured policy (_Fair_, _FIFO_, or _StateAware_) will be used as
+tiebreakers.
+
+The `priority.policy` and `priority.offset` can be set on the `root` queue
+but have no effect. The root `queue` as the top of the hierarchy has no
+parent. This means priority cannot traverse further up the tree. No messages
+will be logged if the `priority.policy` or `priority.offset` are set on the
+`root` queue.
+
+The supported, not case sensitive, values for the `priority.policy` are:
+
+- `default`
+- `fence`
+
+If the value `fence` is set on a queue, the queue on which the property is
+set does not propagate its priority outside of the queue's subtree. The value
+`default` does not fence the queue. If no policy is set the value `default` is
+used.
+
+The `priority.offset` value must be a valid integer in string form. The value
+will be converted into an integer using the standard parsing methods as defined
+via: [`strconv.ParseInt`](https://pkg.go.dev/strconv#ParseInt)`(s, 10, 32)`.
+
+If no `priority.offset` property is specified in the queue the default of `0`
+is used. Specifying the property with an empty value is equivalent to `0`. If
+parsing of the string fails the default of `0` is used.
+
+### Scheduler storage object changes
+
+#### Application
+
+The _Application_ object within the scheduler core will be updated with a
+dynamic priority value, defined as the maximum priority of all pending asks.
+To avoid sorting performance degradation, this value should be cached and
+recomputed only when allocation asks are added, allocated, or removed.
+
+This recalculation can be avoided in many instances. When new asks are added,
+an application's priority need not be recalculated unless the priority of the
+new ask is greater than that of the application as a whole. Similarly, when
+removing an ask, the priority need not be adjusted unless the ask's priority
+is equal to the application's priority. When recalculating priority due to
+adding a high-priority ask, the application's priority can simply be set to
+the new ask's priority. When recalculating due to removal of an ask of equal
+priority to the application, we can stop considering additional asks as soon
+as any other ask of equal priority is encountered. This means that the
+performance impact of application priority updates will be `O(1)` when adding
+an ask, and at worst `O(n)` when removing an ask.
+
+New field: `askMaxPriority`
+
+#### Queue
+
+The new configuration values need to be maintained on the queue. Besides those
+values the _Queue_ object within the scheduler core will be updated with a
+dynamic priority value. This dynamic priority will be defined as the maximum
+priority of all applications within the queue, plus any "priority offset"
+configured.
+
+To avoid sorting performance degradation, this value should be cached and
+recomputed only when necessary. For a _leaf_ queue, this is required if
+application priorities in the queue have changed, and for a _parent_ queue this
+is required if the priorities of child queues have changed.
+
+New fields: `priorityPolicy`, `priorityOffset`, `currentPriority`
+
diff --git a/versioned_docs/version-1.2.0/design/user_group_handling.md b/versioned_docs/version-1.2.0/design/user_group_handling.md
new file mode 100644
index 000000000..57994fadf
--- /dev/null
+++ b/versioned_docs/version-1.2.0/design/user_group_handling.md
@@ -0,0 +1,290 @@
+---
+id: user_group
+title: User/Group handling and lookup design
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Currently, user-handling in YuniKorn is handled in a relatively simple manner. If the submitted pod contains a user label the content will be set as the username. The label name is configurable and defaults to yunikorn.apache.org/username. The value is extracted in the shim code [GetUserFromPod](https://github.com/apache/yunikorn-k8shim/blob/v1.0.0/pkg/common/utils/utils.go#L220) and passed to the core. If the label is not present or no value is set the username will be set to default.
+
+However, this makes it possible for every authenticated user to freely impersonate anyone. This allows a user to circumvent quotas and other user-based restrictions inside YuniKorn. Therefore, we need a robust, secure way to make sure that users are who they say they are. This impersonation should not be confused with [Kubernetes impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation).
+
+In the core part of YuniKorn, we also perform group resolution, similar to a unix based system or for instance Hadoop. The default implementation is a nil-lookup. It simply populates the group slice with a single entry which is the same as the username. This is not fit for purpose and was only added as a placeholder until proper user and group resolution could be added.
+
+## Goals
+
+- Define a stable way for user & group retrieval from Kubernetes
+- Assess fit of the current group resolution in the shim
+- Definition of user and group info on Kubernetes objects
+
+## Non Goals
+
+- Authorisation or Authentication as part of:
+ - YuniKorn
+ - Kubernetes
+- Changes to the way users are propagated between the shim and the core
+
+
+## Kubernetes user and group handling
+
+Dealing with how users are actually managed in Kubernetes is out of scope. What is important is the definitions that are used by Kubernetes for users. In Kubernetes, there are two kinds of accounts defined: [service and user accounts](https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/). User accounts are for humans. Service accounts are set up for processes which run inside a pod and want to access the API server.
+
+### User model
+
+Kubernetes, at this point in time, only manages service accounts, not users. Kubernetes does not have objects which represent user accounts. A user with valid credentials is considered authenticated.
+
+During the authentication phase the user account is mapped to a service account. For authorisation Kubernertes uses a sophisticated role-based (RBAC) or a simpler attribute-based (ABAC) model. All authorisation is linked to the service accounts, not to the user accounts. Beside the authentication there is thus no need for the user accounts. This is reflected in Kubernetes as after the authentication phase the user account is no longer tracked or available.
+
+The model used by Kubernetes thus limits what we can do to retrieve the user and or group details. All objects that are available inside the cluster to a scheduler, like YuniKorn, only have the service account information available. Getting access to the user account means that we need to intercept the information at an earlier stage.
+
+### User information as part of authentication
+
+User accounts are required during the authentication stage. Authentication in Kubernetes is configurable via plugins. For more details, see the official document on [Kubernetes authentication](https://kubernetes.io/docs/reference/access-authn-authz/authentication/). The list of supported authentication plugins, not all of these plugins will be configured and or active by default:
+- X.509 certificate
+- Static token file
+- OpenID connect tokens
+- Authentication proxy
+- Authentication webhooks ([LDAP example](https://itnext.io/implementing-ldap-authentication-for-kubernetes-732178ec2155))
+- Bootstrap token
+- Service Account Tokens
+
+
+When using test clusters created by Minikube or KIND, X509 authentication is used by default and the current context is updated with the appropriate certificates and keys, making cluster access transparent.
+
+Authentication, independent of which option is used, provides the following details:
+- Username
+- userID
+- Groups
+- Extra information
+
+Where the details are stored or how they are retrieved depends on the authentication plugin used.
+
+Kubernetes defines a special user for anonymous access, the `system:anonymous` user belonging to the `system:unauthenticated` group. Anonymous access is a security issue and should have been turned off in most clusters. This default mapping to a standardised user does provide the possibility to process authenticated and anonymous workloads in the same way.
+
+### Groups in authentication
+
+Users may belong to zero or more groups. The authentication system of Kubernetes adds one of two default groups to each user. The `system:authenticated` group is added to the list of groups for authenticated users, the `system:unauthenticated` group is added for anonymous users.
+
+For example, when generating an X509 certificate, we can define the username and organisation information like this:
+```
+openssl req -new -key alice.key \
+-out alice.csr \
+-subj "/CN=alice/O=users/O=devops"
+```
+This certificate, after being processed, provides an authenticated user with the username alice and sets the groups list to users, devops and system:authenticated. Other authentication methods allow specifying groups in their specific way. How groups are specified for each authentication option is not relevant to this design as it is encapsulated in the authentication layer and not in the exposed model.
+
+### Current findings and challenges
+
+None of the objects in Kubernetes contain the authentication information and thus the user details. After the authentication has been processed the objects inside Kubernetes either use the service account that was specified as part of the object definition or the Default service account.
+
+
+That means that the pod which is processed by YuniKorn, represented by the `k8s.io/api/core/v1/Pod` object, only has the service account as a reference.This is similar to the details available to someone using the Kubernetes tools to describe the object.
+
+The only way to retrieve the authentication information is during the admission phase of a new object into the Kubernetes system. This process is handled by the admission controllers which are part of the decision process to allow or disallow the new object to be created, modified or removed.
+
+This finding does allow YuniKorn, via its admission controller, to retrieve the authentication information of objects.
+
+
+
+
+However there are some further complications to this process. Pods are the units that the scheduler works with. Objects like deployments or jobs do not get scheduled, they in turn create a pod to be scheduled. Pods could thus be created in a number of ways.
+
+If a pod is created by a direct request, then the pod creation process contains the authentication information. This allows the YuniKorn admission controller to access and propagate the authenticated user information as part of the MutatingAdmissionWebhook.
+
+An example of a review of a direct call to create a pod with the user information shown:
+```
+INFO webhook/admission_controller.go:180 AdmissionReview {"Namespace": "default", "UID": "3f8c8df5-0272-4c85-874f-0aa1ece74fbc", "Operation": "CREATE", "UserInfo": {"username":"minikube-user","groups":["system:masters","system:authenticated"]}}
+```
+
+On the other hand, if a deployment, job or replica set is submitted, then the pod is not directly created. This also means that the user is not identifiable. The pod that YuniKorn, via the admission controller, sees is created by a controller. The user in the authentication information is not the end user that created the original object but the controller service account.
+
+An example review of a pod created as part of a Kubernetes job with the user information shown:
+
+```
+INFO webhook/admission_controller.go:177 AdmissionReview {"Namespace": "default", "UID": "be94ea46-c1d2-442a-a46a-60589d582abd", "Operation": "CREATE", "UserInfo": {"username":"system:serviceaccount:kube-system:job-controller","uid":"32abc062-eb53-4dca-9669-55251d687939","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]}}
+```
+It is possible to create separate webhooks for the API objects that could result in the creation of pods. Other objects that we should watch and mutate if necessary:
+- Deployment
+- ReplicaSet
+- DaemonSet
+- StatefulSet
+- Job
+- CronJob
+- ReplicationController (deprecated)
+
+For example, adding a modification on the admission controller which instals a MutatingAdmissionWebhook for deployments, the username can be determined:
+
+```
+INFO webhook/admission_controller.go:483 Deployment validation - pass through {"User": {"username":"minikube-user","groups":["system:masters","system:authenticated"]}}
+```
+
+By intercepting the original object the submitting user information can be retrieved in the admission controller. The set of objects of interest, mentioned in the list above, all contain a pod template to specify the pod that will be created by the controller. As part of the mutating admission webhook the pod template can be updated with the user information.
+
+To confirm that this approach is viable a simple proof of concept (POC) was created. The example POC code for the current admission controller adds the currently defined yunikorn.apache.org/username label to the pod template of a deployment: [Github commit](https://github.com/pbacsko/incubator-yunikorn-k8shim/commit/478d688675a1ca98afb895672295d61b47172d19)
+
+There is an interaction with the processing of a directly created pod. Pods that are created via a controller will have the user information already set via the pod template. The admission controller processing the request should allow for the user information to be present and not overwrite or replace it with the controller service account. This case will be further defined and detailed in the chapter on [Providing user and group externally](#providing-user-and-group-externally) due to [...]
+
+## YuniKorn group retrieval
+
+As mentioned in the introduction, the YuniKorn core can retrieve a group for a given user by doing a lookup (see implementation in [usergroup.go](https://github.com/apache/yunikorn-core/blob/161c38100317ec239965e21806ebb35c6583ddcd/pkg/common/security/usergroup.go)). The lookup is pluggable and the current implementations provided are a simple OS-based and a dummy lookup.
+
+The dummy lookup mimics the simple unix OS behaviour for new users. It sets the group to the same value as the username. The dummy lookup is enabled as the default lookup.
+
+Resolving users using the OS-based resolver has limited functionality. The container that the scheduler runs in has only a base OS installed. There is no integration with an external identity provider. That means resolving groups for users running pods would normally fail as they do not exist on the OS layer that is accessible from the scheduler.
+
+The YuniKorn core only performs a group lookup if the UserGroupInformation object passed from the shim to the core does not define a list of groups. This is independent of the resolver defined in the code. By default the shim in its current implementation only passes the user name to the core. That means it always triggers the group resolution.
+
+For our purposes, both options are too simplistic. YuniKorn core should not perform any group retrieval but instead rely on the shim and the group and user information retrieved there. With the current functionality to only resolve if no group details are provided changes required should be minimal. The current default dummy resolver should be updated to not set any group and just leave the current groups. Even if that means the user is not part of any group. Based on the [Groups in auth [...]
+
+The design does not require any code changes in the core code.
+
+## Proposed user information object
+
+The admission controller has access to more detailed user information than what is available on the objects that are available to the shim. See the [Current findings and challenges](#current-findings-and-challenges) for more details.
+
+
+First change is that we need to use annotations instead of [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/). The value of labels is limited to 63 characters, which will not be flexible enough to hold group information. If a user is a member of multiple groups it could easily exceed the value limit. Spitting the groups over multiple labels or encoding is not a viable option as stability is not a guarantee.
+
+Supporting a single label for the user’s name and annotations for the groups is not a good user experience. We must provide a single object that can contain the user name as well as the group details.
+
+### Annotation name
+
+The new name for the annotation should clearly cover the content and not interfere with existing annotations. Beside that we are free to use any name that complies with the naming [conventions for names](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set).
+
+The proposal is to use one annotation:
+`yunikorn.apache.org/user.info`
+
+### Annotation content
+
+There is already a definition for the user information as part of the scheduler interface.The definition in the scheduler interface is:
+
+```
+message UserGroupInformation {
+// the user name
+string user = 1;
+// the list of groups of the user, can be empty
+repeated string groups = 2;
+}
+```
+
+The content of the `UserGroupInformation` message as defined in the Scheduler Interface provides a stable Go object with defined group ordering. Since the content of an annotation must be a simple string we should use a simple json representation for the user info that contains the user name and a list of groups. This also guarantees the ordering of the groups to be stable and will not change when read from the pod.
+
+
+An example of the annotation for a user called joe who is a member of the group developers. The system:authenticated group is added by the Kubernetes authentication layer:
+
+```json
+yunikorn.apache.org/user.info = "
+{
+ username: \"joe\",
+ groups: [
+ \"developers\",
+ \"system:authenticated\"
+ ]
+}"
+```
+
+### Annotation mutation
+
+In the current design a restriction on mutating the annotation is required to prevent users from updating the pod after creation. After YuniKorn is informed of the pod’s existence and the application is created the user information will not change inside YuniKorn. Kubernetes however does not guarantee that the shim will be informed of the pod creation before any modifications are made to the pod. This would leave a short window for a user to create a pod and then update the pod. Allowing [...]
+
+The admission controller must therefore validate update requests that include changes to the annotation. It is the only way to guarantee that the user information in the annotations cannot be modified.
+
+The YuniKorn admission controller is currently only registered as a mutating admission controller for pods with the operation type “create”. This can be extended to include operation type “update” for pods. A validating hook is registered for the configmap object, this one could be extended to include the pod object.
+
+This means that there are two options for the update operation:
+- mutating
+- validating
+
+In the validating case we can simply reject the change if the annotation is changed as part of the update request. In the mutating case we can undo the change by overwriting the annotation with the original value. Both solutions will work, and will take about the same amount of processing time in the admission controller.
+
+The proposal is to add the pod object to the validating webhook for update operations.
+
+### Backwards compatibility
+
+The current label will remain supported for the 1.x minor releases. Deprecation will be announced with the first release that supports the new annotation. Messages mentioning the processing of the old label will also be logged at a WARN level in the logs. The existing behaviour might require changes to the default setting see [Configuration](#configuration) for more details.
+
+Removal of processing of the currently supported label should be part of the next major release. The next major release is 2.0.0. This is based on the fact that we do not have a deprecation policy defined as yet.
+
+Preference in processing will be with the new annotations. In the case that both the user label and new annotations are present on the namespace the new annotation will be used. Using both the user label and new annotations, i.e. merging of the two, will not be supported.
+
+## Providing user and group externally
+
+As with the current label for the username we could allow for providing user and group information via annotations on the submitted objects. Although this is not a secure solution, as is pointed out in the introduction, support for this option might be a requirement to support certain use cases.
+
+We must build in an option to turn the functionality off and or mitigations to limit the possibility to bypass reading the authentication information. Based on the earlier diagram of the admission controller being used to intercept the authentication information a similar diagram for this use case. The admission controller will validate the annotation provided by the user, or system, and enforce compliance with the rules set by the administrator.
+
+
+
+
+### Pods from controllers
+
+The first use case for allowing the user information to be provided on pods when they are created is linked to controller created pods.
+
+To correctly account for these pods the admission controller should be able to not overwrite the user information if the submitting user is one of a specific group of users. Depending on the way the system is configured all controllers run as the same service account or each controller uses a specifically created service account. The [RBAC documentation](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#controller-roles) describes the two options.
+
+If the controller manager runs with the option `--use-service-account-credentials` each controller uses a service account in the kube-system namespace. Running with the option set is recommended from an auditing point of view.
+
+If the option is not enabled, which seems to be the default, the controller submits the pods as the user `system:kube-controller-manager`.
+
+Both values of the option must be supported by the admission controller. That means the admission controller must have an allow list which contains at least the following users:
+- `system:serviceaccount:kube-system:*`
+- `system:kube-controller-manager`
+
+An option would be to further limit the service accounts to a subset of the service account from the kube-system namespace. The configuration should support this without changes, it would come down to a change of the default value for the allow list.
+
+There is only a limited set of controllers that can create pods. The group information does, at this point in time, not look deterministic enough to warrant a group allow list.
+
+Note that the user system:kube-controller-manager name is based on some older github issue references and needs to be confirmed in a test setup.
+
+### External applications
+
+In a similar setup applications could be used to provide a frontend for pod creation. These front-ends might or might not use individual users to submit pods. They could even leverage jobs or other objects to indirectly create pods.
+
+As an example an application might be used to provide a simple user interface to create a Spark application. Apache Airflow, Livy or similar tools come to mind.
+
+The admission controller should allow the annotation to be provided as part of the pod creation. This follows the same code and configuration path as designed for the pods submitted by the controllers.
+
+This functionality must be configurable separately from the pods created by the controllers. A similar allow list should be configurable to limit the user. For this use case we also should support an allow list for groups of users. This should allow administrators to specify one or more groups of users that are allowed to set the user information on the pods.
+
+For the first design there are no plans to add deny lists for either groups or users.
+
+## Configuration
+
+As described in the previous sections, making the behaviour configurable is a requirement. The proposal is to add the following configuration values for the behaviour. All configuration options are prefixed by “ADMISSION_CONTROLLER_”.
+
+
+The setting should be migrated to a configuration object or file when that is implemented for the admission controller. At this point the configuration uses environment settings as part of the deployment file.
+
+
+|Name|Description|Default|
+|--|--|--|
+|`BYPASS_AUTH`|Allow external users to create pods with user information set. Type: boolean |false
+|`BYPASS_CONTROLLERS`|Allow controller users to create pods with user information set. Type: boolean |true
+|`SYSTEM_USERS`|Regular expression for the allowed controller service account list. Type: string|"system:serviceaccount:kube-system:*"
+|`EXTERNAL_USERS`|Regular expression for the allowed external user list. Type: string|""
+|`EXTERNAL_GROUPS`|Regular expression for the allowed external group list. Type: string|""
+
+The configuration will be read as part of the startup of the admission controller. Automatic refresh or reload of the settings to allow changes while the admission controller is running is not part of the design as it is not supported for any other settings at this moment.
+
+If `BYPASS_AUTH` is set to true the admission controller will not add the annotation to a pod if the annotation is not present and the deprecated user labell is set. If the annotation is not set and the user label is not set the new annotation will be added. In the case that `BYPASS_AUTH` is false, the default, the admission controller will always add the new annotation, regardless of the existence of the deprecated label.
+
+The behaviour will take into account the user and group filters as defined in `EXTERNAL_USERS` and `EXTERNAL_GROUPS`. This means that retaining the existing behaviour and preferencing the existing label will require changing a default setting.
diff --git a/versioned_docs/version-1.2.0/design/user_group_manager.md b/versioned_docs/version-1.2.0/design/user_group_manager.md
new file mode 100644
index 000000000..e70cad961
--- /dev/null
+++ b/versioned_docs/version-1.2.0/design/user_group_manager.md
@@ -0,0 +1,410 @@
+---
+id: user_based_resource_usage_tracking
+title: User Based Resource Usage Tracking
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Tracking resource allocation usage is currently limited to a queue. An application, which owns the allocation, is the source of that resource usage. All applications that are part of the queue will count towards the resource usage of the queue. A queue could also have a resource usage quota set. The queue’s resource quota is enforced during the scheduling cycle.
+
+An application is always owned by a user/group. One user/group could own applications in one or more queues. Like a queue a user/group could have a limit set. A limit can specify a resource usage quota, and an application usage limit. Tracking these usages per user/group is currently not implemented and is covered as part of this design.
+
+## Goals
+
+- Specify tracking usage in relation to the following objects:
+ - users
+ - applications
+ - queues
+- An interface to define how to track running applications:
+ - Read a user/group running applications
+ - Increase a user/group running applications *)
+ - Decrease a user/group running applications
+
+- An interface to define how to track usage:
+ - Read a user/group resource usage
+ - Increase a user/group resource usage *)
+ - Decrease a user/group resource usage
+
+ *) This call could be extended for enforcement later.
+
+- Expose user/group tracked information through REST API
+
+## Non Goals
+- User and group retrieval is part of the k8shim which is out of scope
+- Exposing usage as a metric into prometheus
+- Specify enforcement rules
+- Implementation of enforcement
+- Interface definition to allow enforcement
+
+## Tracking definitions
+
+### User vs Group tracking
+
+Tracking is based on the _security.UserGroup_ object defined in the core scheduler. User and group information is retrieved by the shim and communicated via the _UserGroupInformation_ object as defined in the scheduler interface. A _security.UserGroup_ object contains:
+- user (exactly 1)
+- groups (0 or more)
+
+Limits can be configured for both users and groups at the same time in the same queue. So, in the case of users for which no limit has been configured explicitly but group (to which the user belongs to) limits have configured works based on the group limits configuration. In case of users for which a limit has been configured explicitly works based on its own configured limits irrespective of whether configuration has been set for the corresponding group (to which the user belongs to) o [...]
+
+"*" is also allowed for both users and groups limit configuration. In this case, any user or group needs to go through enforcement checks to ensure limit settings are being honoured properly. However, the tracker does not need to track the wildcard user as a specific user or group is already being tracked during their activities.
+
+### Group limitations
+
+Users can belong to zero, one or multiple groups. In some systems, like unix, a concept of a primary group exists. This concept is not universal and there is no consistent definition of a primary group either. If the system does not define a primary group how would we distinguish between the groups?
+
+In the case the user is a member of multiple groups the assignment of usage to a group also becomes problematic. The decision which group to assign the usage to becomes arbitrary. As an example consider a user with multiple groups assigned. Joe has groups dev and test assigned. When an application is started does that application count towards the limits for the dev group or against the limits for the test group or both?
+
+The _UserGroup_ object contains a slice, list or array, of groups which has a fixed order from the moment of creation. The group resolution configured might not guarantee an ordering which means we cannot rely on the order in the _UserGroup_ to be stable. A stable and pre-defined way of choosing a group to track usage against must be defined.
+
+The first point to define is that tracking can only occur against zero (0) or one (1) group. Usage can never be tracked against multiple groups. If a user is a member of multiple groups, a group is selected when the first usage is registered for the application. The group selected out of the list of groups in the _UserGroup_ object to track against is based on the hierarchical queue limit configuration. The hierarchy is considered bottom up, starting from the leaf up to the root. The fir [...]
+
+If no group limits are defined, in the hierarchy, usage is not tracked for a group. If no groups are defined in the _UserGroup_ object no usage will be tracked against a group.
+
+The tracker considers only the group received as part of the _UserGroup_ object at the time of first usage registration for the application in the tracker. In case of group changes while the user’s application is actively being tracked, the tracker does not consider the group change. However, If the user submits a new application, the most current group information at that point in time will be used.
+
+Until there is full support for groups in the k8shim, group tracking will be limited. No groups are passed by the k8shim to the core. This results in the _UserGroup_ object in the current implementation to always have only one group. The group name will be the same as the username. We also have excluded configuration processing for the tracking. For the current tracking implementation as described in the document these limitations will result in the user and group linked to the user to t [...]
+
+### Cluster or queue based tracking
+
+Tracking user usage will be based on the usage per queue. The full hierarchy of the queue will be tracked. We should not just track the leaf or root of the hierarchy. The overall cluster usage is the same as the usage of the root queue. Cluster level usage does not need separate tracking.
+
+The overhead of tracking per queue should be really limited. In most cases a user will run their workloads in a specific queue. Deeply nested queues do also add overhead to the tracking. During testing the impact of workload spread and or deep nesting needs to be assessed. If needed this can be called out in the documentation.
+
+We also need to take into account that multiple members of a group could be using different queues in the hierarchy. If tracking usage would be limited to the leaf level, aggregating group usage becomes more difficult and loses transparency.
+
+The documentation around limits, part of the enforcement, reference different limits at different levels for users or groups. Tracking at just the leaf or root level could break that functionality. The design should support the current documented features. If during enforcement design a different decision is made then the tracking is not the deciding factor.
+
+## Application tracking
+
+Applications go through a number of states within the core scheduler. Allocations linked to applications are the triggers for application state changes. From a tracking perspective we need to define what is considered a running application.
+
+Since placeholders and placeholder timeout can play a role in state changes the implementation of the running application tracking must take into account these cases:
+- no gang defined
+- gang defined, all or some placeholders replaced
+- gang style "soft" defined, all placeholders time out
+- gang style "hard" defined, all placeholders time out
+
+### Running state entry
+
+The application when submitted and placed into a queue is in the _New_ state. At this point there is no allocation or pending request present on the application. After one or more requests, _AllocationAsks_, are added the application moves into an _Accepted_ state. The _Accepted_ state is exited when the first _Allocation_ is added to the application. The application then transitions into the Starting state.
+
+At this point a resource quota would be used by the application and the application should be considered as running from a tracking perspective. This means that the addition of the first _Allocation_ onto the application also must be the trigger point for the increase of the running applications. This trigger point for tracking is when the application is in the _Accepted_ state. This is also the point at which the group for the usage tracking needs to be set as described in the [group li [...]
+
+Note that currently, the application state transition code block in application_state.go updates the application running queue metrics when the application enters _Running_ state. The metric must be updated to be consistent with the above definition of a running application. Linking this back to a state transition the entry into the Starting state should be used.
+
+### Running state exit
+
+An application should be considered running as long as at least one allocation is assigned to the application. The removal of the last allocation should decrease the number of running applications. However, we also need to consider the pending requests.
+
+As part of the application states the application enters into a _Completing_ state when the last allocation is removed and there are no pending requests. This automatically progresses into the _Completed_ state if nothing changes for the application. Applications going into the _Completing_ state all have no allocations left and pending requests are considered. The entry of the _Completing_ state thus could be used as a point for triggering the decrease of the running application count.
+
+However there are circumstances that the application goes from the _Completing_ state back into a _Running_ state. This is linked to auto restart of a pod by Kubernetes. This case must be handled.
+
+A last case that needs to be considered is an application that fails. If an application fails it moves into the Failing state. The application does not enter _Completing_ but could have incremented the running applications. A failure could be triggered by the gang scheduling style "hard" or the removal of a partition.
+
+## Resource usage tracking
+
+Resource usage is tracked simply based on the addition or removal of an allocation.
+
+Removal of an allocation is an out-of-band action. The shim notifies the core scheduler that an allocation no longer exists and the core updates the tracking based on that removal. If an allocation is preempted the core initiates the removal but still gets notified by the shim that the removal has occurred. Until the confirmation of the shim is received no tracking updates must be created.
+
+The replacement of a placeholder by the real allocation for gang scheduling follows a similar concept. However, in that case there is a possibility that there is a discrepancy between the placeholder size and the real allocation. See [YUNIKORN-982](https://issues.apache.org/jira/browse/YUNIKORN-982) for more detailed information on this issue. The changes that fixed the accounting for the queues and nodes also need to be taken into account for user based resource tracking.
+
+Adding a new allocation to an application triggers the update of the tracked resources for the user and or group as described in the [group limitations](#group-limitations). Enforcement is not a goal of the design document but the enforcement implementation needs to be taken into account. The point to add the resource usage must line up with the point that the queue structure is currently updated and the queue quotas are enforced.
+
+The order of the updates: queue structure first then user or user first then queue structure is debatable. The current proposal is to implement the following:
+- Update user and group based usage
+- Queue quota enforcement and update
+- If the queue enforcement fails, roll back the user and group based update.
+
+## Scheduler transparency
+
+Resource usage tracking is based on the allocated resource, and linked to the application’s queue path and the application’s _user_ field. Application tracking is a simple count of the applications, and is linked to the application’s queue path and the application’s user field.
+As an extension on this we could consider adding the _applicationID_ into the tracking of the running applications to help with troubleshooting. This is part of the current proposal. The proposal is to not link the running application tracking to the applicationID.
+
+The user field of an application is the earlier mentioned _security.UserGroup_ object and has all the identifying details needed to track the usage.
+
+From the schedulers perspective there should be no special setup needed. The tracking should always work without the need to pre-create a user, group or queue based entry. This means that the tracking cannot fail even if the user or group has no enforcement linked to it and does not exist in the tracker.
+
+In summary the tracker must implement:
+- User and or group creation on the fly.
+- Auto cleanup of tracking information no longer needed.
+- Configuration processing needed for enforcement.
+
+## User Group Manager (UGM)
+
+Tracking all the resources will be part of a new self-contained module in the scheduler. This means that there must be no imports from the `scheduler.objects` package. This will allow us to run the tracker separately and push updates, if needed, into a background processing queue.
+
+There will be one user group manager created per partition as part of the partition creation process. There will be no tracking of user quotas over a partition boundary.
+
+The new self-contained module, user group manager contains all specifications and code for tracking and can be extended in the future to include enforcement code.
+
+Tracking the resources and application count for all users occurs from startup. This happens irrespective of whether enforcement is configured for the user or group. By starting tracking independently of the enforcement changes in enforcement will always have a consistent view of the resources tracked at least at a user level. Recovery must be tested in combination with the tracking code.
+
+Storing data in the user group manager is based on the _security.UserGroup_ structure content. The built-in _UserGroupCache_ resolution cannot guarantee that the same object is returned for the same user each time.
+
+### UserGroup cache impact
+
+The concept of a stable user entry in the cache is not currently present in the _UserGroupCache_. This concept will most likely also not be possible based on the fact that group resolution does not guarantee ordering. Based on the assumption that we cannot guarantee group ordering for all group resolutions that are possible the tracking will not rely on it.
+
+Currently, _UserGroupCache_ holds the _UserGroup_ object entry only for the cache period. This is either 300 seconds for a positive resolution or 30 seconds for a negative resolution. After this cache time the entry will be deleted as part of the cache cleanup/expiration process. If the same user submits a second application after the expiry, a new _UserGroup_ object will be created and stored in the cache. The _UserGroup_ object has a property "resolved", which holds a timestamp at whic [...]
+
+Other internal changes might be needed to make sure all _UserGroup_ objects are cached correctly and returned as expected internally in the cache. These are left as implementation details for the cache changes.
+
+### Tracking structure
+
+The user group Manager must be able to track usage for users and groups. Users are the primary access point for all tracking. The user group Manager structure will implement the [tracker interface](#tracker-interface).
+
+````
+package user_group_manager
+
+type Manager struct {
+ users map[string]*userTracker
+ groups map[string]*groupTracker
+
+ sync.RWMutex
+}
+(m *Manager) checkAppGroup(queuePath, applicationID string, user *security.UserGroup)
+````
+
+All objects in the user group manager will be managed internally. Creation of user and or group tracker objects is based on the request to track usage. The objects are added to one of the two structures in the user group manager. Adding user or group objects can only be triggered by an increase of tracked resources. The creation of the user and or group tracker objects is handled by the user group manager. The user group manager is also responsible for setting the usage information for t [...]
+
+The resolution of the group that the usage is tracked against for an application is handled by the user group Manager via the checkAppGroup. As part of an increase of resource usage the Manager will check if the application is already tracked for the user. If the application is already tracked nothing changes. If the application is not tracked the manager will resolve the group to track against and update the userTracker object. If there is no active tracker registered for the group a ne [...]
+
+Removal of user or group objects can only be triggered after a decrease of tracked resources has removed the last usage. Usage tracked for a user must be zero. For group tracker objects the application reference count must be zero.
+Removal of the user and or group tracker objects is done periodically. This cleanup must be `go routine` safe. Updates to either map could be triggered from different routines, scheduler allocation processing or a release of an allocation by a shim.
+
+### Tracker interface
+
+The interface describes the functions that will be exposed by the user group manager. The other scheduler components can rely on the functions defined in the interface to be available to increase, decrease applications and or resource usage and get the tracked resources.
+
+```
+package user_group_manager
+
+type Tracker interface {
+ GetUserResources(user *security.UserGroup) *Resource
+ GetGroupResources(group string) *Resource
+
+ GetUsersResources() []*userTracker
+ GetGroupsResources() []*groupTracker
+
+ IncreaseTrackedResource(queuePath, applicationID string, usage *Resource, user *security.UserGroup) error
+ DecreaseTrackedResource(queuePath, applicationID string, usage *Resource, removeApp bool, user *security.UserGroup) error
+}
+```
+
+The _IncreaseTrackedResource()_ and _DecreaseTrackedResource()_ calls will implement the tracking data updates. The Get* calls are providing safe access to the tracked data to be used as part of exposing the data in the REST interface. See [exposure of usage tracking](#exposure-of-usage-tracking) for more detail.
+
+Applications are added automatically during the call to increase the tracked resources. The first time the applicationID is encountered the structures are updated accordingly and tracking starts. The removal of the application is triggered by leaving the [running state](#running-state-exit), this is equivalent to the removal of the last usage and must also set the removeApp flag to true.
+
+Calling _IncreaseTrackedResource()_ and _DecreaseTrackedResource()_ with empty or nil parameters is not allowed and will return an error to be handled by the caller. The _GetUserResource_ and _GetGroupResource_ calls can return a _nil_ Resource value. The _GetUsersResource_ and _GetGroupsResource_ calls can return an empty slice.
+
+### User tracking
+
+The userTracker contains all tracked usage information for a user. Changes for all usage, even the group usage, are part of the change that gets triggered on the user object.
+
+The userTracker object contains a reference map to the usage for each application that is tracked linking it to the groupTracker object for the specific application. Multiple applications can be tracked against the same groupTracker object.
+
+The group references will be set for each application when usage is added for an application and is not mutable. The existence of a groupTracker for the application can be checked by calling _hasGroupForApp_ on the userTracker object. It returns true in case the application ID exists in the group mapping, false if it does not. The return value is only based on the existence of the applicationID as the key, not on the value it references.
+
+The groupTracker can be set in the userTracker object by calling _setGroupForApp_. If the applicationID already exists, the call to set the groupTracker reference is a noop, nothing will change. The reference cannot be changed after it is set. The groupTracker reference can be nil. In the case of a nil reference, no tracking for that application against a group is performed. Even a nil reference cannot be replaced.
+
+An increase or decrease of the resources tracked for the user requires the applicationID to be set to a non-empty value. On removal of the last allocation for the application the group reference will be removed from the userTracker object. To remove a running application from the userTracker the applicationID must be set in the _decreaseResource_ call and the flag removeApp must be set to true. Removal of the groupTracker reference can only occur as part of a _decreaseResource_ call.
+
+If there are no applications left in the group references the object will be removed by the cleanup run from the user group manager. This will also mean that there is no usage tracked anymore by the referenced queueTracker object.
+
+The usage per queue is stored in a queue structure specific for this user. The queueTracker object provides a hierarchical structure. This structure starts at the root of the queue hierarchy and only contains child queues for which a usage is being tracked. The hierarchy must be updated, add or delete of child entries, as part of the updates that are processed.
+
+The creation of the root queueTracker is part of the creating a new userTracker object.
+
+```
+package user_group_manager
+
+type userTracker struct {
+ userName string
+ group map[string]*groupTracker
+ queue *queueTracker
+
+ sync.RWMutex
+}
+
+newUserTracker(queuePath string, user *security.UserGroup) *userTracker
+
+(ut *userTracker) increaseResource(queuePath, applicationID string, usage *Resource)
+(ut *userTracker) decreaseResource(queuePath, applicationID string, usage *Resource, removeApp bool)
+(ut *userTracker) hasGroupForApp(applicationID string) bool
+(ut *userTracker) setGroupForApp(applicationID string, groupTrack *groupTracker)
+```
+
+### Group tracking
+
+The groupTracker contains all tracked usage information for a group. This object can be shared by multiple users for one or more applications. It adds the applications that are tracked for this specific group automatically to the list. For this it expects an applicationID to be set to a non-empty value for each _increaseResource_ call.
+
+To simply decrease the resources tracked for the group the applicationID could be empty. On removal of the last allocation for the application the application will be removed from the groupTracker object. To remove a running application from the groupTracker the applicationID must be set in the _decreaseResource_ call and the flag removeApp must be set to true.
+
+If there are no entries left in the applications tracked in the groupTracker the object will be removed by the cleanup run from the user group manager. This will also mean that there is no usage tracked any more by the referenced queueTracker object.
+
+The usage per queue is stored in a queue structure specific for this group. The queueTracker object provides a hierarchical structure. This structure starts at the root of the queue hierarchy and only contains child queues for which a usage is being tracked. The hierarchy must be updated, add or delete of child entries, as part of the updates that are processed.
+
+The creation of the root queueTracker is part of the creating a new groupTracker object.
+
+```
+package user_group_manager
+
+type groupTracker struct {
+ groupName string
+ applications map[string]bool
+ queue *queueTracker
+
+ sync.RWMutex
+}
+
+newGroupTracker(queuePath string, group string) *groupTracker
+
+(gt *groupTracker) increaseResource(queuePath, applicationID string, usage *Resource)
+(gt *groupTracker) decreaseResource(queuePath, applicationID string, usage *Resource, removeApp bool)
+```
+
+### Queue tracking
+
+The queueTracker acts as a base for both user and group tracking. It provides the actual location for the tracked usage.The top of the structure, which is referenced from the userTracker or groupTracker objects, contains the root queue usage. The queueTracker is only referenced by one userTracker or one groupTracker object and must not be shared. This means that we should not require any locking within the queueTracker object.
+
+The queue hierarchy below the root, for which a usage is registered, is part of the childQueues. Each part of the full queue path creates a new level of objects. This structure shows a sparse hierarchical queue structure. It may not represent the full queue structure as known in the scheduler.
+
+The creation of the root queueTracker returns a plain object without usage set and a fixed queueName of "root". The usage should then be updated by a call to _increaseResource_. The call must handle the recursive creation of the queue path as given. It expects an applicationID to be set to a non-empty value for each call.
+
+To simply decrease the resources tracked for the queue the applicationID can be empty. On removal of the last allocation for the application the application will be removed from the queueTracker. To remove a running application from the queueTracker the applicationID must be set in the _decreaseResource_ call and the flag removeApp must be set to true.
+
+```
+package user_group_manager
+
+type queueTracker struct {
+ queueName string
+ resourceUsage *Resource
+ runningApplications map[string]boolean
+
+ childQueues map[string]*queueTracker
+}
+
+newQueueTracker() *queueTracker
+
+(qt *queueTracker) increaseResource(queuePath, applicationID string, usage *Resource)
+(qt *queueTracker) decreaseResource(queuePath, applicationID string, usage *Resource, removeApp bool)
+```
+
+### Update consistency
+
+All changes are triggered against a userTracker object. Since enforcement of a quota is out of scope of this design we can ignore update failures due to quota exhaustion. There is no requirement or possibility that updates need to be rolled back.
+
+Usage updates can be triggered from two points: allocation by the scheduler and releases by a shim. Allocations as well as releases need to update both user and group usage information. We must guarantee that for both the full hierarchy is changed or neither are changed.
+
+The releases are also processed asynchronously from the scheduling cycle. This means that we have to be able to guarantee that the increases and decreases are applied consistently to both objects also. Locking of the objects to serialise changes must be in place to support this.
+
+The other point is the update ordering and locking. The increase and decrease processing must follow the same order. Both must update the userTracker and related groupTracker objects in the same order. This is a requirement to prevent the processing of being able to deadlock while updating the same user for a simultaneous increase and decrease.
+
+For groupTracker updates that originate from two different users the locking of the groupTracker object is required. One of the two updates will be able to proceed while the other waits for the update to finalise.
+
+A final consideration is the consistency of two updates on the same object that follow each other closely. If the first update removes all existing usage the userTracker or groupTracker object is cleared for removal. While the update is in progress a second update comes in to add a new usage to the same object. The resolution of the userTracker in the user group manager has finished. The code has a reference to the object. Processing is waiting to lock the userTracker object. This point [...]
+
+### Exposure of usage tracking
+
+The usage tracking information that is part of the user group manager must be exposed for external consumption. All our current information is exposed via a REST call.
+
+Based on the current REST api definition for other object the proposal is to expose the following new endpoints:
+
+- /ws/v1/partition/{partitionName}/usage/users
+- /ws/v1/partition/{partitionName}/usage/groups
+
+For both endpoints we expose the full queue hierarchy. As an example below the approximate output for the user's endpoint for one user:
+
+```json
+[
+ {
+ "userName": "user1",
+ "groups": {
+ "app2": "tester"
+ },
+ "queues":
+ {
+ "queuename": "root",
+ "resourceUsage": {
+ "memory": 12000000000,
+ "vcore": 12000
+ },
+ "runningApplications": ["app1", "app2"],
+ "children": [
+ {
+ "queuename": "root.default",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuename": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": [
+ "app2"
+ ],
+ "children": []
+ }]
+ }
+ }
+]
+```
+
+
+An example below the approximate output for the groups endpoint for one group:
+
+```json
+[
+ {
+ "groupName" : "tester",
+ "applications": ["app2"],
+ "queues":
+ {
+ "queuename": "root",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app2"],
+ "children": [
+ {
+ "queuename": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app2"],
+ "children": []
+ }
+ ]
+ }
+ }
+]
+```
\ No newline at end of file
diff --git a/versioned_docs/version-1.2.0/developer_guide/build.md b/versioned_docs/version-1.2.0/developer_guide/build.md
index 586a630ea..81d5c15fe 100644
--- a/versioned_docs/version-1.2.0/developer_guide/build.md
+++ b/versioned_docs/version-1.2.0/developer_guide/build.md
@@ -33,7 +33,7 @@ Read the [environment setup guide](developer_guide/env_setup.md) first to setup
## Build YuniKorn for Kubernetes
Prerequisite:
-- Go 1.16+
+- Golang: check the `.go_version` file in the root of the repositories for the version Yunikorn requires. The minimum version can change per release branch. Earlier Go versions might cause compilation issues.
You can build the scheduler for Kubernetes from [yunikorn-k8shim](https://github.com/apache/yunikorn-k8shim) project.
The build procedure will build all components into a single executable that can be deployed and running on Kubernetes.
@@ -73,7 +73,8 @@ For the deployment that uses a config map you need to set up the ConfigMap in ku
How to deploy the scheduler with a ConfigMap is explained in the [scheduler configuration deployment](developer_guide/deployment.md) document.
The image build command will first build the integrated executable and then create the docker image.
-Currently, there are some published docker images under [this docker hub repo](https://hub.docker.com/r/apache/yunikorn), you are free to fetch and use.
+If you want to use pre-built images based on a release, please check the [Docker Hub repo](https://hub.docker.com/r/apache/yunikorn).
+
The default image tags are not suitable for deployments to an accessible repository as it uses a hardcoded user and would push to Docker Hub with proper credentials.
You *must* update the `TAG` variable in the `Makefile` to push to an accessible repository.
When you update the image tag be aware that the deployment examples given will also need to be updated to reflect the same change.
@@ -84,9 +85,11 @@ The docker image built from previous step has embedded some important build info
these info with docker `inspect` command.
```
-docker inspect apache/yunikorn:scheduler-latest
+docker inspect apache/yunikorn:scheduler-amd64-latest
```
+The `amd64` tag is dependent on your host architecture (i.e. for Intel it would be `amd64` and for Mac M1, it would be `arm64v8`).
+
This info includes git revisions (last commit SHA) for each component, to help you understand which version of the source code
was shipped by this image. They are listed as docker image `labels`, such as
@@ -106,7 +109,7 @@ The dependencies in the projects are managed using [go modules](https://blog.gol
Go Modules require at least Go version 1.11 to be installed on the development system.
If you want to modify one of the projects locally and build with your local dependencies you will need to change the module file.
-Changing dependencies uses mod `replace` directives as explained in the [Update dependencies](#Updating dependencies).
+Changing dependencies uses mod `replace` directives as explained in the [Update dependencies](#updating-dependencies).
The YuniKorn project has four repositories three of those repositories have a dependency at the go level.
These dependencies are part of the go modules and point to the github repositories.
@@ -159,13 +162,12 @@ Further details on the modules' wiki: [When should I use the 'replace' directive
## Build the web UI
Example deployments reference the [YuniKorn web UI](https://github.com/apache/yunikorn-web).
-The YuniKorn web UI has its own specific requirements for the build. The project has specific requirements for the build follow the steps in the README to prepare a development environment and build how to build the projects.
-The scheduler is fully functional without the web UI.
+The `yunikorn-web` project has specific requirements for the build. Follow the steps in the [README](https://github.com/apache/yunikorn-web/blob/master/README.md) to prepare a development environment and build the web UI. However, the scheduler is fully functional without the web UI.
## Locally run the integrated scheduler
-When you have a local development environment setup you can run the scheduler in your local kubernetes environment.
-This has been tested in a Docker desktop with 'Docker for desktop' and Minikube. See the [environment setup guide](developer_guide/env_setup.md) for further details.
+When you have a local development environment setup you can run the scheduler in your local Kubernetes environment.
+This has been tested in a desktop environment with Docker Desktop, Minikube, and Kind. See the [environment setup guide](developer_guide/env_setup.md) for further details.
```
make run
diff --git a/versioned_docs/version-1.2.0/developer_guide/deployment.md b/versioned_docs/version-1.2.0/developer_guide/deployment.md
index 7e5aa5bfa..82b95f465 100644
--- a/versioned_docs/version-1.2.0/developer_guide/deployment.md
+++ b/versioned_docs/version-1.2.0/developer_guide/deployment.md
@@ -33,12 +33,14 @@ Under project root of the `yunikorn-k8shim`, run the command to build an image u
make image
```
-This command will build an image. The image will be tagged with a default version and image tag.
+This command will build an image. The image will be tagged with a default version, image tag and your build architecture.
**Note** the default build uses a hardcoded user and tag. You *must* update the `IMAGE_TAG` variable in the `Makefile` to push to an appropriate repository.
**Note** the latest yunikorn images in docker hub are not updated anymore due to ASF policy. Hence, you should build both scheduler image and web image locally before deploying them.
+**Note** the imaging tagging includes your build architecture. For Intel, it would be `amd64` and for Mac M1, it would be `arm64v8`.
+
## Setup RBAC for Scheduler
The first step is to create the RBAC role for the scheduler, see [yunikorn-rbac.yaml](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/scheduler/yunikorn-rbac.yaml)
diff --git a/versioned_docs/version-1.2.0/developer_guide/env_setup.md b/versioned_docs/version-1.2.0/developer_guide/env_setup.md
index c45d77e21..98327f0ee 100644
--- a/versioned_docs/version-1.2.0/developer_guide/env_setup.md
+++ b/versioned_docs/version-1.2.0/developer_guide/env_setup.md
@@ -22,8 +22,8 @@ specific language governing permissions and limitations
under the License.
-->
-There are several ways to setup a local development environment for Kubernetes, the two most common ones are `Minikube` ([docs](https://kubernetes.io/docs/setup/minikube/)) and `docker-desktop`.
-`Minikube` provisions a local Kubernetes cluster on several Virtual Machines (via VirtualBox or something similar). `docker-desktop` on the other hand, sets up Kubernetes cluster in docker containers.
+There are several ways to setup a local development environment for Kubernetes, the three most common ones are `Minikube` ([docs](https://kubernetes.io/docs/setup/minikube/)), `docker-desktop` and `kind` ([kind](https://kind.sigs.k8s.io/))
+`Minikube` provisions a local Kubernetes cluster on several Virtual Machines (via VirtualBox or something similar). `docker-desktop` on the other hand, sets up Kubernetes cluster in docker containers. `kind` provides lightweight Kubernetes clusters for Windows, Linux and Mac.
## Local Kubernetes cluster using Docker Desktop
@@ -41,18 +41,18 @@ Once Kubernetes is started in docker desktop, you should see something similar b
This means that:
1. Kubernetes is running.
-1. the command line tool `kubctl` is installed in the `/usr/local/bin` directory.
-1. the Kubernetes context is set to `docker-desktop`.
+2. the command line tool `kubctl` is installed in the `/usr/local/bin` directory.
+3. the Kubernetes context is set to `docker-desktop`.
### Deploy and access dashboard
After setting up the local Kubernetes you need to deploy the dashboard using the following steps:
1. follow the instructions in [Kubernetes dashboard doc](https://github.com/kubernetes/dashboard) to deploy the dashboard.
-1. start the Kubernetes proxy in the background from a terminal to get access on the dashboard on the local host:
+2. start the Kubernetes proxy in the background from a terminal to get access on the dashboard on the local host:
```shell script
kubectl proxy &
```
-1. access the dashboard at the following URL: [clickable link](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login)
+3. access the dashboard at the following URL: [clickable link](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login)
### Access local Kubernetes cluster
@@ -87,27 +87,61 @@ Check hypervisor Docker Desktop should have already installed HyperKit. In a ter
chmod +x minikube
sudo mv minikube /usr/local/bin
```
-1. install HyperKit driver (required), you can either use brew or directly via these steps:
+2. install HyperKit driver (required), you can either use brew or directly via these steps:
```shell script
curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-hyperkit
sudo install -o root -g wheel -m 4755 docker-machine-driver-hyperkit /usr/local/bin/
```
-1. update the minikube config to default to the HyperKit install `minikube config set vm-driver hyperkit`
-1. change docker desktop to use minikube for Kubernetes:<br/>
+3. update the minikube config to default to the HyperKit install `minikube config set vm-driver hyperkit`
+4. change docker desktop to use minikube for Kubernetes:<br/>
### Deploy and access the cluster
After the installation is done you can start a new cluster.
-1. start the minikube cluster: `minikube start --kubernetes-version v1.14.2`
-1. start the minikube dashboard: `minikube dashboard &`
+1. start the minikube cluster: `minikube start --kubernetes-version v1.24.7`
+2. start the minikube dashboard: `minikube dashboard &`
### Build impact
When you create images make sure that the build is run after pointing it to the right environment.
Without setting the enviromnent minikube might not find the docker images when deploying the scheduler.
1. make sure minikube is started
-1. in the terminal where you wll run the build execute: `eval $(minikube docker-env)`
-1. run the image build from the yunikorn-k8shim repository root: `make image`
-1. deploy the scheduler as per the normal instructions.
+2. in the terminal where you wll run the build execute: `eval $(minikube docker-env)`
+3. run the image build from the yunikorn-k8shim repository root: `make image`
+4. deploy the scheduler as per the normal instructions.
+
+## Local Kubernetes Cluster with Kind
+
+Kind (Kubernetes in Docker) is a lightweight tool for running lightweight Kubernetes environments. It is very easy to test different Kubernetes versions with kind. You can just select the kind image you want.
+
+### Installation
+
+If you have go installed, you can run `go install sigs.k8s.io/kind@latest`.
+
+Other ways can be found on the Kind [website](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
+
+To use Kind with Kubernetes 1.25, you will need to use kind@v0.15 or greater. The release of kind does allow for particular versions of Kubernetes and you can get that information from the Kind release notes.
+
+### Using Kind
+
+To test a new version of Kubernetes, you can pull a corresponding image from kind's repo.
+
+Creating a v1.24.7 Kubernetes Cluster: `kind create cluster --name test --image kindest/node:v1.24.7`
+
+Deleting a kind cluster: `kind delete cluster --name test`
+
+### Loading your images
+
+In order to use a local image, you have to load your images into kind's registry. If you run `make image`, you could use the following command to load your kind image. This assumes AMD64 architecture.
+
+The scheduler, web-ui and admission-controller examples are below:
+scheduler:
+`kind load docker-image apache/yunikorn:scheduler-amd64-latest`
+
+web:
+`kind load docker-image apache/yunikorn:web-amd64-latest`
+
+admission-controller:
+`kind load docker-image apache/yunikorn:admission-amd64-latest`
## Debug code locally
@@ -135,17 +169,17 @@ For a generic view on how to access a multiple cluster and integrate it follow t
Or follow these simplified steps:
1. get the Kubernetes `config` file from remote cluster, copy it to the local machine and give it a unique name i.e. `config-remote`
-1. save the `KUBECONFIG` environment variable (if set)
+2. save the `KUBECONFIG` environment variable (if set)
```shell script
export KUBECONFIG_SAVED=$KUBECONFIG
```
-1. add the new file to the environment variable
+3. add the new file to the environment variable
```shell script
export KUBECONFIG=$KUBECONFIG:config-remote
```
-1. run the command `kubectl config view` to check that both configs can be accessed
-1. switch context using `kubectl config use-context my-remote-cluster`
-1. confirm that the current context is now switched to the remote cluster config:
+4. run the command `kubectl config view` to check that both configs can be accessed
+5. switch context using `kubectl config use-context my-remote-cluster`
+6. confirm that the current context is now switched to the remote cluster config:
```text
kubectl config get-contexts
CURRENT NAME CLUSTER AUTHINFO NAMESPACE
diff --git a/versioned_docs/version-1.2.0/developer_guide/translation.md b/versioned_docs/version-1.2.0/developer_guide/translation.md
new file mode 100644
index 000000000..2ab5a0d31
--- /dev/null
+++ b/versioned_docs/version-1.2.0/developer_guide/translation.md
@@ -0,0 +1,126 @@
+---
+id: translation
+title: Translation
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Yunikorn website adopting Docusaurus manages documentations.
+[Docusaurus i18n system](https://docusaurus.io/docs/i18n/tutorial) allows developer to translate documentations.
+For adding a new language translation, developers should do following operations.
+
+## Modifying the docusaurus.config.js for supporting the new language
+Assume the translations by a **new language keyword**.
+Expected results will be like this figure.
+
+
+Current Yunikorn website include the en and zh-cn documentaions.
+If developer want to add a new translation with **new language keyword** including fr, jp, the developers need to modify the `i18n` in `docusaurus.config.js`.
+```
+i18n: {
+ defaultLocale: 'en',
+ locales: ['en', 'zh-cn', '<new language keyword>'],
+ localeConfigs: {
+ en: {
+ label: 'English',
+ },
+ "zh-cn": {
+ label: '中文',
+ },
+ "<new language keyword>": {
+ label: 'test',
+ }
+ },
+ },
+```
+## Updating the help information in local-build.sh
+Adding the **new language keyword** to the locale list in `print_usage` function.
+```
+Usage: $(basename "$0") run [locale] | build | clean | help
+ run build the website, and launch the server in a docker image.
+ a locale can be specified, currently supported: "en", "zh-cn", "<new language keyword>"
+```
+
+## Copying latest files to i18n
+```
+mkdir -p i18n/<new language keyword>/docusaurus-plugin-content-docs/current
+cp -r docs/** i18n/<new language keyword>/docusaurus-plugin-content-docs/current
+mkdir -p i18n/<new language keyword>/docusaurus-plugin-content-pages
+cp -r src/pages/** i18n/<new language keyword>/docusaurus-plugin-content-pages
+mkdir -p i18n/<new language keyword>/docusaurus-theme-classic
+```
+
+## Adding the translated information in sidebar.json and footer.json
+Create sidebar.json and and footer.json in docusaurus-theme-classic.
+For example, the footer.json context is following.
+```
+{
+ "link.item.label.Get Involved": {
+ "message": "参与"
+ },
+ "link.title.Code Repositories": {
+ "message": "代码库"
+ },
+ "link.item.label.People": {
+ "message": "人们"
+ },
+ "link.title.Blog": {
+ "message": "博客"
+ },
+ "link.title.Community": {
+ "message": "社区"
+ }
+}
+```
+
+
+
+## linking img, assest and styles.module.css
+Creating linking files in the `i18n/new language keyword/docusaurus-plugin-content-pages`.
+
+```
+# cleaning the duplicate files
+rm -rf img
+rm styles.module.css
+# linking
+ln -s ../../../static/img
+ln -s ../../../src/pages/styles.module.css
+```
+
+Creating the linking file in the 'i18n/new language keyword/docusaurus-plugin-content-docs/current'.
+```
+# cleaning the duplicate files
+rm -rf assests
+# linking
+ln -s ../../../../docs/assets
+```
+
+## Adopting relative paths
+There are some image urls adopting absolute path in `src/pages/index.js` and adopting absolute path could cause png missing.
+Developers could make sure that the img link in `/i18n/new language keyword/docusaurus-plugin-content-pages/index.js` is work.
+For example, there is a `resource-scheduling.png` in `index.js` and the png url is `/img/resource-scheduling.png`.
+```
+/img/resource-scheduling.png -> ./img/resource-scheduling.png
+```
+## Test
+```
+./local-build.sh run <new language keyword>
+```
+
diff --git a/versioned_docs/version-1.2.0/get_started/core_features.md b/versioned_docs/version-1.2.0/get_started/core_features.md
index 8f2258966..8c9076fe5 100644
--- a/versioned_docs/version-1.2.0/get_started/core_features.md
+++ b/versioned_docs/version-1.2.0/get_started/core_features.md
@@ -34,13 +34,22 @@ fine-grained controls on resource quotas, resource fairness and priorities, whic
for a multi-tenancy computing system.
## Hierarchy Resource Queues
-
Hierarchy queues provide an efficient mechanism to manage cluster resources. The hierarchy of the queues can logically
map to the structure of an organization. This gives fine-grained control over resources for different tenants. The YuniKorn
UI provides a centralised view to monitor the usage of resource queues, it helps you to get the insight how the resources are
used across different tenants. What's more, By leveraging the min/max queue capacity, it can define how elastic it can be
in terms of the resource consumption for each tenant.
+## Gang Scheduling
+An application can request a set of resources, i.e. a gang, to be scheduled all at once.
+The gang defines all the resources the application requires to start.
+During the first scheduling phase all resources requested will be reserved.
+The application will only be started when all requested resources are available.
+
+Reservation duration and application behaviour when the reservation fails are configurable.
+It is even possible to create multiple gangs of different specifications for one application.
+See the [gang design](design/gang_scheduling.md) and the Gang Scheduling [user guide](user_guide/gang_scheduling.md) for more details.
+
## Job Ordering and Queuing
Applications can be properly queued in working-queues, the ordering policy determines which application can get resources first.
The policy can be various, such as simple `FIFO`, `Fair`, `StateAware` or `Priority` based. Queues can maintain the order of applications,
@@ -59,7 +68,6 @@ With consideration of weights or priorities, some more important applications ca
This is often associated with resource budget, a more fine-grained fairness mode can further improve the expense control.
## Resource Reservation
-
YuniKorn automatically does reservations for outstanding requests. If a pod could not be allocated, YuniKorn will try to
reserve it on a qualified node and tentatively allocate the pod on this reserved node (before trying rest of nodes).
This mechanism can avoid this pod gets starved by later submitted smaller, less-picky pods.
@@ -71,3 +79,13 @@ Throughput is a key criterion to measure scheduler performance. It is critical f
If throughput is bad, applications may waste time on waiting for scheduling, and further impact service SLAs.
When the cluster gets bigger, it also means the requirement of higher throughput. The [performance evaluation based on Kube-mark](performance/evaluate_perf_function_with_kubemark.md)
reveals some perf numbers.
+
+## MaxApplication Enforcement
+MaxApplication enforcement feature allows users to limit the number of running applications for a configured queue.
+This feature is critical in large scale batch workloads.
+Without this feature, when there are a large number of concurrent jobs launched, they would compete for resources and a certain a mount of resources will be wasted, which could lead to job failure.
+The [Partition and Queue Configuration](user_guide/queue_config.md) provides configuration examples.
+
+## CPU Architecture support
+YuniKorn supports running on ARM as well as on AMD/Intel CPUs.
+With the release of YuniKorn 1.1.0 prebuilt convenience images for both architectures are provided in the docker hub.
diff --git a/versioned_docs/version-1.2.0/performance/performance_tutorial.md b/versioned_docs/version-1.2.0/performance/performance_tutorial.md
index b17923838..51c607faa 100644
--- a/versioned_docs/version-1.2.0/performance/performance_tutorial.md
+++ b/versioned_docs/version-1.2.0/performance/performance_tutorial.md
@@ -87,7 +87,7 @@ Before going into the details, here are the general steps used in our tests:
- [Step 2](#Setup-Kubemark): Deploy hollow pods,which will simulate worker nodes, name hollow nodes. After all hollow nodes in ready status, we need to cordon all native nodes, which are physical presence in the cluster, not the simulated nodes, to avoid we allocated test workload pod to native nodes.
- [Step 3](#Deploy-YuniKorn): Deploy YuniKorn using the Helm chart on the master node, and scale down the Deployment to 0 replica, and [modify the port](#Setup-Prometheus) in `prometheus.yml` to match the port of the service.
- [Step 4](#Run-tests): Deploy 50k Nginx pods for testing, and the API server will create them. But since the YuniKorn scheduler Deployment has been scaled down to 0 replica, all Nginx pods will be stuck in pending.
-- [Step 5](../user_guide/trouble_shooting.md#restart-the-scheduler): Scale up The YuniKorn Deployment back to 1 replica, and cordon the master node to avoid YuniKorn allocating Nginx pods there. In this step, YuniKorn will start collecting the metrics.
+- [Step 5](../user_guide/troubleshooting.md#restart-the-scheduler): Scale up The YuniKorn Deployment back to 1 replica, and cordon the master node to avoid YuniKorn allocating Nginx pods there. In this step, YuniKorn will start collecting the metrics.
- [Step 6](#Collect-and-Observe-YuniKorn-metrics): Observe the metrics exposed in Prometheus UI.
---
diff --git a/versioned_docs/version-1.2.0/user_guide/gang_scheduling.md b/versioned_docs/version-1.2.0/user_guide/gang_scheduling.md
index f7593a573..21739bce2 100644
--- a/versioned_docs/version-1.2.0/user_guide/gang_scheduling.md
+++ b/versioned_docs/version-1.2.0/user_guide/gang_scheduling.md
@@ -30,7 +30,7 @@ will be waiting in the queue. Apps are queued in hierarchy queues,
with gang scheduling enabled, each resource queue is assigned with the
maximum number of applications running concurrently with min resource guaranteed.
-
+
## Enable Gang Scheduling
@@ -46,26 +46,33 @@ treated as the same kind in the scheduler.
### Prerequisite
-For the queues which runs gang scheduling enabled applications, the queue sorting policy needs to be set either
-`FIFO` or `StateAware`. To configure queue sorting policy, please refer to doc: [app sorting policies](user_guide/sorting_policies.md#Application_sorting).
+For the queues which runs gang scheduling enabled applications, the queue sorting policy should be set to `FIFO`.
+To configure queue sorting policy, please refer to doc: [app sorting policies](sorting_policies.md#application-sorting).
+
+#### Why the `FIFO` sorting policy
-:::info Why FIFO based sorting policy?
When Gang Scheduling is enabled, the scheduler proactively reserves resources
for each application. If the queue sorting policy is not FIFO based (StateAware is FIFO based sorting policy),
the scheduler might reserve partial resources for each app and causing resource segmentation issues.
-:::
+
+#### Side effects of `StateAware` sorting policy
+
+We do not recommend using `StateAware`, even-though it is a FIFO based policy. A failure of the first pod or a long initialisation period of that pod could slow down the processing.
+This is specifically an issue with Spark jobs when the driver performs a lot of pre-processing before requesting the executors.
+The `StateAware` timeout in those cases would slow down processing to just one application per timeout.
+This in effect will overrule the gang reservation and cause slowdowns and excessive resource usage.
### App Configuration
On Kubernetes, YuniKorn discovers apps by loading metadata from individual pod, the first pod of the app
-is required to enclosed with a full copy of app metadata. If the app doesn’t have any notion about the first or second pod,
+is required to include a full copy of app metadata. If the app does not have any notion about the first or second pod,
then all pods are required to carry the same taskGroups info. Gang scheduling requires taskGroups definition,
which can be specified via pod annotations. The required fields are:
-| Annotation | Value |
-|----------------------------------------------- |--------------------- |
-| yunikorn.apache.org/task-group-name | Task group name, it must be unique within the application |
-| yunikorn.apache.org/task-groups | A list of task groups, each item contains all the info defined for the certain task group |
+| Annotation | Value |
+|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| yunikorn.apache.org/task-group-name | Task group name, it must be unique within the application |
+| yunikorn.apache.org/task-groups | A list of task groups, each item contains all the info defined for the certain task group |
| yunikorn.apache.org/schedulingPolicyParameters | Optional. A arbitrary key value pairs to define scheduling policy parameters. Please read [schedulingPolicyParameters section](#scheduling-policy-parameters) |
#### How many task groups needed?
@@ -99,7 +106,7 @@ This parameter defines the reservation timeout for how long the scheduler should
The timeout timer starts to tick when the scheduler *allocates the first placeholder pod*. This ensures if the scheduler
could not schedule all the placeholder pods, it will eventually give up after a certain amount of time. So that the resources can be
freed up and used by other apps. If non of the placeholders can be allocated, this timeout won't kick-in. To avoid the placeholder
-pods stuck forever, please refer to [troubleshooting](trouble_shooting.md#gang-scheduling) for solutions.
+pods stuck forever, please refer to [troubleshooting](troubleshooting.md#gang-scheduling) for solutions.
` gangSchedulingStyle`
@@ -201,7 +208,8 @@ Annotations:
```
:::note
-Spark driver and executor pod has memory overhead, that needs to be considered in the taskGroup resources.
+The TaskGroup resources must account for the memory overhead for Spark drivers and executors.
+See the [Spark documentation](https://spark.apache.org/docs/latest/configuration.html#application-properties) for details on how to calculate the values.
:::
For all the executor pods,
@@ -285,4 +293,4 @@ Check field including: namespace, pod resources, node-selector, toleration and a
## Troubleshooting
-Please see the troubleshooting doc when gang scheduling is enabled [here](trouble_shooting.md#gang-scheduling).
+Please see the troubleshooting doc when gang scheduling is enabled [here](troubleshooting.md#gang-scheduling).
diff --git a/versioned_docs/version-1.2.0/user_guide/priorities.md b/versioned_docs/version-1.2.0/user_guide/priorities.md
new file mode 100644
index 000000000..11375084b
--- /dev/null
+++ b/versioned_docs/version-1.2.0/user_guide/priorities.md
@@ -0,0 +1,220 @@
+---
+id: priorities
+title: App & Queue Priorities
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+YuniKorn has advanced support for priority scheduling. Priorities are
+specified on a per-task basis, but how those priorities are used can be
+customized for each queue.
+
+## Request Priority
+
+Every allocation request to the scheduler has a numeric priority associated
+with it. Any 32-bit integer value (positive or negative) may be used. Larger
+values indicate higher relative priorities.
+
+When using Kubernetes, priorities are defined in `PriorityClass`
+objects, which are referenced by `Pod` objects via a `priorityClassName`
+property. If no priority class is referenced, a `Pod` inherits the cluster
+default priority, typically `0`.
+
+See the Kubernetes [Pod Priority and Preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/)
+documentation for more details.
+
+## Application Priority
+
+During scheduling, applications have a dynamic priority value which resolves
+to the highest priority outstanding request in that application. This allows
+the scheduler to dynamically reprioritize scheduling decisions.
+
+For example, if an application has two requests, one with priority `10` and
+another with priority `20`, the application's dynamic priority will be `20`.
+If the higher-priority request is satisfied, the application's priority will
+drop to `10`.
+
+When choosing between applications to schedule, the application sorting policy
+will (by default) schedule requests from higher-priority applications first.
+Priority can be ignored when sorting applications by setting the queue
+property `application.sort.priority` to `disabled` on a _leaf_ queue.
+
+## Queue Priority
+
+As with applications, queue priorities are also dynamically computed. For
+_parent_ queues, the queue's priority will be equal to the highest priority
+child queue it contains. For _leaf_ queues, the queue's priority will be
+equal to the highest priority application it contains.
+
+Queue priorities can also be adjusted automatically up or down by a fixed
+amount, specified in the `priority.offset` queue property. This can be useful
+in larger queue hierarchies to establish low or high priority queues.
+
+For example, if a _leaf_ queue with an offset of `5` contains two
+applications, one with priority `10` and another with priority `20`, the
+queue's priority will evaluate to `25` (`20 + 5`). If the higher-priority
+application no longer has requests, the queue's priority will drop to `15`
+(`10 + 5`).
+
+When choosing between child queues to schedule from, the queue sorting policy
+will (by default) schedule requests from higher-priority queues first.
+Priority can be ignored when sorting queues by setting the queue
+property `application.sort.priority` to `disabled` on a _parent_ queue.
+
+## Priority Fencing
+
+By default, priorities have global scope, meaning that higher-priority queues
+will be serviced before lower-priority queues regardless of their location
+in the queue hierarchy.
+
+However, it can be useful to limit prioritization to a subset of queues. This
+can be accomplished by setting the `priority.policy` queue property to
+`fence`. When a queue's priority is fenced, priorities are still evaluated
+within that queue (and subqueues), but the queue's priority itself will always
+evaluate to the `priority.offset` value or `0` if not specified.
+
+## Effective Priority
+
+Because of offsets and fencing, at any time a request may be thought of as
+having an _effective_ (or computed) priority based on its location within
+the queue hierarchy. Requests with higher effective priorities will be
+scheduled before those with lower effective priorities.
+
+## Examples
+
+### Single queue
+
+This example demonstrates a single leaf queue with all properties specified:
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: default
+ properties:
+ application.sort.policy: fifo
+ application.sort.priority: enabled
+ priority.policy: default
+ priority.offset: "0"
+```
+
+### Multitenancy
+
+This example demonstrates a complex queue tree containing multiple tenants
+with subqueues along with a multiple system queues:
+
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: system
+ queues:
+ - name: system-normal
+ properties:
+ priority.offset: "0"
+ - name: system-high
+ properties:
+ priority.offset: "1000"
+ - name: system-low
+ properties:
+ priority.offset: "-1000"
+ - name: tenants
+ properties:
+ priority.policy: "fence"
+ queues:
+ - name: tenant-a
+ properties:
+ priority.policy: "fence"
+ priority.offset: "10"
+ queues:
+ - name: child-a-1
+ - name: child-a-2
+ - name: tenant-b
+ properties:
+ priority.policy: "fence"
+ priority.offset: "0"
+ queues:
+ - name: child-b-1
+ - name: child-b-2
+
+```
+
+
+The `system-normal`, `system-high` and `system-low` queues are unfenced, and
+can therefore be prioritized above any other queue in the system. The
+`system-high` and `system-low` queues have offsets of `1000` and `-1000`
+respectively, so the priority of requests in those queues will be adjusted
+accordingly.
+
+The `tenants` _parent_ queue is priority-fenced, and has no `priority.offset`
+defined, so this queue will always be treated as though it has priority `0`.
+This ensures that normal and high-priority system tasks schedule ahead of
+tenant tasks, and low priority system tasks schedule after tenant tasks.
+
+The `tenant-a` and `tenant-b` _parent_ queues are also priority-fenced,
+preventing tenants from adjusting their priority relative to one another.
+The `tenant-a` queue also has a priority offset to ensure that it always
+schedules before `tenant-b`.
+
+The _leaf_ queues of `tenant-a` and `tenant-b` are not fenced, so tasks from
+the entire `tenant-a` or `tenant-b` subtree will prioritize relative to each
+other, but not outside their respective subtrees.
+
+
+
+In the figure above, multiple requests are shown with various priorities.
+Before scheduling, the queue priorities will be as follows:
+
+* root
+ * system: 1001
+ * system-normal: 10
+ * system-high: 1001
+ * system-low: -997
+ * tenants: 0 (fence)
+ * tenant-a: 10 (fence)
+ * child-a-1: 8
+ * child-a-2: 6
+ * tenant-b: 0 (fence)
+ * child-b-1: 9
+ * child-b-2: 8
+
+Queue traversal starts from the root, descending into each child queue in order
+starting with the highest effective priority. Assuming sufficient scheduling
+resources, the order of scheduling and effective queue priority changes are
+as follows:
+
+| Step | Queue | Task | Result |
+|------|---------------------------------|------|----------------------------------------------------------------------------------------|
+| 1 | root.system.system-high | P1 | **system-high**: `1001` -> n/a <br/> **system**: `1001` -> `10` |
+| 2 | root.system.system-normal | P10 | **system-normal**: `10` -> `2` <br/> **system**: `10` -> `2` |
+| 3 | root.system.system-normal | P2 | **system-normal**: `2` -> n/a <br/> **system**: `2` -> `-997` |
+| 4 | root.tenants.tenant-a.child-a-1 | P8 | **child-a-1**: `8` -> `5` |
+| 5 | root.tenants.tenant-a.child-a-2 | P6 | **child-a-2**: `6` -> `4` |
+| 6 | root.tenants.tenant-a.child-a-1 | P5 | **child-a-1**: `5` -> n/a |
+| 7 | root.tenants.tenant-a.child-a-2 | P4 | **child-a-2**: `4` -> n/a <br/> **tenant-a**: `10` -> n/a |
+| 8 | root.tenants.tenant-b.child-b-1 | P9 | **child-b-1**: `9` -> `7` |
+| 9 | root.tenants.tenant-b.child-b-2 | P8 | **child-b-2**: `8` -> n/a |
+| 10 | root.tenants.tenant-b.child-b-1 | P7 | **child-b-1**: `7` -> n/a <br/> **tenant-b**: `0` -> n/a <br/> **tenants**: `0` -> n/a |
+| 11 | root.system.system-low | P3 | **system-low**: `-997` -> n/a <br/> **system**: `-997` -> n/a |
+
diff --git a/versioned_docs/version-1.2.0/user_guide/queue_config.md b/versioned_docs/version-1.2.0/user_guide/queue_config.md
index 73fd227b7..8d2196fbf 100644
--- a/versioned_docs/version-1.2.0/user_guide/queue_config.md
+++ b/versioned_docs/version-1.2.0/user_guide/queue_config.md
@@ -35,11 +35,10 @@ The current shim identifies the user and the groups the user belongs to using th
## Configuration
The configuration file for the scheduler that is described here only provides the configuration for the partitions and queues.
-By default we use the file called `queues.yaml` in our deployments.
-The filename can be changed via the command line flag `policyGroup` of the scheduler.
-Changing the filename must be followed by corresponding changes in the deployment details, either the `configmap` or the file included in the docker container.
+By default the scheduler reads the ConfigMap section `queues.yaml` for partition and queue configuration. The section name can
+be changed by updating the `service.policyGroup` ConfigMap entry to be something other than `queues`.
-The example file for the configuration is located in the scheduler core's [queues.yaml](https://github.com/apache/yunikorn-core/blob/master/config/queues.yaml).
+The example reference for the configuration is located in the scheduler core's [queues.yaml](https://github.com/apache/yunikorn-core/blob/master/config/queues.yaml) file.
## Partitions
Partitions are the top level of the scheduler configuration.
@@ -63,7 +62,7 @@ The queues configuration is explained below.
Optionally the following keys can be defined for a partition:
* [placementrules](#placement-rules)
-* [statedumpfilepath](#statedump-filepath)
+* [statedumpfilepath](#statedump-filepath) (deprecated since v1.2.0)
* [limits](#limits)
* nodesortpolicy
* preemption
@@ -133,7 +132,8 @@ Supported parameters for the queues:
* name
* parent
* queues
-* properties
+* maxapplications
+* [properties](#properties)
* adminacl
* submitacl
* [resources](#resources)
@@ -159,12 +159,13 @@ Trying to override a _parent_ queue type in the configuration will cause a parsi
Sub queues for a parent queue are defined under the `queues` entry.
The `queues` entry is a recursive entry for a queue level and uses the exact same set of parameters.
+The `maxapplications` property is an integer value, larger than 1, which allows you to limit the number of running applications for the queue. Specifying a zero for `maxapplications` is not allowed as it would block all allocations for applications in the queue. The `maxapplications` value for a _child_ queue must be smaller or equal to the value for the _parent_ queue.
-The `properties` parameter is a simple key value pair list.
-The list provides a simple set of properties for the queue.
-There are no limitations on the key or value values, anything is allowed.
-Currently, the property list is only used in the scheduler to define a [sorting order](sorting_policies.md#application-sorting) for a leaf queue.
-Future expansions, like the option to turn on or off preemption on a queue or other sorting policies, would use this same property construct without the need to change the configuration.
+The [properties](#properties) section contains simple key/value pairs. This is
+used for further queue customization of features such as
+[application sorting](sorting_policies.md#application-sorting) and priority
+scheduling. Future features will use the exisitng `properties` section as well
+to avoid the need to define a new structure for queue configuration.
Access to a queue is set via the `adminacl` for administrative actions and for submitting an application via the `submitacl` entry.
ACLs are documented in the [Access control lists](acls.md) document.
@@ -180,11 +181,20 @@ partitions:
queues:
- name: namespaces
parent: true
+ maxapplications: 12
resources:
guaranteed:
{memory: 1G, vcore: 10}
max:
{memory: 10G, vcore: 100}
+ queues:
+ - name: level1
+ maxapplications: 8
+ resources:
+ guaranteed:
+ {memory: 0.5G, vcore: 5}
+ max:
+ {memory: 5G, vcore: 50}
```
### Placement rules
@@ -196,18 +206,11 @@ If no rules are defined the placement manager is not started and each applicatio
### Statedump filepath
-The statedump filepath defines the output file for YuniKorn statedumps. It is optionally set on the partition level. If set,
-the value of this field can be either a relative (to working directory) or absolute path. YuniKorn scheduler will be unable
-to start if it does not have sufficient permissions to create the statedump file at the specified path.
+**Status** : Deprecated and ignored since v1.2.0, no replacement.
```yaml
statedumpfilepath: <path/to/statedump/file>
```
-If the above key is not specified in the partition config, its value will default to `yunikorn-state.txt`. If the key is specified
-in multiple partitions, the value of its first occurrence will take precedence.
-
-The statedump file also has a fixed rotation policy. Currently, each statedump file has a capacity of 10MB and there can be a maximum
-of 10 such files. The statedump file currently being written to will always be the configured value above or default `yunikorn-state.txt`. When the file size limit is reached, the log rotator (`lumberjack`) will modify the file by prefixing it with a timestamp and create a new file with the same non-prefixed name to write statedumps to. If the maximum number of statedump files are reached, the oldest file timestamped as per the rotation policy will be deleted.
### Limits
Limits define a set of limit objects for a partition or queue.
@@ -264,7 +267,7 @@ Specifying a star beside other list elements is not allowed.
_maxapplications_ is an unsigned integer value, larger than 1, which allows you to limit the number of running applications for the configured user or group.
Specifying a zero maximum applications limit is not allowed as it would implicitly deny access.
-Denying access must be handled via the ACL entries.
+Denying access must be handled via the ACL entries.
The _maxresources_ parameter can be used to specify a limit for one or more resources.
The _maxresources_ uses the same syntax as the [resources](#resources) parameter for the queue.
@@ -290,6 +293,80 @@ users:
```
In this case both the users `sue` and `bob` are allowed to run 10 applications.
+### Properties
+
+Additional queue configuration can be added via the `properties` section,
+specified as simple key/value pairs. The following parameters are currently
+supported:
+
+#### `application.sort.policy`
+
+Supported values: `fifo`, `fair`, `stateaware`
+
+Default value: `fifo`
+
+Sets the policy to be used when sorting applications within a queue. This
+setting has no effect on a _parent_ queue.
+
+See the documentation on [application sorting](sorting_policies.md#application-sorting)
+for more information.
+
+
+#### `application.sort.priority`
+
+Supported values: `enabled`, `disabled`
+
+Default value: `enabled`
+
+When this property is `enabled`, priority will be considered when sorting
+queues and applications. Setting this value to `disabled` will ignore
+priorities when sorting. This setting can be specified on a _parent_ queue and
+will be inherited by _child_ queues.
+
+**NOTE:** YuniKorn releases prior to 1.2.0 did not support priorities when
+sorting. To keep the legacy behavior, set `application.sort.priority` to
+`disabled`.
+
+#### `priority.policy`
+
+Supported values: `default`, `fence`
+
+Default value: `default`
+
+Sets the inter-queue priority policy to use when scheduling requests.
+
+**NOTE**: This value is not inherited by child queues.
+
+By default, priority applies across queues globally. In other words,
+higher-priority requests will be satisfied prior to lower-priority requests
+regardless of which queue they exist within.
+
+When the `fence` policy is in use on a queue, the priorities of _child_ queues
+(in the case of a _parent_ queue) or applications (in the case of a _leaf_
+queue) will not be exposed outside the fence boundary.
+
+See the documentation on [priority support](priorities.md) for more information.
+
+#### `priority.offset`
+
+Supported values: any positive or negative 32-bit integer
+
+Default value: `0`
+
+Adjusts the priority of the queue relative to it's siblings. This can be useful
+to create high or low-priority queues without needing to set every task's
+priority manually.
+
+**NOTE**: This value is not inherited by child queues.
+
+When using the `default` priority policy, the queue's priority is adjusted up
+or down by this amount.
+
+When using the `fence` policy, the queue's priority is always set to the offset
+value (in other words, the priorities of tasks in the queue are ignored).
+
+See the documentation on [priority support](priorities.md) for more information.
+
### Resources
The resources entry for the queue can set the _guaranteed_ and or _maximum_ resources for a queue.
Resource limits are checked recursively.
diff --git a/versioned_docs/version-1.2.0/user_guide/resource_quota_mgmt.md b/versioned_docs/version-1.2.0/user_guide/resource_quota_mgmt.md
index 0741975f0..adca022db 100644
--- a/versioned_docs/version-1.2.0/user_guide/resource_quota_mgmt.md
+++ b/versioned_docs/version-1.2.0/user_guide/resource_quota_mgmt.md
@@ -228,14 +228,22 @@ In the case that YuniKorn is used for quota enforcement no quota must be set on
To allow specifying a quota on the namespace the following annotations should be set in the namespace object:
```yaml
-yunikorn.apache.org/namespace.max.cpu: "64"
-yunikorn.apache.org/namespace.max.memory: "100Gi"
+yunikorn.apache.org/namespace.quota: "{\"cpu\": \"64\", \"memory\": \"100G\", \"nvidia.com/gpu\": \"1\"}"
```
YuniKorn will parse these annotations and set the maximum capacity of the queue mapped to this namespace.
The values specified follow the standard Kubernetes formatting and unit specification.
-Currently, we only support mapping memory and cpu not other resource types.
+Annotation value must be a single json compliant string. Ensure double quotes iare escaped properly to not cause any problems.
-The example above will limit the queue mapped to the annotated namespace to 64 CPUs and 100GB memory.
+The example above will limit the queue mapped to the annotated namespace to 64 CPUs, 100GB memory and 1 `nvidia.com/gpu`.
+
+[DEPRECATED]
+The below annotations are deprecated and will be removed from next major release.
+They only support mapping memory and cpu, not other resource types.
+```yaml
+yunikorn.apache.org/namespace.max.cpu: "64"
+yunikorn.apache.org/namespace.max.memory: "100Gi"
+```
+The example for the deprecated annotation will set the queue quota to 64 CPUs and 100GB memory.
### Run a workload
@@ -264,19 +272,21 @@ First we set the following configuration to YuniKorn's configmap:
```yaml
partitions:
- - name: default
- placementrules:
- - name: tag
- value: namespace
- create: true
- parent:
- - name: tag
- value: namespace.parentqueue
- queues:
- - name: root
- queues:
- - name: production
- - name: development
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ name: tag
+ value: namespace.parentqueue
+ queues:
+ - name: root
+ queues:
+ - name: production
+ parent: true
+ - name: development
+ parent: true
```
The configuration used for the namespace to queue mapping is the same as [above](#Namespace-to-queue-mapping).
diff --git a/versioned_docs/version-1.2.0/user_guide/service_config.md b/versioned_docs/version-1.2.0/user_guide/service_config.md
new file mode 100644
index 000000000..7456f7dd2
--- /dev/null
+++ b/versioned_docs/version-1.2.0/user_guide/service_config.md
@@ -0,0 +1,848 @@
+---
+id: service_config
+title: Service Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The official distribution of YuniKorn is deployed via Helm charts to
+Kubernetes. Configuration for YuniKorn is split into two parts: Helm
+configuration, and YuniKorn service configuration.
+
+## Helm Configuration
+Helm configuration is used to configure options for the deployment of
+YuniKorn to Kubernetes.
+
+The following settings can be configured during YuniKorn installation
+via Helm, either via Helm's command-line, as in `--set key=value`, or
+via an external file: `-f file.yaml`. The examples below will be given in
+YAML syntax.
+
+### Container images
+YuniKorn ships as a set of container images. The locations and pull
+policies can be customized as follows:
+
+ # Image information for the standard scheduler
+ image:
+ repository: apache/yunikorn
+ tag: scheduler-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+
+ # Image information for the plugin scheduler
+ pluginImage:
+ repository: apache/yunikorn
+ tag: scheduler-plugin-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+
+ # Image information for the web UI
+ web:
+ image:
+ repository: apache/yunikorn
+ tag: web-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+
+ # Image information for the admission controller
+ admissionController:
+ image:
+ repository: apache/yunikorn
+ tag: admission-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+
+### Kubernetes configuration
+
+#### affinity
+Sets the affinity for the YuniKorn scheduler pod.
+
+Default: `{}`
+
+Example:
+
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - primary1
+ - primary2
+
+#### admissionController.affinity
+Sets the affinity for the YuniKorn admission controller pod.
+
+Default: `{}`
+
+Example:
+
+ admissionController:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - primary1
+ - primary2
+
+#### hostNetwork
+Controls whether the scheduler should run in the host network.
+
+Default: `false`
+
+Example:
+
+ hostNetwork: true
+
+#### admissionController.hostNetwork
+Controls whether the admission controller should run in the host network.
+
+Default: `true`
+
+Example:
+
+ admissionController:
+ hostNetwork: false
+
+#### imagePullSecrets
+Provides secrets needed for pulling YuniKorn images.
+
+Default: `[]`
+
+Example:
+
+ imagePullSecrets:
+ - secret1
+ - secret2
+
+#### nodeSelector
+Sets a node selector(s) to use for placement of the YuniKorn scheduler pod.
+
+Default: `{}`
+
+Example:
+
+ nodeSelector:
+ role.node.kubernetes.io/infra: "true"
+
+#### admissionController.nodeSelector
+Sets a node selector(s) to use for placement of the YuniKorn admission controller pod.
+
+Default: `{}`
+
+Example:
+
+ admissionController:
+ nodeSelector:
+ role.node.kubernetes.io/infra: "true"
+
+#### admissionController.replicaCount
+Sets the number of replicas to use for the YuniKorn admission controller. This
+can be set to greater than 1 for high-availability.
+
+Default: `1`
+
+Example:
+
+ admissionController:
+ replicaCount: 2
+
+#### serviceAccount
+Sets an alternate service account for the YuniKorn scheduler.
+
+Changing this value is not recommended, as Helm installs role-based access
+control (RBAC) policies for the default user that are required for proper
+functionaliy.
+
+Default: `yunikorn-admin`
+
+Example:
+
+ serviceAccount: my-account
+
+#### admissionController.serviceAccount
+Sets an alternate service account for the YuniKorn admission controller.
+
+Changing this value is not recommended, as Helm installs role-based access
+control (RBAC) policies for the default user that are required for proper
+functionaliy.
+
+Default: `yunikorn-admission-controller`
+
+Example:
+
+ admissionController:
+ serviceAccount: my-account
+
+#### service.type
+Sets the type of service used for the scheduler.
+
+Default: `ClusterIP`
+
+Example:
+
+ service:
+ type: ClusterIP
+
+#### admissionController.service.type
+Sets the type of service used for the admission controller.
+
+Default: `ClusterIP`
+
+Example:
+
+ admissionController:
+ service:
+ type: ClusterIP
+
+#### service.port
+Sets the port exposed in the YuniKorn scheduler service for the REST API.
+It is not recommended to change this value.
+
+Default: 9080
+
+Example:
+
+ service:
+ port: 9080
+
+#### service.portWeb
+Sets the port exposed in the YuniKorn scheduler service for the Web UI.
+It is not recommended to change this value.
+
+Default: 9889
+
+Example:
+
+ service:
+ portWeb: 9889
+
+#### tolerations
+Sets the tolerations for the YuniKorn scheduler pod.
+
+Default: `[]`
+
+Example:
+
+ tolerations:
+ - key: *infraRoleKey
+ operator: Equal
+ value: "true"
+ effect: NoSchedule
+ - key: CriticalAddonsOnly
+ operator: Exists
+
+#### admissionController.tolerations
+Sets the tolerations for the YuniKorn admission controller pod.
+
+Default: `[]`
+
+Example:
+
+ admissionController:
+ tolerations:
+ - key: *infraRoleKey
+ operator: Equal
+ value: "true"
+ effect: NoSchedule
+ - key: CriticalAddonsOnly
+ operator: Exists
+
+### Resource utilization
+The resources requested for YuniKorn pods can be customized as follows:
+
+ # Scheduler container resources
+ resources:
+ requests:
+ cpu: 200m
+ memory: 1Gi
+ limits:
+ cpu: 4
+ memory: 2Gi
+
+ # Web UI container resources
+ web:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 100Mi
+ limits:
+ cpu: 100m
+ memory: 500Mi
+
+ # Admission controller resources
+ admissionController:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 500Mi
+ limits:
+ cpu: 500m
+ memory: 500mi
+
+### Optional features
+
+#### embedAdmissionController
+Controls whether to enable the YuniKorn admission controller.
+
+Default: `true`
+
+Example:
+
+ embedAdmissionController: false
+
+#### enableSchedulerPlugin
+Controls whether to run YuniKorn in scheduler plugin mode.
+
+Default: `false`
+
+Example:
+
+ enableSchedulerPlugin: true
+
+### YuniKorn defaults
+
+#### yunikornDefaults
+Sets entries which will be rendered to the `yunikorn-defaults` ConfigMap. This
+can be used to pre-configure YuniKorn at deployment time. Any settings
+declared in [YuniKorn configuration](#yunikorn-configuration) may be set here.
+
+Default: `{}`
+
+Example:
+
+ yunikornDefaults:
+ service.clusterId: yunikorn-01
+ service.policyGroup: group-01
+ group-01.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'`
+
+
+### Deprecated settings
+The following settings are deprecated, and will be removed from a future
+YuniKorn release. They should now be specified in the `yunikorn-configs` ConfigMap
+or via the Helm `yunikornDefaults` section:
+
+| Deprecated setting | ConfigMap replacement |
+| --------------------------------------- | ----------------------------------------------- |
+| operatorPlugins | service.operatorPlugins |
+| placeHolderImage | service.placeholderImage |
+| admissionController: processNamespaces | admissionController.filtering.processNamespaces |
+| admissionController: bypassNamespaces | admissionController.filtering.bypassNamespaces |
+| admissionController: labelNamespaces | admissionController.filtering.labelNamespaces |
+| admissionController: noLabelNamespaces | admissionController.filtering.noLabelNamespaces |
+| configuration | queues.yaml |
+
+Deprecated example:
+
+ operatorPlugins: general
+ placeHolderImage: registry.k8s.io/pause:3.7
+ admissionController:
+ processNamespaces: "^spark-,^mpi-"
+ bypassNamespaces: "^kube-system$"
+ labelNamespaces: "^spark-"
+ noLabelNamespaces: "^mpi-legacy-"
+ configuration: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'`
+
+Replacement example:
+
+ yunikornDefaults:
+ service.policyGroup: queues
+ service.operatorPlugins: general
+ service.placeholderImage: registry.k8s.io/pause:3.7
+ admissionController.processNamespaces: "^spark-,^mpi-"
+ admissionController.bypassNamespaces: "^kube-system$"
+ admissionController.labelNamespaces: "^spark-"
+ admissionController.noLabelNamespaces: "^mpi-legacy-"
+ queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'`
+
+Currently, if both the deprecated parameter and the replacement ConfigMap entry are specified, the ConfigMap entry will take precedence.
+
+
+## YuniKorn Configuration
+
+Service configuration for YuniKorn is controlled by two Kubernetes ConfigMaps
+in the namespace where YuniKorn is installed: `yunikorn-defaults` and
+`yunikorn-configs`.
+
+At runtime, these ConfigMaps are polled by YuniKorn and merged together to form an
+effective configuration. If a setting is present in both ConfigMaps, the
+`yunikorn-configs` setting will override the one present in `yunikorn-defaults`.
+
+The purpose of `yunikorn-defaults` is to provide a mechanism for Helm to configure
+initial service configuration details. It should not be modified directly.
+
+The `yunikorn-configs` ConfigMap is completely unmanaged by Helm, and is meant for
+configurations which may change over time, such as queue configuration. All changes
+to YuniKorn configuration outside of provisioning infrastructure should be made here.
+
+### Default ConfigMap
+
+If neither ConfigMap is provided, or if an option is not specified, YuniKorn will
+use the default values listed here:
+
+ apiVersion: v1
+ kind: ConfigMap
+ metadata:
+ name: yunikorn-configs
+ data:
+ service.clusterId: "mycluster"
+ service.policyGroup: "queues"
+ service.schedulingInterval: "1s"
+ service.volumeBindTimeout: "10s"
+ service.eventChannelCapacity: "1048576"
+ service.dispatchTimeout: "5m"
+ service.operatorPlugins: "general"
+ service.disableGangScheduling: "false"
+ service.enableConfigHotRefresh: "true"
+ service.placeholderImage: "registry.k8s.io/pause:3.7"
+ health.checkInterval: "30s"
+ log.level: "0"
+ kubernetes.qps: "1000"
+ kubernetes.burst: "1000"
+ admissionController.webHook.amServiceName: "yunikorn-admission-controller-service"
+ admissionController.webHook.schedulerServiceAddress: "yunikorn-service:9080"
+ admissionController.filtering.processNamespaces: ""
+ admissionController.filtering.bypassNamespaces: "^kube-system$"
+ admissionController.filtering.labelNamespaces: ""
+ admissionController.filtering.noLabelNamespaces: ""
+ admissionController.accessControl.bypassAuth: "false"
+ admissionController.accessControl.trustControllers: "true"
+ admissionController.accessControl.systemUsers: "^system:serviceaccount:kube-system:"
+ admissionController.accessControl.externalUsers: ""
+ admissionController.accessControl.externalGroups: ""
+ queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'`
+
+### Service settings
+The following parameters are understood by YuniKorn:
+
+#### service.clusterId
+Sets an identifier for the cluster being configured. This is returned as part
+of REST API calls.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `mycluster`
+
+Example:
+
+ service.clusterId: "yunikorn-east"
+
+#### service.policyGroup
+Defines the policy group in use by this scheduler. The policy group is used to
+choose one of several queue configurations. The value of this setting plus an
+extension of `.yaml` controls the ConfigMap entry used to retrieve partition
+and queue configuration.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `queues`
+
+Example:
+
+ service.policyGroup: group_b
+ group_b.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'`
+
+#### service.schedulingInterval
+Controls the frequency with which YuniKorn executes scheduling runs.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1s`
+
+Example:
+
+ service.schedulingInterval: "5s"
+
+#### service.volumeBindTimeout
+Controls the timeout before volume binding fails.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `10s`
+
+Example:
+
+ service.volumeBindTimeout: "30s"
+
+#### service.eventChannelCapacity
+Controls the number of internal scheduling events that YuniKorn will allow
+to be in-flight at one time. This acts as an out-of-memory guard.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1048576`
+
+Example:
+
+ service.eventChannelCapacity: "1000000"
+
+#### service.dispatchTimeout
+Controls how long internal events will reattempt dispatching if the event
+channel is full. Warnings will be emitted if this timeout is exceeded.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `5m`
+
+Example:
+
+ service.dispatchTimeout: "10m"
+
+#### service.operatorPlugins
+Controls the set of operator plugins which are enabled within YuniKorn.
+Currently, only the `general`, `spark-k8s-operator`, and `yunikorn-app`
+plugins are implemented. The `general` plugin should not be disabled.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `general`
+
+Example:
+
+ service.operatorPlugins: "general,spark-k8s-operator"
+
+#### service.disableGangScheduling
+Allows global disabling of the gang scheduling feature (not recommended).
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `false`
+
+Example:
+
+ service.disableGangScheduling: "true"
+
+#### service.enableConfigHotRefresh
+Controls whether configuration should be hot-reloaded. By default, this
+is set to `true`, but it can be disabled to avoid changes to the
+ConfigMaps from being picked up until a scheduler restart.
+
+A change to this setting will be picked up without a restart of YuniKorn.
+
+NOTE: If this setting is disabled, it may not be re-enabled again without
+a restart of YuniKorn.
+
+Default: `true`
+
+Example:
+
+ service.enableConfigHotRefresh: "false"
+
+#### service.placeholderImage
+Sets the Pod image that will be used for gang scheduling placeholders.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `registry.k8s.io/pause:3.7`
+
+Example:
+
+ service.placeholderImage: "registry.k8s.io/pause:3.6"
+
+### Health settings
+
+#### health.checkInterval
+Sets the time between automatic health checks of YuniKorn.
+
+Setting the value to `0` or a negative interval will disable background health
+checking.
+
+A change to this setting will be picked up without a restart of YuniKorn.
+
+Default: `30s`
+
+Example:
+
+ health.checkInterval: "1m"
+
+### Log settings
+
+#### log.level
+Sets the verbosity that YuniKorn will log at.
+
+A change to this setting will be picked up without a restart of YuniKorn. The available
+values are:
+
+ - `-1`: Debug
+ - `0`: Info
+ - `1`: Warn
+ - `2`: Error
+ - `3`: DPanic
+ - `4`: Panic
+ - `5`: Fatal
+
+Default: `0` (Info)
+
+Example:
+
+ log.level: "-1"
+
+### Kubernetes settings
+
+#### kubernetes.qps
+Sets the number of Kubernetes queries per second (QPS) used by YuniKorn's
+Kubernetes client. This number must be >= 0.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1000`
+
+Example:
+
+ kubernetes.qps: "500"
+
+#### kubernetes.burst
+Sets the maximum size of bursty queries to Kubernetes, temporarily allowing
+events to burst to this number while not still exceeding `kubernetes.qps`.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1000`
+
+Example:
+
+ kubernetes.burst: "500"
+
+### Admission controller webhook settings
+
+#### admissionController.webHook.amServiceName
+Sets the name of the service that the YuniKorn admission controller is
+registered under. This is required for the admission controller to register
+itself properly with Kubernetes, and should normally not be changed.
+
+A change to this setting requires a restart of the YuniKorn admission controller
+to take effect.
+
+Default: `yunikorn-admission-controller-service`
+
+Example:
+
+ admissionController.webHook.amServiceName: "yunikorn-admission-controller-alt-service-name"
+
+#### admissionController.webHook.schedulerServiceAddress
+Sets the address of the YuniKorn scheduler service. This address must be
+reachable by the admission controller, and is used by the admission
+controller when validating ConfigMap changes. The admission controller
+will contact the REST API on the scheduler to validate any proposed
+ConfigMap changes. This setting should not normally be changed.
+
+A change to this setting requires a restart of the YuniKorn admission controller
+to take effect.
+
+Default: `yunikorn-service:9080`
+
+Example:
+
+ admissionController.webHook.schedulerServiceAddress: "alt-yunikorn-service:9080"
+
+### Admission controller filtering settings
+
+#### admissionController.filtering.processNamespaces
+Controls which namespaces will have pods forwarded to YuniKorn for scheduling.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, pods created in all namespaces will be scheduled by YuniKorn.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+
+ # Schedule only pods in spark-* and mpi-* namespaces with YuniKorn
+ admissionController.filtering.processNamespaces: "^spark-,^mpi-"
+
+#### admissionController.filtering.bypassNamespaces
+Controls which namespaces will *not* have pods forwarded to YuniKorn for scheduling.
+This acts as an exception list to `admissionController.filtering.processNamespaces`.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, no pods in any namespaces will be excluded from processing by
+YuniKorn.
+
+By default, this setting excludes pods in the `kube-system` namespace as scheduling
+of these pods is often required for a node to be added to a cluster successfully.
+This could possibly prevent starting of YuniKorn itself or other critical services.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `^kube-system$`
+
+Example:
+
+ # Don't schedule pods in kube-system or fluentd-* namespaces
+ admissionController.filtering.bypassNamespaces: "^kube-system$,^fluentd-"
+
+#### admissionController.filtering.labelNamespaces
+Controls which namespaces will have pods labeled with an `applicationId`. By default,
+all pods which are scheduled by YuniKorn will have an `applicationId` label applied.
+
+When running YuniKorn using the standard deployment model, all pods should be labeled,
+as YuniKorn is unable to schedule pods without an `applicationId` defined.
+
+When running YuniKorn using the scheduler plugin deployment model, this setting can
+be used to filter which namespaces should be scheduled via YuniKorn's queueing model,
+and which should bypass queueing and be scheduled by the embedded default scheduler.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, all pods forwarded to YuniKorn will have an `applicationId` label
+applied.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+
+ # Add applicationId labels to pods spark-* namespaces
+ admissionController.filtering.labelNamespaces: "^spark-"
+
+#### admissionController.filtering.noLabelNamespaces
+Controls which namespaces will *not* have pods labeled with an `applicationId`. This
+acts as an exception list to `admissionController.filtering.labelNamespaces`.
+
+When running YuniKorn using the standard deployment model, all pods should be labeled,
+as YuniKorn is unable to schedule pods without an `applicationId` defined.
+
+When running YuniKorn using the scheduler plugin deployment model, this setting can
+be used to filter which namespaces should be scheduled via YuniKorn's queueing model,
+and which should bypass queueing and be scheduled by the embedded default scheduler.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, no exclusions to `admissionController.filtering.labelNamespaces` will
+be applied.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+
+ # Skip queueing in the noqueue namespace
+ admissionController.filtering.labelNamespaces: "^noqueue$"
+
+### Admission controller ACL settings
+
+#### admissionController.accessControl.bypassAuth
+Allow external users to create pods with user information already set.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `false`
+
+Example:
+
+ admissionController.accessControl.bypassAuth: "true"
+
+#### admissionController.accessControl.trustControllers
+Allow controller users to create pods with user information already set.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `true`
+
+Example:
+
+ admissionController.accessControl.trustControllers: "false"
+
+#### admissionController.accessControl.systemUsers
+Comma-separated list of regular expressions that match allowed controller
+service accounts.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `^system:serviceaccount:kube-system:`
+
+Example:
+
+ # allow all kube-system accounts as well as kube-controller-manager
+ admissionController.accessControl.systemUsers: "^system:serviceaccount:kube-system,^system:kube-controller-manager$"
+
+#### admissionController.accessControl.externalUsers
+Comma-separated list of regular expressions that match allowed external users.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+
+ # allow 'alice', 'bob', and 'admin-*'
+ admissionController.accessControl.externalUsers: "^alice$,^bob$,^admin-"
+
+#### admissionController.accessControl.externalGroups
+Comma-separated list of regular expressions that match allowed external groups.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+
+ # allow 'sales', 'marketing', and 'admin-*'
+ admissionController.accessControl.externalGroups: "^sales$,^marketing$,^admin-"
+
diff --git a/versioned_docs/version-1.2.0/user_guide/sorting_policies.md b/versioned_docs/version-1.2.0/user_guide/sorting_policies.md
index c5f4bfcff..f806f9ef1 100644
--- a/versioned_docs/version-1.2.0/user_guide/sorting_policies.md
+++ b/versioned_docs/version-1.2.0/user_guide/sorting_policies.md
@@ -29,8 +29,8 @@ Policies can be set for:
## Application sorting
The application sorting policy is set for each queue via the config.
-A sorting policy setting is only effective on a `leaf` queue.
-Each `leaf` queue can use a different policy.
+A sorting policy setting is only effective on a _leaf_ queue.
+Each _leaf_ queue can use a different policy.
A sorting policy only specifies the order in which the applications are sorted within a queue.
That order is crucial in specifying which application is considered first when assigning resources.
@@ -38,7 +38,12 @@ Sorting policies do _not_ affect the number of applications that are scheduled o
All applications that have pending resource requests can and will be scheduled in a queue unless specifically filtered out.
Even when applications are sorted using a first in first out policy multiple applications will run in a queue in parallel.
-A `parent` queue will always use the fair policy to sort the child queues.
+A _parent_ queue will always use the fair policy to sort the child queues.
+
+The relative priority of child queues (in the case of _parent_ queue sorting)
+and applications (in the case of _leaf_ queue sorting) will be considered first.
+To ignore application and queue priorities when scheduling, set the queue
+property `application.sort.priority` to `disabled`.
The following configuration entry sets the application sorting policy to `fifo` for the queue `root.sandbox`:
```yaml
@@ -57,9 +62,10 @@ A filter is applied _while_ sorting the applications to remove all that do not h
### FifoSortPolicy
Short description: first in first out, based on application create time
-Config value: fifo (default)
-Behaviour:
-Before sorting the applications are filtered and must have pending resource requests.
+
+Config value: `fifo` (default)
+
+Before sorting, the applications are filtered and must have pending resource requests.
After filtering the applications left are sorted based on the application create time stamp only, no other filtering is applied.
Since applications can only be added while the system is locked there can never be two applications with the exact same time stamp.
@@ -69,8 +75,9 @@ Younger applications will be given resources when all the current requests of ol
### FairSortPolicy
Short description: fair based on usage
-Config value: fair
-Behaviour:
+
+Config value: `fair`
+
Before sorting the applications are filtered and must have pending resource requests.
After filtering the applications left are sorted based on the application usage.
@@ -81,8 +88,9 @@ The result is that the resources available are spread equally over all applicati
### StateAwarePolicy
Short description: limit of one (1) application in Starting or Accepted state
-Config value: stateaware
-Behaviour:
+
+Config value: `stateaware`
+
This sorting policy requires an understanding of the application states.
Applications states are described in the [application states](design/scheduler_object_states.md#application-state) documentation.
@@ -182,4 +190,4 @@ There is currently one policy for sorting requests within an application.
This policy is not configurable.
Sorting requests is only possible based on the priority of the request.
If there are multiple requests within an application that have the same priority the order of the requests is undetermined.
-This means that the order of requests with the same priority can, and most likely will, change between runs.
\ No newline at end of file
+This means that the order of requests with the same priority can, and most likely will, change between runs.
diff --git a/versioned_docs/version-1.2.0/user_guide/trouble_shooting.md b/versioned_docs/version-1.2.0/user_guide/troubleshooting.md
similarity index 80%
rename from versioned_docs/version-1.2.0/user_guide/trouble_shooting.md
rename to versioned_docs/version-1.2.0/user_guide/troubleshooting.md
index 549d5e0e3..9da841852 100644
--- a/versioned_docs/version-1.2.0/user_guide/trouble_shooting.md
+++ b/versioned_docs/version-1.2.0/user_guide/troubleshooting.md
@@ -1,6 +1,6 @@
---
-id: trouble_shooting
-title: Trouble Shooting
+id: troubleshooting
+title: Troubleshooting
---
<!--
@@ -46,7 +46,7 @@ The recommended setup is to leverage [fluentd](https://www.fluentd.org/) to coll
### Set Logging Level
:::note
-Changing the logging level requires a restart of the scheduler pod.
+We recommend altering the log level via REST API call as this way we don't need to restart the scheduler pod every time. But changing the logging level via editing the deployment config requires a restart of the scheduler pod and it's not highly recommended.
:::
Stop the scheduler:
@@ -134,8 +134,37 @@ is running out of capacity.
The pod will be allocated if some other pods in this queue is completed or removed. If the pod remains pending even
the queue has capacity, that may because it is waiting for the cluster to scale up.
+## Obtain full state dump
+
+A Yunikorn state dump contains the every state object for every process which getting dumped. With endpoint to retrieve we can have many useful information in a single response for troubleshooting for example: list of partitions, list of applications which includes running, completed also historical application details, number of nodes, utilization of nodes, generic cluster information, cluster utilization details, container history and queues information.
+
+The state dump is a valuable resource that Yunikorn offers for use while troubleshooting.
+
+There are a few ways to obtain the full state dump.
+
+### 1. Scheduler URL
+
+STEPS:
+* Open the Scheduler URL in your browser window/tab and edit the URL as follows:
+* Replace `/#/dashboard` with `/ws/v1/fullstatedump`, (For example, `http://localhost:9889/ws/v1/fullstatedump`)
+* Press Enter
+
+That displays and provides an easy user experience to view live full state dump.
+
+### 2. Scheduler REST API
+
+With the below scheduler REST API returns information about full state dump used by the YuniKorn Scheduler.
+
+`curl -X 'GET' http://localhost:9889/ws/v1/fullstatedump -H 'accept: application/json'`
+
+For more details around the content of the state dump, please refer to the documentation on [retrieve-full-state-dump](api/scheduler.md#retrieve-full-state-dump)
+
## Restart the scheduler
+:::note
+In accordance with best practices for troubleshooting, restarting the scheduler should only be done as a last effort to get everything back up and running. It should never be done before gathering all logs and state dumps.
+:::
+
YuniKorn can recover its state upon a restart. YuniKorn scheduler pod is deployed as a deployment, restart the scheduler
can be done by scale down and up the replica:
@@ -189,4 +218,4 @@ No problem! The Apache YuniKorn community will be happy to help. You can reach o
1. Post your questions to dev@yunikorn.apache.org
2. Join the [YuniKorn slack channel](https://join.slack.com/t/yunikornworkspace/shared_invite/enQtNzAzMjY0OTI4MjYzLTBmMDdkYTAwNDMwNTE3NWVjZWE1OTczMWE4NDI2Yzg3MmEyZjUyYTZlMDE5M2U4ZjZhNmYyNGFmYjY4ZGYyMGE) and post your questions to the `#yunikorn-user` channel.
-3. Join the [community sync up meetings](http://yunikorn.apache.org/community/getInvolved#community-meetings) and directly talk to the community members.
\ No newline at end of file
+3. Join the [community sync up meetings](http://yunikorn.apache.org/community/get_involved#community-meetings) and directly talk to the community members.
diff --git a/versioned_docs/version-1.2.0/user_guide/usergroup_resolution.md b/versioned_docs/version-1.2.0/user_guide/usergroup_resolution.md
index 564f5cecd..2eae43511 100644
--- a/versioned_docs/version-1.2.0/user_guide/usergroup_resolution.md
+++ b/versioned_docs/version-1.2.0/user_guide/usergroup_resolution.md
@@ -26,11 +26,19 @@ under the License.
User information is an important aspect of the scheduling cycle. It is one of the key identifier that can be used to determine the queue to which a job should be submitted. The Yunikorn Scheduler relies on the K8s Shim to provide user information. In the world of Kubernetes, there is no object defined that identfies the actual user. This is by design and more information can be found [here](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#users-in-kubernetes).
+In Yunikorn, there are two ways of handling users and groups. The first is the legacy way, which uses the label `yunikorn.apache.org/username`. If this label is set on a pod, then the value is automatically extracted in the shim and will be used accordingly. Group resolution is also done in the shim and is disabled by default. The problem with this approach is twofold: user restrictions can be easily bypassed because the submitter is free to set this label to any value, therefore this on [...]
+
+A more reliable and robust mechanism is using the `yunikorn.apache.org/user.info` annotation, where the user information can be set externally by an allowed list of users or groups or the admission controller can attach this automatically to every workload.
+
+## Legacy user handling
+
+### Using the `yunikorn.apache.org/username` label
+
Since, Kubernetes has no pre-defined field or resource for user information and individual cluster deployments with unique user identification tools can vary, we have defined a standard way of identifying the user. Yunikorn requires a Kubernetes [Label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) added. Using the [recommendation](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/) provided here, the default label is defined as below:
-| Label | Value |
-|----------------------------------------------- |--------------------- |
-| yunikorn.apache.org/username | User name. It can have duplicate entries but only the first value will be used. The default user is `nobody` |
+| Label | Value |
+|--------------------------------|--------------------------------------------------------------------------------------------------------------|
+| yunikorn.apache.org/username | User name. It can have duplicate entries but only the first value will be used. The default user is `nobody` |
Example:
```yaml
@@ -55,7 +63,7 @@ The `yunikorn.apache.org/username` key can be customized by overriding the defau
value: "custom_user_label"
```
-## Group resolution
+### Group resolution
Group membership resolution is pluggables and is defined here. Groups do not have to be part of provided user and group object. When the object is added to the cache the groups are automatically resolved based on the resolution that is configured.
The resolver which is linked to the cache can be set per partition.
@@ -66,3 +74,47 @@ This resolver just echos the user name and a primary group with the same name as
Other resolvers are:
* OS resolver
* test resolver
+
+## The new, recommended way of handling users
+
+Since Yunikorn 1.2 a more sophisticated way of user/group resolution is available.
+
+In this mode, Yunikorn no longer relies on the `yunikorn.apache.org/username` label, instead, the annotation `yunikorn.apache.org/user.info` is attached to the workload. The value is simple JSON, which defines the user name and groups:
+
+```yaml
+metadata:
+ annotations:
+ yunikorn.apache.org/user.info: "
+ {
+ username: \"yunikorn\",
+ groups: [
+ \"developers\",
+ \"devops\"
+ ]
+ }"
+```
+
+However, to enhance security, the following is enforced in the admission controller:
+* not every user in the cluster is allowed to attach this annotation, only those which are configured
+* if the annotation is missing, the admission controller will add this information automatically
+* attempts to change this annotation will be rejected
+
+We also no longer do this on pods only, but also on Deployments, ReplicaSets, DeamonSets, StatefulSets, Jobs and CronJobs.
+
+Group resolution is no longer necessary inside the shim.
+
+### Configuring the admission controller
+
+The admission controller can be configured with the `yunikorn-configs` configmap. All entries start with the prefix `admissionController.accessControl.`.
+
+| Variable | Default value | Description |
+|--------------------|---------------------------------------|----------------------------------------------------------------------------|
+| `bypassAuth` | false | Allow any external user to create pods with user information set |
+| `trustControllers` | true | Allow Kubernetes controller users to create pods with user information set |
+| `systemUsers` | "^system:serviceaccount:kube-system:" | Regular expression for the allowed controller service account list |
+| `externalUsers` | "" | Regular expression for the allowed external user list |
+| `externalGroups` | "" | Regular expression for the allowed external group list |
+
+If `bypassAuth` is set to true the admission controller will not add the annotation to a pod if the annotation is not present and the deprecated user labell is set. If the annotation is not set and the user label is not set the new annotation will be added. In the case that `bypassAuth` is false, the default, the admission controller will always add the new annotation, regardless of the existence of the deprecated label.
+
+In certain scenarios, users and groups must be provided to Yunikorn upon submission because the user and group management is provided by external systems and the lookup mechanism is not trivial. In these cases, the `externalUsers` and `externalGroups` can be configured which are treated as regular expressions. Matching users and groups are allowed to set the `yunikorn.apache.org/user.info` annotation to any arbitrary value. Since this has implications which affects scheduling inside Yuni [...]
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/run_flink.md b/versioned_docs/version-1.2.0/user_guide/workloads/run_flink.md
index d20e8158a..e031f9af8 100644
--- a/versioned_docs/version-1.2.0/user_guide/workloads/run_flink.md
+++ b/versioned_docs/version-1.2.0/user_guide/workloads/run_flink.md
@@ -32,7 +32,7 @@ used to run Flink on Kubernetes, the configuration is slight different.
## Standalone mode
Please follow [Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html) to get details and examples of standalone deploy mode.
-In this mode, we can directly add required labels (applicationId and queue) in Deployment/Job spec to run flink application with YuniKorn scheduler, as well as [Run workloads with YuniKorn Scheduler](#run-workloads-with-yunikorn-scheduler).
+In this mode, we can directly add required labels (applicationId and queue) in Deployment/Job spec to run flink application with YuniKorn scheduler, see the [overview](./workload_overview.md) for the label specification.
## Native mode
@@ -63,4 +63,4 @@ Examples:
-Dkubernetes.jobmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
-Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
local:///opt/flink/usrlib/my-flink-job.jar
-```
\ No newline at end of file
+```
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/run_mpi.md b/versioned_docs/version-1.2.0/user_guide/workloads/run_mpi.md
new file mode 100644
index 000000000..f69f8c705
--- /dev/null
+++ b/versioned_docs/version-1.2.0/user_guide/workloads/run_mpi.md
@@ -0,0 +1,112 @@
+---
+id: run_mpi
+title: Run MPI Jobs
+description: How to run MPI jobs with YuniKorn
+keywords:
+ - mpi
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This guide walks through how to setup the [MPI Operator](https://github.com/kubeflow/mpi-operator) and to how to run a MPIJob with the YuniKorn scheduler.
+
+## Installing the MPI Operator
+
+You can use the following command to install the mpi operator. If you have problems with installation,
+please refer to [this doc](https://github.com/kubeflow/mpi-operator) for details.
+```
+kubectl create -f https://raw.githubusercontent.com/kubeflow/mpi-operator/master/deploy/v2beta1/mpi-operator.yaml
+```
+
+## Run a MPI Job
+
+This example shows to run a pure MPI application.
+
+The program prints some basic information about the workers.
+Then, it calculates an approximate value for pi.
+
+Here is a Pi YAML example [example](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/examples/mpioperator/Pi/pi.yaml).
+```yaml
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+ name: pi
+spec:
+ slotsPerWorker: 1
+ runPolicy:
+ cleanPodPolicy: Running
+ ttlSecondsAfterFinished: 60
+ sshAuthMountPath: /home/mpiuser/.ssh
+ mpiReplicaSpecs:
+ Launcher:
+ replicas: 1
+ template:
+ labels:
+ applicationId: "mpi_job_pi"
+ queue: root.mpi
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - image: mpioperator/mpi-pi
+ name: mpi-launcher
+ securityContext:
+ runAsUser: 1000
+ command:
+ - mpirun
+ args:
+ - -n
+ - "2"
+ - /home/mpiuser/pi
+ resources:
+ limits:
+ cpu: 1
+ memory: 1Gi
+ Worker:
+ replicas: 2
+ template:
+ labels:
+ applicationId: "mpi_job_pi"
+ queue: root.mpi
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - image: mpioperator/mpi-pi
+ name: mpi-worker
+ securityContext:
+ runAsUser: 1000
+ command:
+ - /usr/sbin/sshd
+ args:
+ - -De
+ - -f
+ - /home/mpiuser/.sshd_config
+ resources:
+ limits:
+ cpu: 1
+ memory: 1Gi
+```
+Create the MPIJob.
+```
+kubectl create -f deployments/examples/mpioperator/Pi/pi.yaml
+```
+
+We added Yunikorn labels to the Pi example to demonstrate using the yunikorn scheduler.
+
+
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/run_nvidia.md b/versioned_docs/version-1.2.0/user_guide/workloads/run_nvidia.md
new file mode 100644
index 000000000..644910851
--- /dev/null
+++ b/versioned_docs/version-1.2.0/user_guide/workloads/run_nvidia.md
@@ -0,0 +1,346 @@
+---
+id: run_nvidia
+title: Run NVIDIA GPU Jobs
+description: How to run generic example of GPU scheduling with Yunikorn.
+keywords:
+ - NVIDIA GPU
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Yunikorn with NVIDIA GPUs
+This guide gives an overview of how to set up NVIDIA Device Plugin which enable user to run GPUs with Yunikorn, for more details please check [**Kubernetes with GPUs**](https://docs.nvidia.com/datacenter/cloud-native/kubernetes/install-k8s.html#option-2-installing-kubernetes-using-kubeadm).
+
+### Prerequisite
+Before following the steps below, Yunikorn need to deploy on the [**Kubernetes with GPUs**](https://docs.nvidia.com/datacenter/cloud-native/kubernetes/install-k8s.html#install-kubernetes).
+
+### Install NVIDIA Device Plugin
+Add the nvidia-device-plugin helm repository.
+```
+helm repo add nvdp https://nvidia.github.io/k8s-device-plugin
+helm repo update
+helm repo list
+```
+
+Verify the latest release version of the plugin is available.
+```
+helm search repo nvdp --devel
+NAME CHART VERSION APP VERSION DESCRIPTION
+nvdp/nvidia-device-plugin 0.12.3 0.12.3 A Helm chart for ...
+```
+
+Deploy the device plugin
+```
+kubectl create namespace nvidia
+helm install --generate-name nvdp/nvidia-device-plugin --namespace nvidia --version 0.12.3
+```
+
+Check the status of the pods to ensure NVIDIA device plugin is running
+```
+kubectl get pods -A
+
+NAMESPACE NAME READY STATUS RESTARTS AGE
+kube-flannel kube-flannel-ds-j24fx 1/1 Running 1 (11h ago) 11h
+kube-system coredns-78fcd69978-2x9l8 1/1 Running 1 (11h ago) 11h
+kube-system coredns-78fcd69978-gszrw 1/1 Running 1 (11h ago) 11h
+kube-system etcd-katlantyss-nzxt 1/1 Running 3 (11h ago) 11h
+kube-system kube-apiserver-katlantyss-nzxt 1/1 Running 4 (11h ago) 11h
+kube-system kube-controller-manager-katlantyss-nzxt 1/1 Running 3 (11h ago) 11h
+kube-system kube-proxy-4wz7r 1/1 Running 1 (11h ago) 11h
+kube-system kube-scheduler-katlantyss-nzxt 1/1 Running 4 (11h ago) 11h
+kube-system nvidia-device-plugin-1659451060-c92sb 1/1 Running 1 (11h ago) 11h
+```
+
+### Testing NVIDIA Device Plugin
+Create a gpu test yaml file.
+```
+# gpu-pod.yaml
+ apiVersion: v1
+ kind: Pod
+ metadata:
+ name: gpu-operator-test
+ spec:
+ restartPolicy: OnFailure
+ containers:
+ - name: cuda-vector-add
+ image: "nvidia/samples:vectoradd-cuda10.2"
+ resources:
+ limits:
+ nvidia.com/gpu: 1
+```
+Deploy the application.
+```
+kubectl apply -f gpu-pod.yaml
+```
+Check the logs to ensure the app completed successfully.
+```
+kubectl get pods gpu-operator-test
+
+NAME READY STATUS RESTARTS AGE
+gpu-operator-test 0/1 Completed 0 9d
+```
+Check the result.
+```
+kubectl logs gpu-operator-test
+
+[Vector addition of 50000 elements]
+Copy input data from the host memory to the CUDA device
+CUDA kernel launch with 196 blocks of 256 threads
+Copy output data from the CUDA device to the host memory
+Test PASSED
+Done
+```
+
+---
+## Enable GPU Time-Slicing (Optional)
+GPU time-slicing allow multi-tenant to share single GPU.
+To know how the GPU time-slicing works, please refer to [**Time-Slicing GPUs in Kubernetes**](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/gpu-sharing.html#introduction). This page covers ways to enable GPU scheduling in Yunikorn using [**NVIDIA GPU Operator**](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/gpu-operator).
+
+
+### Configuration
+Specify multiple configurations in a `ConfigMap` as in the following example.
+```yaml
+# time-slicing-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: time-slicing-config
+ namespace: nvidia
+data:
+ a100-40gb: |-
+ version: v1
+ sharing:
+ timeSlicing:
+ resources:
+ - name: nvidia.com/gpu
+ replicas: 8
+ - name: nvidia.com/mig-1g.5gb
+ replicas: 2
+ - name: nvidia.com/mig-2g.10gb
+ replicas: 2
+ - name: nvidia.com/mig-3g.20gb
+ replicas: 3
+ - name: nvidia.com/mig-7g.40gb
+ replicas: 7
+ rtx-3070: |-
+ version: v1
+ sharing:
+ timeSlicing:
+ resources:
+ - name: nvidia.com/gpu
+ replicas: 8
+```
+
+:::note
+If the GPU type in nodes do not include the a100-40gb or rtx-3070, you could modify the yaml file based on existing GPU types.
+For example, there are only multiple rtx-2080ti in the local kubernetes cluster.
+MIG is not supported by rtx-2080ti, so it could not replace the a100-40gb.
+Time slicing is supported by rtx-2080ti, so it could replace rtx-3070.
+:::
+
+:::info
+MIG support was added to Kubernetes in 2020. Refer to [**Supporting MIG in Kubernetes**](https://www.google.com/url?q=https://docs.google.com/document/d/1mdgMQ8g7WmaI_XVVRrCvHPFPOMCm5LQD5JefgAh6N8g/edit&sa=D&source=editors&ust=1655578433019961&usg=AOvVaw1F-OezvM-Svwr1lLsdQmu3) for details on how this works.
+:::
+
+Create a `ConfigMap` in the operator namespace.
+```bash
+kubectl create namespace nvidia
+kubectl create -f time-slicing-config.yaml
+```
+
+### Install NVIDIA GPU Operator
+Add the nvidia-gpu-operator helm repository.
+```bash
+helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
+helm repo update
+helm repo list
+```
+
+Enabling shared access to GPUs with the NVIDIA GPU Operator.
+- During fresh install of the NVIDIA GPU Operator with time-slicing enabled.
+ ```bash
+ helm install gpu-operator nvidia/gpu-operator \
+ -n nvidia \
+ --set devicePlugin.config.name=time-slicing-config
+ ```
+
+- For dynamically enabling time-slicing with GPU Operator already installed.
+ ```bash
+ kubectl patch clusterpolicy/cluster-policy \
+ -n nvidia --type merge \
+ -p '{"spec": {"devicePlugin": {"config": {"name": "time-slicing-config"}}}}'
+ ```
+
+### Applying the Time-Slicing Configuration
+There are two methods:
+- Across the cluster
+
+ Install the GPU Operator by passing the time-slicing `ConfigMap` name and the default configuration.
+ ```bash
+ kubectl patch clusterpolicy/cluster-policy \
+ -n nvidia --type merge \
+ -p '{"spec": {"devicePlugin": {"config": {"name": "time-slicing-config", "default": "rtx-3070"}}}}'
+ ```
+
+- On certain nodes
+
+ Label the node with the required time-slicing configuration in the `ConfigMap`.
+ ```bash
+ kubectl label node <node-name> nvidia.com/device-plugin.config=rtx-3070
+ ```
+
+Once the GPU Operator and Time-Slicing GPUs is installed, check the status of the pods to ensure all the containers are running and the validation is complete.
+```bash
+kubectl get pods -n nvidia
+```
+
+```bash
+NAME READY STATUS RESTARTS AGE
+gpu-feature-discovery-qbslx 2/2 Running 0 20h
+gpu-operator-7bdd8bf555-7clgv 1/1 Running 0 20h
+gpu-operator-node-feature-discovery-master-59b4b67f4f-q84zn 1/1 Running 0 20h
+gpu-operator-node-feature-discovery-worker-n58dv 1/1 Running 0 20h
+nvidia-container-toolkit-daemonset-8gv44 1/1 Running 0 20h
+nvidia-cuda-validator-tstpk 0/1 Completed 0 20h
+nvidia-dcgm-exporter-pgk7v 1/1 Running 1 20h
+nvidia-device-plugin-daemonset-w8hh4 2/2 Running 0 20h
+nvidia-device-plugin-validator-qrpxx 0/1 Completed 0 20h
+nvidia-operator-validator-htp6b 1/1 Running 0 20h
+```
+Verify that the time-slicing configuration is applied successfully.
+```bash
+kubectl describe node <node-name>
+```
+
+```bash
+...
+Capacity:
+ nvidia.com/gpu: 8
+...
+Allocatable:
+ nvidia.com/gpu: 8
+...
+```
+
+### Testing GPU Time-Slicing
+Create a wordload test file `plugin-test.yaml`.
+```yaml
+# plugin-test.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: nvidia-plugin-test
+ labels:
+ app: nvidia-plugin-test
+spec:
+ replicas: 5
+ selector:
+ matchLabels:
+ app: nvidia-plugin-test
+ template:
+ metadata:
+ labels:
+ app: nvidia-plugin-test
+ spec:
+ tolerations:
+ - key: nvidia.com/gpu
+ operator: Exists
+ effect: NoSchedule
+ containers:
+ - name: dcgmproftester11
+ image: nvidia/samples:dcgmproftester-2.1.7-cuda11.2.2-ubuntu20.04
+ command: ["/bin/sh", "-c"]
+ args:
+ - while true; do /usr/bin/dcgmproftester11 --no-dcgm-validation -t 1004 -d 300; sleep 30; done
+ resources:
+ limits:
+ nvidia.com/gpu: 1
+ securityContext:
+ capabilities:
+ add: ["SYS_ADMIN"]
+```
+
+Create a deployment with multiple replicas.
+```bash
+kubectl apply -f plugin-test.yaml
+```
+
+Verify that all five replicas are running.
+- In pods
+ ```bash
+ kubectl get pods
+ ```
+
+ ```bash
+ NAME READY STATUS RESTARTS AGE
+ nvidia-plugin-test-677775d6c5-bpsvn 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-m95zm 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-9kgzg 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-lrl2c 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-9r2pz 1/1 Running 0 8m8s
+ ```
+- In node
+ ```bash
+ kubectl describe node <node-name>
+ ```
+
+ ```bash
+ ...
+ Allocated resources:
+ (Total limits may be over 100 percent, i.e., overcommitted.)
+ Resource Requests Limits
+ -------- -------- ------
+ ...
+ nvidia.com/gpu 5 5
+ ...
+ ```
+- In NVIDIA system management Interface
+ ```bash
+ nvidia-smi
+ ```
+
+ ```bash
+ +-----------------------------------------------------------------------------+
+ | NVIDIA-SMI 520.61.05 Driver Version: 520.61.05 CUDA Version: 11.8 |
+ |-------------------------------+----------------------+----------------------+
+ | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
+ | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
+ | | | MIG M. |
+ |===============================+======================+======================|
+ | 0 NVIDIA GeForce ... On | 00000000:01:00.0 On | N/A |
+ | 46% 86C P2 214W / 220W | 4297MiB / 8192MiB | 100% Default |
+ | | | N/A |
+ +-------------------------------+----------------------+----------------------+
+
+ +-----------------------------------------------------------------------------+
+ | Processes: |
+ | GPU GI CI PID Type Process name GPU Memory |
+ | ID ID Usage |
+ |=============================================================================|
+ | 0 N/A N/A 1776886 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1776921 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1776937 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1777068 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1777079 C /usr/bin/dcgmproftester11 764MiB |
+ +-----------------------------------------------------------------------------+
+ ```
+
+- In Yunikorn UI applications
+
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/run_spark.md b/versioned_docs/version-1.2.0/user_guide/workloads/run_spark.md
index ca7552600..c49e373f3 100644
--- a/versioned_docs/version-1.2.0/user_guide/workloads/run_spark.md
+++ b/versioned_docs/version-1.2.0/user_guide/workloads/run_spark.md
@@ -32,13 +32,16 @@ This document assumes you have YuniKorn and its admission-controller both instal
## Prepare the docker image for Spark
-To run Spark on Kubernetes, you'll need the Spark docker images. You can 1) use the docker images provided by the YuniKorn
-team, or 2) build one from scratch. If you want to build your own Spark docker image, you can
+To run Spark on Kubernetes, you'll need the Spark docker images. You can 1) use the docker images provided by the Spark
+team, or 2) build one from scratch.
+If you want to build your own Spark docker image, you can find the [full instructions](https://spark.apache.org/docs/latest/building-spark.html)
+in the Spark documentation. Simplified steps:
* Download a Spark version that has Kubernetes support, URL: https://github.com/apache/spark
* Build spark with Kubernetes support:
```shell script
-mvn -Pyarn -Phadoop-2.7 -Dhadoop.version=2.7.4 -Phive -Pkubernetes -Phive-thriftserver -DskipTests package
+./build/mvn -Pkubernetes -DskipTests clean package
```
+Recommendation is to use the official images with different spark versions in the [dockerhub](https://hub.docker.com/r/apache/spark/tags)
## Create a namespace for Spark jobs
@@ -53,7 +56,9 @@ metadata:
EOF
```
-Create service account and cluster role bindings under `spark-test` namespace:
+## Create service account and role binding
+
+Create service account and role bindings inside the `spark-test` namespace:
```shell script
cat <<EOF | kubectl apply -n spark-test -f -
@@ -64,9 +69,9 @@ metadata:
namespace: spark-test
---
apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
metadata:
- name: spark-cluster-role
+ name: spark-role
namespace: spark-test
rules:
- apiGroups: [""]
@@ -77,17 +82,17 @@ rules:
verbs: ["get", "create", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
metadata:
- name: spark-cluster-role-binding
+ name: spark-role-binding
namespace: spark-test
subjects:
- kind: ServiceAccount
name: spark
namespace: spark-test
roleRef:
- kind: ClusterRole
- name: spark-cluster-role
+ kind: Role
+ name: spark-role
apiGroup: rbac.authorization.k8s.io
EOF
```
@@ -104,28 +109,39 @@ If this is running from local machine, you will need to start the proxy in order
kubectl proxy
```
-Run a simple SparkPi job (this assumes that the Spark binaries are installed to `/usr/local` directory).
+There are official images with different spark versions in the [dockerhub](https://hub.docker.com/r/apache/spark/tags)
+Run a simple SparkPi job, this assumes that the Spark binaries are installed locally in the `/usr/local` directory.
```shell script
-export SPARK_HOME=/usr/local/spark-2.4.4-bin-hadoop2.7/
+export SPARK_HOME=/usr/local/spark/
${SPARK_HOME}/bin/spark-submit --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
--master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
--class org.apache.spark.examples.SparkPi \
--conf spark.executor.instances=1 \
--conf spark.kubernetes.namespace=spark-test \
--conf spark.kubernetes.executor.request.cores=1 \
- --conf spark.kubernetes.container.image=apache/yunikorn:spark-2.4.4 \
- --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-test:spark \
- local:///opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar
+ --conf spark.kubernetes.container.image=docker.io/apache/spark:v3.3.0 \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+ local:///opt/spark/examples/jars/spark-examples_2.12-3.3.0.jar
+```
+
+:::note
+There are more options for setting the driver and executor in the [spark](https://spark.apache.org/docs/latest/running-on-kubernetes.html#configuration).
+Assigning the applicationId and the queue path are possible.
+```
+--conf spark.kubernetes.executor.label.applicationId=application-spark-0001
+--conf spark.kubernetes.driver.label.applicationId=application-spark-0001
+--conf spark.kubernetes.executor.label.queue=root.default.sandbox
+--conf spark.kubernetes.driver.label.queue=root.default.sandbox
```
+:::
You'll see Spark driver and executors been created on Kubernetes:
-
+
-You can also view the job info from YuniKorn UI. If you do not know how to access the YuniKorn UI, please read the document
-[here](../../get_started/get_started.md#access-the-web-ui).
+The spark-pi result is in the driver pod.
-
+
## What happens behind the scenes?
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/run_tensorflow.md b/versioned_docs/version-1.2.0/user_guide/workloads/run_tensorflow.md
index 367ac6e6c..bff16c59a 100644
--- a/versioned_docs/version-1.2.0/user_guide/workloads/run_tensorflow.md
+++ b/versioned_docs/version-1.2.0/user_guide/workloads/run_tensorflow.md
@@ -91,3 +91,135 @@ You can view the job info from YuniKorn UI. If you do not know how to access the
please read the document [here](../../get_started/get_started.md#access-the-web-ui).
+
+## Run a TensorFlow job with GPU scheduling
+To use Time-Slicing GPU your cluster must be configured to use [GPUs and Time-Slicing GPUs](https://yunikorn.apache.org/docs/next/user_guide/workloads/run_nvidia)
+This section covers a workload test scenario to validate TFJob with Time-slicing GPU.
+
+:::note
+Verify that the time-slicing configuration is applied successfully
+```bash
+kubectl describe node
+```
+
+```bash
+Capacity:
+ nvidia.com/gpu: 8
+...
+Allocatable:
+ nvidia.com/gpu: 8
+...
+```
+:::
+
+Create a workload test file `tf-gpu.yaml`
+```yaml
+# tf-gpu.yaml
+apiVersion: "kubeflow.org/v1"
+kind: "TFJob"
+metadata:
+ name: "tf-smoke-gpu"
+ namespace: kubeflow
+spec:
+ tfReplicaSpecs:
+ PS:
+ replicas: 1
+ template:
+ metadata:
+ creationTimestamp:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - args:
+ - python
+ - tf_cnn_benchmarks.py
+ - --batch_size=32
+ - --model=resnet50
+ - --variable_update=parameter_server
+ - --flush_stdout=true
+ - --num_gpus=1
+ - --local_parameter_device=cpu
+ - --device=cpu
+ - --data_format=NHWC
+ image: docker.io/kubeflow/tf-benchmarks-cpu:v20171202-bdab599-dirty-284af3
+ name: tensorflow
+ ports:
+ - containerPort: 2222
+ name: tfjob-port
+ workingDir: /opt/tf-benchmarks/scripts/tf_cnn_benchmarks
+ restartPolicy: OnFailure
+ Worker:
+ replicas: 1
+ template:
+ metadata:
+ creationTimestamp: null
+ labels:
+ applicationId: "tf_job_20200521_001"
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - args:
+ - python
+ - tf_cnn_benchmarks.py
+ - --batch_size=32
+ - --model=resnet50
+ - --variable_update=parameter_server
+ - --flush_stdout=true
+ - --num_gpus=1
+ - --local_parameter_device=cpu
+ - --device=gpu
+ - --data_format=NHWC
+ image: docker.io/kubeflow/tf-benchmarks-gpu:v20171202-bdab599-dirty-284af3
+ name: tensorflow
+ ports:
+ - containerPort: 2222
+ name: tfjob-port
+ resources:
+ limits:
+ nvidia.com/gpu: 2
+ workingDir: /opt/tf-benchmarks/scripts/tf_cnn_benchmarks
+ restartPolicy: OnFailure
+```
+Create the TFJob
+```bash
+kubectl apply -f tf-gpu.yaml
+kubectl get pods -n kubeflow
+```
+```bash
+NAME READY STATUS RESTARTS AGE
+tf-smoke-gpu-ps-0 1/1 Running 0 18m
+tf-smoke-gpu-worker-0 1/1 Running 0 18m
+training-operator-7d98f9dd88-dd45l 1/1 Running 0 19m
+```
+
+Verify that TFJob are running.
+- In pod logs
+ ```bash
+ kubectl logs tf-smoke-gpu-worker-0 -n kubeflow
+ ```
+ ```
+ .......
+ ..Found device 0 with properties
+ ..name: NVIDIA GeForce RTX 3080 major: 8 minor: 6 memoryClockRate(GHz): 1.71
+
+ .......
+ ..Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: NVIDIA GeForce RTX 3080, pci bus id: 0000:01:00.0, compute capability: 8.6)
+ .......
+ ```
+
+- In node
+ ```bash
+ ...
+ Allocated resources:
+ (Total limits may be over 100 percent, i.e., overcommitted.)
+ Resource Requests Limits
+ -------- -------- ------
+ ...
+ nvidia.com/gpu 2 2
+ ...
+ ```
+
+- In Yunikorn UI applications
+ 
diff --git a/versioned_docs/version-1.2.0/user_guide/workloads/workload_overview.md b/versioned_docs/version-1.2.0/user_guide/workloads/workload_overview.md
index cae7392bf..7040e79bd 100644
--- a/versioned_docs/version-1.2.0/user_guide/workloads/workload_overview.md
+++ b/versioned_docs/version-1.2.0/user_guide/workloads/workload_overview.md
@@ -53,6 +53,8 @@ omitted as it will be set automatically on newly created pods.
Examples of more advanced use cases can be found here:
+* [Run NVIDIA GPU Jobs](run_nvidia)
* [Run Spark Jobs](run_spark)
* [Run Flink Jobs](run_flink)
* [Run TensorFlow Jobs](run_tf)
+* [Run MPI Jobs](run_mpi)
diff --git a/versioned_sidebars/version-1.2.0-sidebars.json b/versioned_sidebars/version-1.2.0-sidebars.json
index fcd05a448..70e5eb490 100644
--- a/versioned_sidebars/version-1.2.0-sidebars.json
+++ b/versioned_sidebars/version-1.2.0-sidebars.json
@@ -6,10 +6,12 @@
],
"User Guide": [
"user_guide/deployment_modes",
+ "user_guide/service_config",
"user_guide/queue_config",
"user_guide/placement_rules",
"user_guide/usergroup_resolution",
"user_guide/sorting_policies",
+ "user_guide/priorities",
"user_guide/acls",
"user_guide/resource_quota_management",
"user_guide/gang_scheduling",
@@ -19,9 +21,11 @@
"label": "Workloads",
"items": [
"user_guide/workloads/workload_overview",
+ "user_guide/workloads/run_nvidia",
"user_guide/workloads/run_spark",
"user_guide/workloads/run_flink",
- "user_guide/workloads/run_tf"
+ "user_guide/workloads/run_tf",
+ "user_guide/workloads/run_mpi"
]
},
{
@@ -33,7 +37,7 @@
"api/system"
]
},
- "user_guide/trouble_shooting"
+ "user_guide/troubleshooting"
],
"Developer Guide": [
"developer_guide/env_setup",
@@ -41,6 +45,7 @@
"developer_guide/dependencies",
"developer_guide/deployment",
"developer_guide/openshift_development",
+ "developer_guide/translation",
{
"type": "category",
"label": "Designs",
@@ -49,19 +54,32 @@
"design/k8shim",
"design/scheduler_plugin",
"design/gang_scheduling",
+ "design/scheduler_plugin",
+ "design/gang_scheduling",
+ "design/user_group",
+ "design/user_based_resource_usage_tracking",
"design/interface_message_simplification",
"design/cache_removal",
"design/simple_preemptor",
"design/generic_resource",
- "design/namespace_resource_quota",
"design/pluggable_app_management",
+ "design/priority_scheduling",
"design/resilience",
- "design/predicates",
- "design/scheduler_configuration",
"design/state_aware_scheduling",
"design/scheduler_object_states",
- "design/scheduler_core_design",
- "design/cross_queue_preemption"
+ "design/config_v2",
+ "design/scheduler_configuration"
+ ]
+ },
+ {
+ "type": "category",
+ "label": "Archived Designs",
+ "items": [
+ "archived_design/k8shim",
+ "archived_design/namespace_resource_quota",
+ "archived_design/predicates",
+ "archived_design/scheduler_core_design",
+ "archived_design/cross_queue_preemption"
]
}
],
diff --git a/versions.json b/versions.json
index 135205d08..5625ba50d 100644
--- a/versions.json
+++ b/versions.json
@@ -1,4 +1,5 @@
[
+ "1.2.0",
"1.1.0",
"1.0.0",
"0.12.2",