You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@yunikorn.apache.org by yu...@apache.org on 2021/12/08 19:21:00 UTC
[incubator-yunikorn-site] branch YUNIKORN-901 created (now b87f47d)
This is an automated email from the ASF dual-hosted git repository.
yuchaoran pushed a change to branch YUNIKORN-901
in repository https://gitbox.apache.org/repos/asf/incubator-yunikorn-site.git.
at b87f47d Added versioned doc for v0.12.0
This branch includes the following new commits:
new b87f47d Added versioned doc for v0.12.0
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-yunikorn-site] 01/01: Added versioned doc for v0.12.0
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuchaoran pushed a commit to branch YUNIKORN-901
in repository https://gitbox.apache.org/repos/asf/incubator-yunikorn-site.git
commit b87f47d129cf4870a011971e08b75e4bc567e56a
Author: Chaoran Yu <yu...@gmail.com>
AuthorDate: Wed Dec 8 11:20:50 2021 -0800
Added versioned doc for v0.12.0
---
versioned_docs/version-0.12.0/api/cluster.md | 128 ++
versioned_docs/version-0.12.0/api/scheduler.md | 1316 ++++++++++++++++++++
versioned_docs/version-0.12.0/api/system.md | 225 ++++
.../version-0.12.0/assets/application-state.png | Bin 0 -> 117964 bytes
.../version-0.12.0/assets/architecture.png | Bin 0 -> 188534 bytes
.../version-0.12.0/assets/cpu_profile.jpg | Bin 0 -> 259919 bytes
.../version-0.12.0/assets/dashboard_secret.png | Bin 0 -> 31815 bytes
.../assets/dashboard_token_select.png | Bin 0 -> 47533 bytes
.../assets/docker-dektop-minikube.png | Bin 0 -> 249095 bytes
.../version-0.12.0/assets/docker-desktop.png | Bin 0 -> 247891 bytes
.../version-0.12.0/assets/fifo-state-example.png | Bin 0 -> 239302 bytes
.../version-0.12.0/assets/gang_clean_up.png | Bin 0 -> 169887 bytes
.../version-0.12.0/assets/gang_generic_flow.png | Bin 0 -> 222524 bytes
.../assets/gang_scheduling_iintro.png | Bin 0 -> 43907 bytes
.../version-0.12.0/assets/gang_timeout.png | Bin 0 -> 179466 bytes
.../version-0.12.0/assets/gang_total_ask.png | Bin 0 -> 22557 bytes
.../version-0.12.0/assets/goland_debug.jpg | Bin 0 -> 198845 bytes
.../assets/k8shim-application-state.png | Bin 0 -> 129912 bytes
.../version-0.12.0/assets/k8shim-node-state.png | Bin 0 -> 52316 bytes
.../assets/k8shim-scheduler-state.png | Bin 0 -> 53283 bytes
.../version-0.12.0/assets/k8shim-task-state.png | Bin 0 -> 144820 bytes
.../version-0.12.0/assets/namespace-mapping.png | Bin 0 -> 327547 bytes
.../version-0.12.0/assets/node-bin-packing.png | Bin 0 -> 232909 bytes
versioned_docs/version-0.12.0/assets/node-fair.png | Bin 0 -> 462310 bytes
.../version-0.12.0/assets/object-state.png | Bin 0 -> 39732 bytes
.../version-0.12.0/assets/pluggable-app-mgmt.jpg | Bin 0 -> 79170 bytes
.../version-0.12.0/assets/prometheus.png | Bin 0 -> 88021 bytes
.../version-0.12.0/assets/queue-fairness.png | Bin 0 -> 173299 bytes
.../assets/queue-resource-quotas.png | Bin 0 -> 283689 bytes
.../assets/resilience-node-recovery.jpg | Bin 0 -> 319477 bytes
.../version-0.12.0/assets/resilience-workflow.jpg | Bin 0 -> 441551 bytes
.../version-0.12.0/assets/spark-jobs-on-ui.png | Bin 0 -> 528736 bytes
.../version-0.12.0/assets/spark-pods.png | Bin 0 -> 303407 bytes
.../version-0.12.0/assets/tf-job-on-ui.png | Bin 0 -> 327800 bytes
.../version-0.12.0/assets/throughput.png | Bin 0 -> 252615 bytes
.../version-0.12.0/assets/yk-ui-screenshots.gif | Bin 0 -> 813848 bytes
.../version-0.12.0/design/architecture.md | 62 +
.../version-0.12.0/design/cache_removal.md | 446 +++++++
.../design/cross_queue_preemption.md | 126 ++
.../version-0.12.0/design/gang_scheduling.md | 605 +++++++++
versioned_docs/version-0.12.0/design/k8shim.md | 86 ++
.../design/namespace_resource_quota.md | 183 +++
.../design/pluggable_app_management.md | 75 ++
versioned_docs/version-0.12.0/design/predicates.md | 80 ++
versioned_docs/version-0.12.0/design/resilience.md | 144 +++
.../design/scheduler_configuration.md | 246 ++++
.../version-0.12.0/design/scheduler_core_design.md | 401 ++++++
.../design/scheduler_object_states.md | 127 ++
.../design/state_aware_scheduling.md | 112 ++
.../version-0.12.0/developer_guide/build.md | 184 +++
.../version-0.12.0/developer_guide/deployment.md | 133 ++
.../version-0.12.0/developer_guide/env_setup.md | 156 +++
.../developer_guide/openshift_development.md | 182 +++
.../version-0.12.0/get_started/core_features.md | 73 ++
.../version-0.12.0/get_started/get_started.md | 73 ++
.../evaluate_perf_function_with_kubemark.md | 103 ++
.../version-0.12.0/performance/metrics.md | 109 ++
.../performance/performance_tutorial.md | 452 +++++++
.../version-0.12.0/performance/profiling.md | 122 ++
versioned_docs/version-0.12.0/user_guide/acls.md | 119 ++
.../version-0.12.0/user_guide/gang_scheduling.md | 285 +++++
.../version-0.12.0/user_guide/placement_rules.md | 354 ++++++
.../version-0.12.0/user_guide/queue_config.md | 353 ++++++
.../user_guide/resource_quota_mgmt.md | 324 +++++
.../version-0.12.0/user_guide/sorting_policies.md | 185 +++
.../version-0.12.0/user_guide/trouble_shooting.md | 192 +++
.../user_guide/usergroup_resolution.md | 68 +
.../user_guide/workloads/run_flink.md | 66 +
.../user_guide/workloads/run_spark.md | 149 +++
.../user_guide/workloads/run_tensorflow.md | 93 ++
versioned_sidebars/version-0.12.0-sidebars.json | 208 ++++
versions.json | 1 +
72 files changed, 8346 insertions(+)
diff --git a/versioned_docs/version-0.12.0/api/cluster.md b/versioned_docs/version-0.12.0/api/cluster.md
new file mode 100644
index 0000000..5ad3f4f
--- /dev/null
+++ b/versioned_docs/version-0.12.0/api/cluster.md
@@ -0,0 +1,128 @@
+---
+id: cluster
+title: Cluster
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Clusters
+
+Returns general information about the clusters managed by the YuniKorn Scheduler. Information includes number of (total, failed, pending, running, completed) applications and containers.
+
+**URL** : `/ws/v1/clusters`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+As an example, here is a response from a 2-node cluster with 3 applications and 4 running containers.
+
+```json
+[
+ {
+ "partition": "[rm-123]default",
+ "clusterName": "kubernetes",
+ "totalApplications": "3",
+ "failedApplications": "1",
+ "pendingApplications": "",
+ "runningApplications": "3",
+ "completedApplications": "",
+ "totalContainers": "4",
+ "failedContainers": "",
+ "pendingContainers": "",
+ "runningContainers": "4",
+ "activeNodes": "2",
+ "totalNodes": "2",
+ "failedNodes": ""
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Clusters utilization
+
+Returns statistical data related the cluster resource utilization
+
+**URL** : `/ws/v1/clusters/utilization`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "partition": "[mycluster]default",
+ "utilization": [
+ {
+ "type": "memory",
+ "total": 5076,
+ "used": 1500,
+ "usage": "29%"
+ },
+ {
+ "type": "vcore",
+ "total": 4000,
+ "used": 300,
+ "usage": "7%"
+ }
+ ]
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
diff --git a/versioned_docs/version-0.12.0/api/scheduler.md b/versioned_docs/version-0.12.0/api/scheduler.md
new file mode 100644
index 0000000..c7385be
--- /dev/null
+++ b/versioned_docs/version-0.12.0/api/scheduler.md
@@ -0,0 +1,1316 @@
+---
+id: scheduler
+title: Scheduler
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Partitions
+
+Displays general information about the partition like name, state, capacity, used capacity and node sorting policy.
+
+**URL** : `/ws/v1/partitions`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "name": "[mycluster]default",
+ "state": "Active",
+ "lastStateTransitionTime": "2021-05-20 12:25:49.018953 +0530 IST m=+0.005949717",
+ "capacity": {
+ "capacity": "[memory:1000 vcore:1000]",
+ "usedCapacity": "[memory:800 vcore:500]"
+ },
+ "nodeSortingPolicy": {
+ "type": "fair",
+ "resourceWeights": {
+ "memory": 1.5,
+ "vcore": 1.3
+ }
+ },
+ "applications": {
+ "New": 5,
+ "Pending": 5,
+ "total": 10
+ }
+ },
+ {
+ "name": "[mycluster]gpu",
+ "state": "Active",
+ "lastStateTransitionTime": "2021-05-19 12:25:49.018953 +0530 IST m=+0.005949717",
+ "capacity": {
+ "capacity": "[memory:2000 vcore:2000]",
+ "usedCapacity": "[memory:500 vcore:300]"
+ },
+ "nodeSortingPolicy": {
+ "type": "binpacking",
+ "resourceWeights": {
+ "memory": 0,
+ "vcore": 4.11
+ }
+ },
+ "applications": {
+ "New": 5,
+ "Running": 10,
+ "Pending": 5,
+ "total": 20
+ }
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Queues
+
+### Partition queues
+
+Fetch all Queues associated with given Partition and displays general information about the queues like name, status, capacities and properties.
+The queues' hierarchy is kept in the response json.
+
+**URL** : `/ws/v1/partition/{partitionName}/queues`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+For the default queue hierarchy (only `root.default` leaf queue exists) a similar response to the following is sent back to the client:
+
+```json
+[
+ {
+
+ "queuename": "root",
+ "status": "Active",
+ "maxResource": "[memory:8000 vcore:8000]",
+ "guaranteedResource": "[memory:54 vcore:80]",
+ "allocatedResource": "[memory:54 vcore:80]",
+ "isLeaf": "false",
+ "isManaged": "false",
+ "properties": {
+ "application.sort.policy":"stateaware"
+ },
+ "parent": "",
+ "partition": "[mycluster]default",
+ "children": [
+ {
+ "queuename": "root.default",
+ "status": "Active",
+ "maxResource": "[memory:8000 vcore:8000]",
+ "guaranteedResource": "[memory:54 vcore:80]",
+ "allocatedResource": "[memory:54 vcore:80]",
+ "isLeaf": "true",
+ "isManaged": "false",
+ "parent": "root"
+ }
+ ]
+ }
+]
+```
+
+### Error response
+
+**Code** : `400 Bad Request`
+
+**Content examples**
+
+```json
+{
+ "status_code": 400,
+ "message": "Partition is missing in URL path. Please check the usage documentation",
+ "description": "Partition is missing in URL path. Please check the usage documentation"
+}
+```
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+### All Queues
+
+Fetch all Queues across different Partitions and displays general information about the queues like name, status, capacities and properties.
+The queues' hierarchy is kept in the response json.
+
+**URL** : `/ws/v1/queues`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+For the default queue hierarchy (only `root.default` leaf queue exists) a similar response to the following is sent back to the client:
+
+```json
+{
+ "partitionName": "[mycluster]default",
+ "capacity": {
+ "capacity": "map[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:80000 pods:110 vcore:60000]",
+ "usedCapacity": "0"
+ },
+ "nodes": null,
+ "queues": {
+ "queuename": "root",
+ "status": "Active",
+ "capacities": {
+ "capacity": "[]",
+ "maxCapacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:80000 pods:110 vcore:60000]",
+ "usedCapacity": "[memory:8000 vcore:8000]",
+ "absUsedCapacity": "[memory:54 vcore:80]"
+ },
+ "queues": [
+ {
+ "queuename": "default",
+ "status": "Active",
+ "capacities": {
+ "capacity": "[]",
+ "maxCapacity": "[]",
+ "usedCapacity": "[memory:8000 vcore:8000]",
+ "absUsedCapacity": "[]"
+ },
+ "queues": null,
+ "properties": {}
+ }
+ ],
+ "properties": {}
+ }
+}
+```
+
+## Applications
+
+### Queue applications
+
+Fetch all Applications for the given Partition Queue combination and displays general information about the applications like used resources, queue name, submission time and allocations.
+
+**URL** : `/ws/v1/partition/{partitioName}/queue/{queueName}/applications`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+In the example below there are three allocations belonging to two applications.
+
+```json
+[
+ {
+ "applicationID": "application-0001",
+ "usedResource": "[memory:4000 vcore:4000]",
+ "partition": "[mycluster]default",
+ "queueName": "root.default",
+ "submissionTime": 1595939756253216000,
+ "allocations": [
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": null,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "resource": "[memory:4000 vcore:4000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default"
+ }
+ ],
+ "applicationState": "Running"
+ },
+ {
+ "applicationID": "application-0002",
+ "usedResource": "[memory:4000 vcore:4000]",
+ "partition": "[mycluster]default",
+ "queueName": "root.default",
+ "submissionTime": 1595939756253460000,
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": null,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default"
+ },
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": null,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0002",
+ "applicationId": "application-0002",
+ "partition": "default"
+ }
+ ],
+ "applicationState": "Running"
+ }
+]
+```
+
+### Error response
+
+**Code** : `400 Bad Request`
+
+**Content examples**
+
+```json
+{
+ "status_code": 400,
+ "message": "Partition is missing in URL path. Please check the usage documentation",
+ "description": "Partition is missing in URL path. Please check the usage documentation"
+}
+```
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+### All applications
+
+Fetch all Applications across different Partitions and displays general information about the applications like used resources, queue name, submission time and allocations.
+
+**URL** : `/ws/v1/apps`
+
+**Method** : `GET`
+
+**Query Params** :
+
+1. queue=<fully qualified queue name\>
+
+The fully qualified queue name used to filter the applications that run within the given queue. For example, "/ws/v1/apps?queue=root.default" returns the applications running in "root.default" queue.
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+In the example below there are three allocations belonging to two applications.
+
+```json
+[
+ {
+ "applicationID": "application-0002",
+ "usedResource": "[memory:4000 vcore:4000]",
+ "partition": "[mycluster]default",
+ "queueName": "root.default",
+ "submissionTime": 1595939756253216000,
+ "allocations": [
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": null,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "resource": "[memory:4000 vcore:4000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default"
+ }
+ ],
+ "applicationState": "Running"
+ },
+ {
+ "applicationID": "application-0001",
+ "usedResource": "[memory:4000 vcore:4000]",
+ "partition": "[mycluster]default",
+ "queueName": "root.default",
+ "submissionTime": 1595939756253460000,
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": null,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default"
+ },
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": null,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0002",
+ "applicationId": "application-0001",
+ "partition": "default"
+ }
+ ],
+ "applicationState": "Running"
+ }
+]
+```
+
+## Nodes
+
+### Partition nodes
+
+Fetch all Nodes associated with given Partition and displays general information about the nodes managed by YuniKorn.
+Node details include host and rack name, capacity, resources and allocations.
+
+**URL** : `/ws/v1/partition/{partitionName}/nodes`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+Here you can see an example response from a 2-node cluster having 3 allocations.
+
+```json
+[
+ {
+ "nodeID": "node-0001",
+ "hostName": "",
+ "rackName": "",
+ "capacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:14577 pods:110 vcore:10000]",
+ "allocated": "[memory:6000 vcore:6000]",
+ "occupied": "[memory:154 vcore:750]",
+ "available": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:6423 pods:110 vcore:1250]",
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": null,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default"
+ },
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": null,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "resource": "[memory:4000 vcore:4000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default"
+ }
+ ],
+ "schedulable": true
+ },
+ {
+ "nodeID": "node-0002",
+ "hostName": "",
+ "rackName": "",
+ "capacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:14577 pods:110 vcore:10000]",
+ "allocated": "[memory:2000 vcore:2000]",
+ "occupied": "[memory:154 vcore:750]",
+ "available": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:6423 pods:110 vcore:1250]",
+ "allocations": [
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": null,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0002",
+ "applicationId": "application-0001",
+ "partition": "default"
+ }
+ ],
+ "schedulable": true
+ }
+]
+```
+
+### Error response
+
+**Code** : `400 Bad Request`
+
+**Content examples**
+
+```json
+{
+ "status_code": 400,
+ "message": "Partition is missing in URL path. Please check the usage documentation",
+ "description": "Partition is missing in URL path. Please check the usage documentation"
+}
+```
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+### All nodes
+
+Fetch all Nodes acrosss different Partitions and displays general information about the nodes managed by YuniKorn.
+Node details include host and rack name, capacity, resources and allocations.
+
+**URL** : `/ws/v1/nodes`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+Here you can see an example response from a 2-node cluster having 3 allocations.
+
+```json
+[
+ {
+ "partitionName": "[mycluster]default",
+ "nodesInfo": [
+ {
+ "nodeID": "node-0001",
+ "hostName": "",
+ "rackName": "",
+ "capacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:14577 pods:110 vcore:10000]",
+ "allocated": "[memory:6000 vcore:6000]",
+ "occupied": "[memory:154 vcore:750]",
+ "available": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:6423 pods:110 vcore:1250]",
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": null,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default"
+ },
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": null,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "resource": "[memory:4000 vcore:4000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default"
+ }
+ ],
+ "schedulable": true
+ },
+ {
+ "nodeID": "node-0002",
+ "hostName": "",
+ "rackName": "",
+ "capacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:14577 pods:110 vcore:10000]",
+ "allocated": "[memory:2000 vcore:2000]",
+ "occupied": "[memory:154 vcore:750]",
+ "available": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:6423 pods:110 vcore:1250]",
+ "allocations": [
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": null,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0002",
+ "applicationId": "application-0001",
+ "partition": "default"
+ }
+ ],
+ "schedulable": true
+ }
+ ]
+ }
+]
+```
+
+## Nodes utilization
+
+Shows how nodes are distributed with regarding the utilization
+
+**URL** : `/ws/v1/nodes/utilization`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+[
+ {
+ partition: default,
+ utilization: [ {
+ type: "cpu",
+ total: 100,
+ used: 50,
+ usage: 50%
+ },
+ {
+ type: "memory",
+ total: 1000,
+ used: 500,
+ usage: 50%
+ }
+ ]
+ },
+ ...
+]
+```
+
+## Goroutines info
+
+Dumps the stack traces of the currently running goroutines.
+
+**URL** : `/ws/v1/stack`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+goroutine 356 [running
+]:
+github.com/apache/incubator-yunikorn-core/pkg/webservice.getStackInfo.func1(0x30a0060,
+0xc003e900e0,
+0x2)
+ /yunikorn/go/pkg/mod/github.com/apache/incubator-yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/handlers.go: 41 +0xab
+github.com/apache/incubator-yunikorn-core/pkg/webservice.getStackInfo(0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /yunikorn/go/pkg/mod/github.com/apache/incubator-yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/handlers.go: 48 +0x71
+net/http.HandlerFunc.ServeHTTP(0x2df0e10,
+0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /usr/local/go/src/net/http/server.go: 1995 +0x52
+github.com/apache/incubator-yunikorn-core/pkg/webservice.Logger.func1(0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /yunikorn/go/pkg/mod/github.com/apache/incubator-yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/webservice.go: 65 +0xd4
+net/http.HandlerFunc.ServeHTTP(0xc00003a570,
+0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /usr/local/go/src/net/http/server.go: 1995 +0x52
+github.com/gorilla/mux.(*Router).ServeHTTP(0xc00029cb40,
+0x30a0060,
+0xc003e900e0,
+0xc0063fee00)
+ /yunikorn/go/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go: 212 +0x140
+net/http.serverHandler.ServeHTTP(0xc0000df520,
+0x30a0060,
+0xc003e900e0,
+0xc0063fee00)
+ /usr/local/go/src/net/http/server.go: 2774 +0xcf
+net/http.(*conn).serve(0xc0000eab40,
+0x30a61a0,
+0xc003b74000)
+ /usr/local/go/src/net/http/server.go: 1878 +0x812
+created by net/http.(*Server).Serve
+ /usr/local/go/src/net/http/server.go: 2884 +0x4c5
+
+goroutine 1 [chan receive,
+ 26 minutes
+]:
+main.main()
+ /yunikorn/pkg/shim/main.go: 52 +0x67a
+
+goroutine 19 [syscall,
+ 26 minutes
+]:
+os/signal.signal_recv(0x1096f91)
+ /usr/local/go/src/runtime/sigqueue.go: 139 +0x9f
+os/signal.loop()
+ /usr/local/go/src/os/signal/signal_unix.go: 23 +0x30
+created by os/signal.init.0
+ /usr/local/go/src/os/signal/signal_unix.go: 29 +0x4f
+
+...
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Metrics
+
+Endpoint to retrieve metrics from the Prometheus server.
+The metrics are dumped with help messages and type information.
+
+**URL** : `/ws/v1/metrics`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"} 2.567e-05
+go_gc_duration_seconds{quantile="0.25"} 3.5727e-05
+go_gc_duration_seconds{quantile="0.5"} 4.5144e-05
+go_gc_duration_seconds{quantile="0.75"} 6.0024e-05
+go_gc_duration_seconds{quantile="1"} 0.00022528
+go_gc_duration_seconds_sum 0.021561648
+go_gc_duration_seconds_count 436
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines 82
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+go_info{version="go1.12.17"} 1
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes 9.6866248e+07
+
+...
+
+# HELP yunikorn_scheduler_vcore_nodes_usage Nodes resource usage, by resource name.
+# TYPE yunikorn_scheduler_vcore_nodes_usage gauge
+yunikorn_scheduler_vcore_nodes_usage{range="(10%, 20%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(20%,30%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(30%,40%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(40%,50%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(50%,60%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(60%,70%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(70%,80%]"} 1
+yunikorn_scheduler_vcore_nodes_usage{range="(80%,90%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(90%,100%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="[0,10%]"} 0
+```
+
+## Configuration validation
+
+**URL** : `/ws/v1/validate-conf`
+
+**Method** : `POST`
+
+**Auth required** : NO
+
+### Success response
+
+Regardless whether the configuration is allowed or not if the server was able to process the request, it will yield a 200 HTTP status code.
+
+**Code** : `200 OK`
+
+#### Allowed configuration
+
+Sending the following simple configuration yields an accept
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+```
+
+Reponse
+
+```json
+{
+ "allowed": true,
+ "reason": ""
+}
+```
+
+#### Disallowed configuration
+
+The following configuration is not allowed due to the "wrong_text" field put into the yaml file.
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+ - wrong_text
+```
+
+Reponse
+
+```json
+{
+ "allowed": false,
+ "reason": "yaml: unmarshal errors:\n line 7: cannot unmarshal !!str `wrong_text` into configs.PartitionConfig"
+}
+```
+
+## Configuration Create
+
+Endpoint to create scheduler configuration, but currently limited for configuration validation purpose alone
+
+**URL** : `/ws/v1/config`
+
+**Method** : `POST`
+
+**Query Params** :
+
+1. dry_run
+
+Mandatory Parameter. Only dry_run=1 is allowed and can be used for configuration validation only, not for the actual config creation.
+
+**Auth required** : NO
+
+### Success response
+
+Regardless whether the configuration is allowed or not if the server was able to process the request, it will yield a 200 HTTP status code.
+
+**Code** : `200 OK`
+
+#### Allowed configuration
+
+Sending the following simple configuration yields an accept
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+```
+
+Response
+
+```json
+{
+ "allowed": true,
+ "reason": ""
+}
+```
+
+#### Disallowed configuration
+
+The following configuration is not allowed due to the "wrong_text" field put into the yaml file.
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+ - wrong_text
+```
+
+Response
+
+```json
+{
+ "allowed": false,
+ "reason": "yaml: unmarshal errors:\n line 7: cannot unmarshal !!str `wrong_text` into configs.PartitionConfig"
+}
+```
+
+### Error response
+
+**Code** : `400 Bad Request`
+
+**Content examples**
+
+```json
+{
+ "status_code": 400,
+ "message": "Dry run param is missing. Please check the usage documentation",
+ "description": "Dry run param is missing. Please check the usage documentation"
+}
+```
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Configuration
+
+Endpoint to retrieve the current scheduler configuration
+
+**URL** : `/ws/v1/config`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content example**
+
+```yaml
+partitions:
+- name: default
+ queues:
+ - name: root
+ parent: true
+ submitacl: '*'
+ placementrules:
+ - name: tag
+ create: true
+ value: namespace
+checksum: D75996C07D5167F41B33E27CCFAEF1D5C55BE3C00EE6526A7ABDF8435DB4078E
+```
+
+## Configuration update
+
+Endpoint to override scheduler configuration.
+
+**URL** : `/ws/v1/config`
+
+**Method** : `PUT`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content example**
+
+```yaml
+partitions:
+ -
+ name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ properties:
+ application.sort.policy: stateaware
+checksum: BAB3D76402827EABE62FA7E4C6BCF4D8DD9552834561B6B660EF37FED9299791
+```
+**Note:** Updates must use a current running configuration as the base.
+The base configuration is the configuration version that was retrieved earlier via a GET request and updated by the user.
+The update request must contain the checksum of the _base_ configuration.
+If the checksum provided in the update request differs from the currently running configuration checksum the update will be rejected.
+
+### Failure response
+
+The configuration update can fail due to different reasons such as:
+- invalid configuration,
+- incorrect base checksum.
+
+In each case the transaction will be rejected, and the proper
+error message will be returned as a response.
+
+**Code** : `409 Conflict`
+
+**Message example** : root queue must not have resource limits set
+
+**Content example**
+
+```yaml
+partitions:
+ -
+ name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ resources:
+ guaranteed:
+ memory: "512"
+ vcore: "1"
+ properties:
+ application.sort.policy: stateaware
+checksum: BAB3D76402827EABE62FA7E4C6BCF4D8DD9552834561B6B660EF37FED9299791
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Application history
+
+Endpoint to retrieve historical data about the number of total applications by timestamp.
+
+**URL** : `/ws/v1/history/apps`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "timestamp": 1595939966153460000,
+ "totalApplications": "1"
+ },
+ {
+ "timestamp": 1595940026152892000,
+ "totalApplications": "1"
+ },
+ {
+ "timestamp": 1595940086153799000,
+ "totalApplications": "2"
+ },
+ {
+ "timestamp": 1595940146154497000,
+ "totalApplications": "2"
+ },
+ {
+ "timestamp": 1595940206155187000,
+ "totalApplications": "2"
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Container history
+
+Endpoint to retrieve historical data about the number of total containers by timestamp.
+
+**URL** : `/ws/v1/history/containers`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "timestamp": 1595939966153460000,
+ "totalContainers": "1"
+ },
+ {
+ "timestamp": 1595940026152892000,
+ "totalContainers": "1"
+ },
+ {
+ "timestamp": 1595940086153799000,
+ "totalContainers": "3"
+ },
+ {
+ "timestamp": 1595940146154497000,
+ "totalContainers": "3"
+ },
+ {
+ "timestamp": 1595940206155187000,
+ "totalContainers": "3"
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+
+## Endpoint healthcheck
+
+Endpoint to retrieve historical data about critical logs, negative resource on node/cluster/app, ...
+
+**URL** : `/ws/v1/scheduler/healthcheck`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+{
+ "Healthy": true,
+ "HealthChecks": [
+ {
+ "Name": "Scheduling errors",
+ "Succeeded": true,
+ "Description": "Check for scheduling error entries in metrics",
+ "DiagnosisMessage": "There were 0 scheduling errors logged in the metrics"
+ },
+ {
+ "Name": "Failed nodes",
+ "Succeeded": true,
+ "Description": "Check for failed nodes entries in metrics",
+ "DiagnosisMessage": "There were 0 failed nodes logged in the metrics"
+ },
+ {
+ "Name": "Negative resources",
+ "Succeeded": true,
+ "Description": "Check for negative resources in the partitions",
+ "DiagnosisMessage": "Partitions with negative resources: []"
+ },
+ {
+ "Name": "Negative resources",
+ "Succeeded": true,
+ "Description": "Check for negative resources in the nodes",
+ "DiagnosisMessage": "Nodes with negative resources: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if a node's allocated resource <= total resource of the node",
+ "DiagnosisMessage": "Nodes with inconsistent data: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if total partition resource == sum of the node resources from the partition",
+ "DiagnosisMessage": "Partitions with inconsistent data: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if node total resource = allocated resource + occupied resource + available resource",
+ "DiagnosisMessage": "Nodes with inconsistent data: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if node capacity >= allocated resources on the node",
+ "DiagnosisMessage": "Nodes with inconsistent data: []"
+ },
+ {
+ "Name": "Reservation check",
+ "Succeeded": true,
+ "Description": "Check the reservation nr compared to the number of nodes",
+ "DiagnosisMessage": "Reservation/node nr ratio: [0.000000]"
+ }
+ ]
+}
+```
+
+## Retrieve full state dump
+
+Endpoint to retrieve the following information in a single response:
+
+* List of partitions
+* List of applications (running and completed)
+* Application history
+* Nodes
+* Utilization of nodes
+* Generic cluster information
+* Cluster utilization
+* Container history
+* Queues
+
+**URL** : `/ws/v1/fullstatedump`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+The output of this REST query can be rather big and it is a combination of those which have already been demonstrated.
+
+### Failure response
+
+**Code**: `500 Internal Server Error`
+
+## Enable or disable periodic state dump to an external file inside the container which runs Yunikorn
+
+Endpoint to enable a state dump to be written periodically. By default, it is 60 seconds. The output goes to a file called `yunikorn-state.txt`. In the current version, the file is located in the current working directory of Yunikorn and it is not configurable.
+
+Trying to enable or disable this feature more than once in a row results in an error.
+
+**URL** : `/ws/v1/periodicstatedump/{switch}/{periodSeconds}`
+
+**Method** : `PUT`
+
+**Auth required** : NO
+
+The value `{switch}` can be either `disable` or `enable`. The `{periodSeconds}` defines how often state snapshots should be taken. It is expected to be a positive integer and only interpreted in case of `enable`.
+
+### Success response
+
+**Code** : `200 OK`
+
+### Error response
+
+**Code**: `400 Bad Request`
+
+**Content examples**
+
+```json
+{
+ "status_code": 400,
+ "message": "required parameter enabled/disabled is missing",
+ "description": "required parameter enabled/disabled is missing"
+}
+```
diff --git a/versioned_docs/version-0.12.0/api/system.md b/versioned_docs/version-0.12.0/api/system.md
new file mode 100644
index 0000000..1d685ff
--- /dev/null
+++ b/versioned_docs/version-0.12.0/api/system.md
@@ -0,0 +1,225 @@
+---
+id: system
+title: System
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+These endpoints are for the [pprof](https://github.com/google/pprof) profiling tool.
+
+## pprof
+
+**URL** : `/debug/pprof/`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+/debug/pprof/
+
+Types of profiles available:
+Count Profile
+273 allocs
+0 block
+0 cmdline
+78 goroutine
+273 heap
+0 mutex
+0 profile
+29 threadcreate
+0 trace
+full goroutine stack dump
+Profile Descriptions:
+
+allocs: A sampling of all past memory allocations
+block: Stack traces that led to blocking on synchronization primitives
+cmdline: The command line invocation of the current program
+goroutine: Stack traces of all current goroutines
+heap: A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
+mutex: Stack traces of holders of contended mutexes
+profile: CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.
+threadcreate: Stack traces that led to the creation of new OS threads
+trace: A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.
+```
+
+## Heap
+
+**URL** : `/debug/pprof/heap`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Thread create
+
+**URL** : `/debug/pprof/threadcreate`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Goroutine
+
+**URL** : `/debug/pprof/goroutine`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Allocations
+
+**URL** : `/debug/pprof/allocs`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Block
+
+**URL** : `/debug/pprof/block`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Mutex
+
+**URL** : `/debug/pprof/mutex`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Cmdline
+
+**URL** : `/debug/pprof/cmdline`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Profile
+
+**URL** : `/debug/pprof/profile`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Symbol
+
+**URL** : `/debug/pprof/symbol`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Trace
+
+**URL** : `/debug/pprof/trace`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
diff --git a/versioned_docs/version-0.12.0/assets/application-state.png b/versioned_docs/version-0.12.0/assets/application-state.png
new file mode 100644
index 0000000..6c7f27e
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/application-state.png differ
diff --git a/versioned_docs/version-0.12.0/assets/architecture.png b/versioned_docs/version-0.12.0/assets/architecture.png
new file mode 100644
index 0000000..a19dcaa
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/architecture.png differ
diff --git a/versioned_docs/version-0.12.0/assets/cpu_profile.jpg b/versioned_docs/version-0.12.0/assets/cpu_profile.jpg
new file mode 100644
index 0000000..7e99f62
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/cpu_profile.jpg differ
diff --git a/versioned_docs/version-0.12.0/assets/dashboard_secret.png b/versioned_docs/version-0.12.0/assets/dashboard_secret.png
new file mode 100644
index 0000000..60b4f97
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/dashboard_secret.png differ
diff --git a/versioned_docs/version-0.12.0/assets/dashboard_token_select.png b/versioned_docs/version-0.12.0/assets/dashboard_token_select.png
new file mode 100644
index 0000000..59173fd
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/dashboard_token_select.png differ
diff --git a/versioned_docs/version-0.12.0/assets/docker-dektop-minikube.png b/versioned_docs/version-0.12.0/assets/docker-dektop-minikube.png
new file mode 100644
index 0000000..48b3584
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/docker-dektop-minikube.png differ
diff --git a/versioned_docs/version-0.12.0/assets/docker-desktop.png b/versioned_docs/version-0.12.0/assets/docker-desktop.png
new file mode 100644
index 0000000..9224360
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/docker-desktop.png differ
diff --git a/versioned_docs/version-0.12.0/assets/fifo-state-example.png b/versioned_docs/version-0.12.0/assets/fifo-state-example.png
new file mode 100644
index 0000000..ca04c17
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/fifo-state-example.png differ
diff --git a/versioned_docs/version-0.12.0/assets/gang_clean_up.png b/versioned_docs/version-0.12.0/assets/gang_clean_up.png
new file mode 100644
index 0000000..baf5acc
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/gang_clean_up.png differ
diff --git a/versioned_docs/version-0.12.0/assets/gang_generic_flow.png b/versioned_docs/version-0.12.0/assets/gang_generic_flow.png
new file mode 100644
index 0000000..381b7d0
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/gang_generic_flow.png differ
diff --git a/versioned_docs/version-0.12.0/assets/gang_scheduling_iintro.png b/versioned_docs/version-0.12.0/assets/gang_scheduling_iintro.png
new file mode 100644
index 0000000..b3be207
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/gang_scheduling_iintro.png differ
diff --git a/versioned_docs/version-0.12.0/assets/gang_timeout.png b/versioned_docs/version-0.12.0/assets/gang_timeout.png
new file mode 100644
index 0000000..a8ea0da
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/gang_timeout.png differ
diff --git a/versioned_docs/version-0.12.0/assets/gang_total_ask.png b/versioned_docs/version-0.12.0/assets/gang_total_ask.png
new file mode 100644
index 0000000..928d7fd
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/gang_total_ask.png differ
diff --git a/versioned_docs/version-0.12.0/assets/goland_debug.jpg b/versioned_docs/version-0.12.0/assets/goland_debug.jpg
new file mode 100644
index 0000000..c9ab94c
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/goland_debug.jpg differ
diff --git a/versioned_docs/version-0.12.0/assets/k8shim-application-state.png b/versioned_docs/version-0.12.0/assets/k8shim-application-state.png
new file mode 100644
index 0000000..c286591
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/k8shim-application-state.png differ
diff --git a/versioned_docs/version-0.12.0/assets/k8shim-node-state.png b/versioned_docs/version-0.12.0/assets/k8shim-node-state.png
new file mode 100644
index 0000000..5d5db56
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/k8shim-node-state.png differ
diff --git a/versioned_docs/version-0.12.0/assets/k8shim-scheduler-state.png b/versioned_docs/version-0.12.0/assets/k8shim-scheduler-state.png
new file mode 100644
index 0000000..2eb6e29
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/k8shim-scheduler-state.png differ
diff --git a/versioned_docs/version-0.12.0/assets/k8shim-task-state.png b/versioned_docs/version-0.12.0/assets/k8shim-task-state.png
new file mode 100644
index 0000000..11c95d1
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/k8shim-task-state.png differ
diff --git a/versioned_docs/version-0.12.0/assets/namespace-mapping.png b/versioned_docs/version-0.12.0/assets/namespace-mapping.png
new file mode 100644
index 0000000..9ad07da
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/namespace-mapping.png differ
diff --git a/versioned_docs/version-0.12.0/assets/node-bin-packing.png b/versioned_docs/version-0.12.0/assets/node-bin-packing.png
new file mode 100644
index 0000000..9267a03
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/node-bin-packing.png differ
diff --git a/versioned_docs/version-0.12.0/assets/node-fair.png b/versioned_docs/version-0.12.0/assets/node-fair.png
new file mode 100644
index 0000000..2e404ab
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/node-fair.png differ
diff --git a/versioned_docs/version-0.12.0/assets/object-state.png b/versioned_docs/version-0.12.0/assets/object-state.png
new file mode 100644
index 0000000..9baca07
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/object-state.png differ
diff --git a/versioned_docs/version-0.12.0/assets/pluggable-app-mgmt.jpg b/versioned_docs/version-0.12.0/assets/pluggable-app-mgmt.jpg
new file mode 100644
index 0000000..443b8ad
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/pluggable-app-mgmt.jpg differ
diff --git a/versioned_docs/version-0.12.0/assets/prometheus.png b/versioned_docs/version-0.12.0/assets/prometheus.png
new file mode 100644
index 0000000..964c79c
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/prometheus.png differ
diff --git a/versioned_docs/version-0.12.0/assets/queue-fairness.png b/versioned_docs/version-0.12.0/assets/queue-fairness.png
new file mode 100644
index 0000000..7c78ed7
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/queue-fairness.png differ
diff --git a/versioned_docs/version-0.12.0/assets/queue-resource-quotas.png b/versioned_docs/version-0.12.0/assets/queue-resource-quotas.png
new file mode 100644
index 0000000..fb72138
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/queue-resource-quotas.png differ
diff --git a/versioned_docs/version-0.12.0/assets/resilience-node-recovery.jpg b/versioned_docs/version-0.12.0/assets/resilience-node-recovery.jpg
new file mode 100644
index 0000000..3847451
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/resilience-node-recovery.jpg differ
diff --git a/versioned_docs/version-0.12.0/assets/resilience-workflow.jpg b/versioned_docs/version-0.12.0/assets/resilience-workflow.jpg
new file mode 100644
index 0000000..40ab6ba
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/resilience-workflow.jpg differ
diff --git a/versioned_docs/version-0.12.0/assets/spark-jobs-on-ui.png b/versioned_docs/version-0.12.0/assets/spark-jobs-on-ui.png
new file mode 100644
index 0000000..dabeb30
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/spark-jobs-on-ui.png differ
diff --git a/versioned_docs/version-0.12.0/assets/spark-pods.png b/versioned_docs/version-0.12.0/assets/spark-pods.png
new file mode 100644
index 0000000..e1f72e0
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/spark-pods.png differ
diff --git a/versioned_docs/version-0.12.0/assets/tf-job-on-ui.png b/versioned_docs/version-0.12.0/assets/tf-job-on-ui.png
new file mode 100644
index 0000000..06acabe
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/tf-job-on-ui.png differ
diff --git a/versioned_docs/version-0.12.0/assets/throughput.png b/versioned_docs/version-0.12.0/assets/throughput.png
new file mode 100644
index 0000000..8ced22c
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/throughput.png differ
diff --git a/versioned_docs/version-0.12.0/assets/yk-ui-screenshots.gif b/versioned_docs/version-0.12.0/assets/yk-ui-screenshots.gif
new file mode 100644
index 0000000..77dec56
Binary files /dev/null and b/versioned_docs/version-0.12.0/assets/yk-ui-screenshots.gif differ
diff --git a/versioned_docs/version-0.12.0/design/architecture.md b/versioned_docs/version-0.12.0/design/architecture.md
new file mode 100644
index 0000000..e7e2569
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/architecture.md
@@ -0,0 +1,62 @@
+---
+id: architecture
+title: Architecture
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Apache YuniKorn (Incubating) is a light-weight, universal resource scheduler for container orchestrator systems.
+It is created to achieve fine-grained resource sharing for various workloads efficiently on a large scale, multi-tenant,
+and cloud-native environment. YuniKorn brings a unified, cross-platform, scheduling experience for mixed workloads that
+consist of stateless batch workloads and stateful services.
+
+YuniKorn now supports K8s and can be deployed as a custom K8s scheduler. YuniKorn's architecture design also allows
+adding different shim layer and adopt to different ResourceManager implementation including Apache Hadoop YARN,
+or any other systems.
+
+## Architecture
+
+Following chart illustrates the high-level architecture of YuniKorn.
+
+<img src={require('./../assets/architecture.png').default} />
+
+## Components
+
+### Scheduler interface
+
+[Scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface) is an abstract layer
+which resource management platform (like YARN/K8s) will speak with, via API like GRPC/programing language bindings.
+
+### Scheduler core
+
+Scheduler core encapsulates all scheduling algorithms, it collects resources from underneath resource management
+platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
+best spot for each request and then sends response allocations to the resource management platform.
+Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface).
+Please read more about the design of schedule core [here](scheduler_core_design.md).
+
+### Kubernetes shim
+
+The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is responsible for translating the Kubernetes
+cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
+And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
+between the shim and the scheduler core is through the [scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface).
+Please read more about the design of the Kubernetes shim [here](k8shim.md)
+
diff --git a/versioned_docs/version-0.12.0/design/cache_removal.md b/versioned_docs/version-0.12.0/design/cache_removal.md
new file mode 100644
index 0000000..9e6c4d4
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/cache_removal.md
@@ -0,0 +1,446 @@
+---
+id: cache_removal
+title: Scheduler cache removal design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+# Proposal to combine Cache and Scheduler's implementation in the core
+This document describes the current state of the scheduler and cache implementation.
+It describes the changes planned based on the analysis that was done of the current behaviour.
+
+## Goals
+The goal is to provide the same functionality before and after the change.
+- unit tests before and after the merge must all pass.
+- Smoke tests defined in the core should all pass without major changes <sup id="s1">[definition](#f1)</sup>.
+- End-to-end tests that are part of the shim code must all pass without changes.
+
+## Background
+The current Scheduler Core is build up around two major components to store the data: the cache and scheduler objects.
+The cache objects form the base for most data to be tracked.
+The Scheduler objects track specific in flight details and are build on top of a cache object.
+
+The communication between the two layers uses a-synchronous events and in some cases direct updates.
+A synchronous update between the scheduler and the cache does mean that there is a short period the scheduler is "out of sync" with the cache.
+This short period can have an impact on the scheduling decisions.
+One of which is logged as [YUNIKORN-169](https://issues.apache.org/jira/browse/YUNIKORN-169).
+
+A further point is the complexity that the two structure brings to the code.
+A distinct set of messages to communicate between the scheduler and the cache.
+A one on one mapping between the scheduler and cache objects shows that the distinction is probably more artificial than required.
+---
+<b id="f1"></b>definition: Major changes for smoke tests are defined as changes to the tests that alter use case and thus test flows. Some changes will be needed as checks made could rely on cache objects which have been removed. [↩](#s1)
+## Structure analysis
+### Objects
+The existing objects as per the code analysis.
+The overlap between the scheduler and the cache objects is shown by showing them at the same line.
+N/A means that there is no equivalent object in either the scheduler or cache.
+
+| Cache Object | Scheduler Object |
+| ------------------------------ | ------------------------------ |
+| ClusterInfo | ClusterSchedulingContext |
+| PartitionInfo | partitionSchedulingContext |
+| AllocationInfo | schedulingAllocation |
+| N/A | schedulingAllocationAsk |
+| N/A | reservation |
+| ApplicationInfo | SchedulingApplication |
+| applicationState | N/A |
+| NodeInfo | SchedulingNode |
+| QueueInfo | SchedulingQueue |
+| SchedulingObjectState | N/A |
+
+The `initializer` code that is part of the cache does not define a specific object.
+It contains a mixture of code defined at the package level and code that is part of the `ClusterInfo` object.
+
+### Events
+Events defined in the core have multiple origins and destinations.
+Some events are only internal for the core between the cache and scheduler.
+These events will be removed.
+
+| Event | Flow | Proposal |
+| ----------------------------------------- | --------------------- | -------- |
+| AllocationProposalBundleEvent | Scheduler -> Cache | Remove |
+| RejectedNewApplicationEvent | Scheduler -> Cache | Remove |
+| ReleaseAllocationsEvent | Scheduler -> Cache | Remove |
+| RemoveRMPartitionsEvent | Scheduler -> Cache | Remove |
+| RemovedApplicationEvent | Scheduler -> Cache | Remove |
+| SchedulerNodeEvent | Cache -> Scheduler | Remove |
+| SchedulerAllocationUpdatesEvent | Cache -> Scheduler | Remove |
+| SchedulerApplicationsUpdateEvent | Cache -> Scheduler | Remove |
+| SchedulerUpdatePartitionsConfigEvent | Cache -> Scheduler | Remove |
+| SchedulerDeletePartitionsConfigEvent | Cache -> Scheduler | Remove |
+| RMApplicationUpdateEvent (add/remove app) | Cache/Scheduler -> RM | Modify |
+| RMRejectedAllocationAskEvent | Cache/Scheduler -> RM | Modify |
+| RemoveRMPartitionsEvent | RM -> Scheduler | |
+| RMUpdateRequestEvent | RM -> Cache | Modify |
+| RegisterRMEvent | RM -> Cache | Modify |
+| ConfigUpdateRMEvent | RM -> Cache | Modify |
+| RMNewAllocationsEvent | Cache -> RM | Modify |
+| RMReleaseAllocationEvent | Cache -> RM | Modify |
+| RMNodeUpdateEvent | Cache -> RM | Modify |
+| | | |
+
+Events that are handled by the cache will need to be handled by the core code after the removal of the cache.
+Two events are handled by the cache and the scheduler.
+
+## Detailed flow analysis
+### Object existing in both cache and scheduler
+The current design is based on the fact that the cache object is the basis for all data storage.
+Each cache object must have a corresponding scheduler object.
+The contract in the core around the cache and scheduler objects was simple.
+If the object exists in both scheduler and cache the object will be added to cache triggering the creation of the corresponding scheduler object.
+Removing the object is always handled in reverse: first from the scheduler which will trigger the removal from the cache.
+An example would be the creation of an application triggered by the `RMUpdateRequestEvent` would be processed by the cache.
+Creating a `SchedulerApplicationsUpdateEvent` to create the corresponding application in the scheduler.
+
+When the application and object state were added they were added into the cache objects.
+The cache objects were considered the data store and thus also contain the state.
+There were no corresponding state objects in the scheduler.
+Maintaining two states for the same object is not possible.
+
+The other exceptions to that rule are two objects that were considered volatile and scheduler only.
+The `schedulingAllocationAsk` tracks outstanding requests for an application in the scheduler.
+The `reservation` tracks a temporary reservation of a node for an application and ask combination.
+
+### Operations to add/remove app
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request an application to be added or removed.
+
+**application add or delete**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processApplicationUpdateFromRMUpdate
+ 2.1: Add new apps to the partition.
+ 2.2: Send removed apps to scheduler (but not remove anything from cache)
+3. scheduler.processApplicationUpdateEvent
+ 3.1: Add new apps to scheduler
+ (when fails, send RejectedNewApplicationEvent to cache)
+ No matter if failed or not, send RMApplicationUpdateEvent to RM.
+ 3.2: Remove app from scheduler
+ Send RemovedApplicationEvent to cache
+```
+
+### Operations to remove allocations and add or remove asks
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request an allocation to be removed.
+The RM can request an ask to be added or removed
+
+**allocation delete**
+This describes the allocation delete initiated by the RM only
+````
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: (by-pass): Send to scheduler via event SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: Update ReconcilePlugin
+ 3.2: Send confirmation of the releases back to Cache via event ReleaseAllocationsEvent
+4. cluster_info.processAllocationReleases to process the confirmed release
+````
+
+**ask add**
+If the ask already exists this add is automatically converted into an update.
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: Ask sanity check (such as existence of partition/app), rejections are send back to the RM via RMRejectedAllocationAskEvent
+ 2.2: pass checked asks to scheduler via SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: Update scheduling application with the new or updated ask.
+ 3.2: rejections are send back to the RM via RMRejectedAllocationAskEvent
+ 3.3: accepted asks are not confirmed to RM or cache
+```
+
+**ask delete**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: (by-pass): Send to scheduler via event SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationReleaseByAllocationKey
+ 3.1: Update scheduling application and remove the ask.
+```
+
+### Operations to add, update or remove nodes
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request a node to be added, updated or removed.
+
+**node add**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewSchedulableNodes
+ 2.1: node sanity check (such as existence of partition/node)
+ 2.2: Add new nodes to the partition.
+ 2.3: notify scheduler of new node via SchedulerNodeEvent
+3. notify RM of node additions and rejections via RMNodeUpdateEvent
+ 3.1: notify the scheduler of allocations to recover via SchedulerAllocationUpdatesEvent
+4. scheduler.processAllocationUpdateEvent
+ 4.1: scheduler creates a new ask based on the Allocation to recover
+ 4.2: recover the allocation on the new node using a special process
+ 4.3: confirm the allocation in the scheduler, on failure update the cache with a ReleaseAllocationsEvent
+```
+
+**node update and removal**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNodeActions
+ 2.1: node sanity check (such as existence of partition/node)
+ 2.2: Node info update (resource change)
+ 2.2.1: update node in cache
+ 2.2.2: notify scheduler of the node update via SchedulerNodeEvent
+ 2.3: Node status update (not removal), update node status in cache only
+ 2.4: Node removal
+ 2.4.1: update node status and remove node from the cache
+ 2.4.2: remove alloations and inform RM via RMReleaseAllocationEvent
+ 2.4.3: notify scheduler of the node removal via SchedulerNodeEvent
+3. scheduler.processNodeEvent add/remove/update the node
+```
+
+### Operations to add, update or remove partitions
+**Add RM**
+```
+1. RMProxy sends commonevents.RemoveRMPartitionsEvent
+ if RM is already registered
+ 1.1: scheduler.removePartitionsBelongToRM
+ 1.1.1: scheduler cleans up
+ 1.1.2: scheduler sends commonevents.RemoveRMPartitionsEvent
+ 1.2: cluster_info.processRemoveRMPartitionsEvent
+ 1.2.1: cache cleans up
+2. RMProxy sends commonevents.RegisterRMEvent
+3. cluster_info.processRMRegistrationEvent
+ 2.1: cache update internal partitions/queues accordingly.
+ 2.2: cache sends to scheduler SchedulerUpdatePartitionsConfigEvent.
+3. scheduler.processUpdatePartitionConfigsEvent
+ 3.1: Scheduler update partition/queue info accordingly.
+```
+
+**Update and Remove partition**
+Triggered by a configuration file update.
+```
+1. RMProxy sends commonevents.ConfigUpdateRMEvent
+2. cluster_info.processRMConfigUpdateEvent
+ 2.1: cache update internal partitions/queues accordingly.
+ 2.2: cache sends to scheduler SchedulerUpdatePartitionsConfigEvent.
+ 2.3: cache marks partitions for deletion (not removed yet).
+ 2.4: cache sends to scheduler SchedulerDeletePartitionsConfigEvent
+3. scheduler.processUpdatePartitionConfigsEvent
+ 3.1: scheduler updates internal partitions/queues accordingly.
+4. scheduler.processDeletePartitionConfigsEvent
+ 4.1: Scheduler set partitionManager.stop = true.
+ 4.2: PartitionManager removes queues, applications, nodes async.
+ This is the REAL CLEANUP including the cache
+```
+
+### Allocations
+Allocations are initiated by the scheduling process.
+The scheduler creates a SchedulingAllocation on the scheduler side which then gets wrapped in an AllocationProposal.
+The scheduler has checked resources etc already and marked the allocation as inflight.
+This description picks up at the point the allocation will be confirmed and finalised.
+
+**New allocation**
+```
+1. Scheduler wraps an SchedulingAllocation in an AllocationProposalBundleEvent
+2. cluster_info.processAllocationProposalEvent
+ preemption case: release preempted allocations
+ 2.1: release the allocation in the cache
+ 2.2: inform the scheduler the allocation is released via SchedulerNodeEvent
+ 2.3: inform the RM the allocation is released via RMReleaseAllocationEvent
+ all cases: add the new allocation
+ 2.4: add the new allocation to the cache
+ 2.5: rejections are send back to the scheduler via SchedulerAllocationUpdatesEvent
+ 2.6: inform the scheduler the allocation is added via SchedulerAllocationUpdatesEvent
+ 2.7: inform the RM the allocation is added via RMNewAllocationsEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: confirmations are added to the scheduler and change from inflight to confirmed.
+ On failure of processing a ReleaseAllocationsEvent is send to the cache *again* to clean up.
+ This is part of the issue in [YUNIKORN-169]
+ cluster_info.processAllocationReleases
+ 3.2: rejections remove the inflight allocation from the scheduler.
+```
+
+## Current locking
+**Cluster Lock:**
+A cluster contains one or more Partition objects. A partition is a sub object of Cluster.
+Adding or Removing ANY Partition requires a write-lock of the cluster.
+Retrieving any object within the cluster will require iterating over the Partition list and thus a read-lock of the cluster
+
+**Partition Lock:**
+The partition object contains all links to Queue, Application or Node objects.
+Adding or Removing ANY Queue, Application or Node needs a write-lock of the partition.
+Retrieving any object within the partition will require a read-lock of the partition to prevent data races
+
+Examples of operation needing a write-lock
+- Allocation processing after scheduling, will change application, queue and node objects.
+ Partition lock is required due to possible updates to reservations.
+- Update of Node Resource
+ It not only affect node's available resource, it also affects the Partition's total allocatable Resource
+
+Example of operations that need a read-lock:
+- Retrieving any Queue, Application or Node needs a read-lock
+ The object itself is not locked as part of the retrieval
+- Confirming an allocation after processing in the cache
+ The partition is only locked for reading to allow retrieval of the objects that will be changed.
+ The changes are made on the underlying objects.
+
+Example of operations that do not need any lock:
+- Scheduling
+ Locks are taken on the specific objects when needed, no direct updates to the partition until the allocation is confirmed.
+
+**Queue lock:**
+A queue can track either applications (leaf type) or other queues (parent type).
+Resources are tracked for both types in the same way.
+
+Adding or removing an Application (leaf type), or a direct child queue (parent type) requires a write-lock of the queue.
+Updating tracked resources requires a write-lock.
+Changes are made recursively never locking more than 1 queue at a time.
+Updating any configuration property on the queue requires a write-lock.
+Retrieving any configuration value, or tracked resource, application or queue requires a read-lock.
+
+Examples of operation needing a write-lock
+- Adding an application to a leaf queue
+- Updating the reservations
+
+Examples of operation needing a read-lock
+- Retrieving an application from a leaf type queue
+- Retrieving the pending resources
+
+**Application lock:**
+An application tracks resources of different types, the allocations and outstanding requests.
+Updating any tracked resources, allocations or requests requires a write-lock.
+Retrieving any of those values requires a read-lock.
+
+Scheduling also requires a write-lock of the application.
+During scheduling the write-lock is held for the application.
+Locks will be taken on the node or queue that need to be accessed or updated.
+Examples of the locks taken on other objects are:
+- a read lock to access queue tracked resources
+- a write-lock to update the in progress allocations on the node
+
+Examples of operation needing a write-lock
+- Adding a new ask
+- Trying to schedule a pending request
+
+Examples of operation needing a read-lock
+- Retrieving the allocated resources
+- Retrieving the pending requests
+
+**Node lock:**
+An node tracks resources of different types and allocations.
+Updating any tracked resources or allocations requires a write-lock.
+Retrieving any of those values requires a read-lock.
+
+Checks run during the allocation phases take locks as required.
+Read-locks when checking write-locks when updating.
+A node is not locked for the whole allocation cycle.
+
+Examples of operation needing a write-lock
+- Adding a new allocation
+- updating the node resources
+
+Examples of operation needing a read-lock
+- Retrieving the allocated resources
+- Retrieving the reservation status
+
+## How to merge Cache and scheduler objects
+Since there is no longer the requirement to distinguish the objects in the cache and scheduler the `scheduling` and `info` parts of the name will be dropped.
+
+Overview of the main moves and merges:
+1. `application_info` & `scheduling_application`: **merge** to `scheduler.object.application`
+2. `allocation_info` & `scheduling_allocation`: **merge** to `scheduler.object.allocation`
+3. `node_info` & `scheduling_node`: **merge** to `scheduler.object.node`
+4. `queue_info` & `scheduling_queue`: **merge** to `scheduler.object.queue`
+5. `partition_info` & `scheduling_partition`: **merge** to `scheduler.PartitionContext`
+6. `cluster_info` & `scheduling_context`: **merge** to `scheduler.ClusterContext`
+7. `application_state`: **move** to `scheduler.object.applicationState`
+8. `object_state`: **move** to `scheduler.object.objectState`
+9. `initializer`: **merge** into `scheduler.ClusterContext`
+
+This move and merge of code includes a refactor of the objects into their own package.
+That thus affects the two scheduler only objects, reservations and schedulingAsk, that are already defined.
+Both will be moved into the objects package.
+
+The top level scheduler package remains for the contexts and scheduling code.
+
+## Code merges
+The first change is the event processing.
+All RM events will now directly be handled in the scheduler.
+Event handling will undergo a major change, far more than a simple merge.
+Only the RM generated events will be left after the merge.
+As described in the analysis above the scheduler is, in almost all cases, notified of changes from RM events.
+
+Broadly speaking there are only three types of changes triggered by the event removal:
+- configuration changes: new scheduler code required as the cache handling is not transferable to the scheduler
+- node, ask and application changes: merge of the cache code into the scheduler
+- allocation changes: removal of confirmation cycle and simplification of the scheduler code
+
+Part of the event handling is the processing of the configuration changes.
+All configuration changes will now update the scheduler objects directly.
+The way the scheduler works is slightly different from the cache which means the code is not transferable.
+
+Nodes and applications are really split between the cache and scheduler.
+Anything that is tracked in the cache object that does not have an equivalent value in the scheduler object will be moved into the scheduler object.
+All references to scheduler objects will be removed.
+With the code merges existing scheduler code that calls out directly into the cache objects will return the newly tracked value in the scheduler object.
+These calls will thus become locked calls in the scheduler.
+
+The concept of an in flight allocation will be removed.
+Allocation will be made in the same scheduling iteration without events or creation of a proposal.
+Removing the need for tracking of allocating resources on the scheduler objects.
+In flight resource tracking was required to make sure that an allocation while not confirmed by the cache would being taken into account while making scheduling decisions.
+
+The application and object state will be an integrated part of the scheduler object.
+A state change is thus immediate and this should prevent an issue like [YUNIKORN-169](https://issues.apache.org/jira/browse/YUNIKORN-169) from occuring.
+
+## Locking after merge
+
+### Direction of lock
+It is possible to acquire another lock while holding a lock, but we need to make sure that we do not allow:
+- Holding A.lock and acquire B's lock.
+- Holding B.lock and acquire B's lock.
+
+The current code in the scheduler takes a lock as late as possible and only for the time period needed.
+Some actions are not locked on the scheduler side just on the cache side as each object has its own lock.
+This means that a read of a value from the cache would not lock the scheduling object.
+
+With the integration of the cache into the scheduler the number of locks will decrease as the number of objects decreases.
+Each equivalent object, cache and scheduler, which used to have their own lock will now have just one.
+After the merge of the code is performed one lock will be left.
+Locking will occur more frequently as the number of fields in the scheduler objects has increased.
+
+Calls that did not lock the scheduler object before the merge will become locked.
+Lock contention could lead to performance degradation.
+The reduced overhead in objects and event handling can hopefully compensate for this.
+One point to keep track of is the change in locking behaviour.
+New behaviour could lead to new deadlock situations when code is simply merged without looking at the order.
+
+### Mitigations for deadlocks
+The locking inside the scheduler will be left as is.
+This means that the main scheduling logic will be taking and releasing locks as required on the objects.
+There are no long held read-locks or write-locks until the application is locked to schedule it.
+
+A major point of attention will need to be that no iterations of objects should be performed while holding on to a lock.
+For instance during scheduling while iterating over a queue's application we should not lock the queue.
+
+Another example would be that event processing in the partition should not lock the partition unneeded.
+The partition should be locked while retrieving for instance the node that needs updating and release the lock before it tries to lock the node itself.
+
+This approach fits in with the current locking approach and will keep the locking changes to a minimum.
+Testing, specifically end-to-end testing, should catch these deadlocks.
+There are no known tools that could be used to detect or describe lock order.
diff --git a/versioned_docs/version-0.12.0/design/cross_queue_preemption.md b/versioned_docs/version-0.12.0/design/cross_queue_preemption.md
new file mode 100644
index 0000000..51c8033
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/cross_queue_preemption.md
@@ -0,0 +1,126 @@
+---
+id: cross_queue_preemption
+title: Cross Queue Preemption
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Problems:
+
+According to lessons we learned from YARN Scheduler preemption.
+
+**Here're top bad things:**
+
+- Preemption is a shotgun instead of a sniper, when a preemption decision is made, nobody knows if preempted resources will go to demanding queue/app/user or not.
+- Preemption logic and allocation is different, we have to implement (and mimic) what we have done in scheduler allocation logic.
+
+**Here're top good things:**
+
+- Preemption is fast (thanks to the shotgun), reclaiming thousands of containers only takes ~ 1 sec.
+- We have understand how painful it is to handle DRF, multiple preemption policies (inter/intra-queue, shotgun/surgical preemption, etc.) And we have developed some good logic
+to make sure better modularization and plug-ability
+
+## Answer some questions for design/implementation choices
+
+**1\. Do we really want preemption-delay? (Or we just want to control pace)**
+
+In CS, we have preemption-delay, which select victims in preemption candidates, wait for a certain time before killing it.
+
+The purposes of preemption delay are: a. give heads-up time to apps so
+they can prepare bad things happen (unfortunately no app do anything for these heads up, at least from what I knew). b. control preemption pace.
+
+And in practice, I found it causes a lot of issues, for example when a
+cluster state keep changing, it is very hard to ensure accurate preemption.
+
+**Proposal:**
+
+Remove the preemption-delay, keep the logics of controlling preemption pace. (such as ```yarn.resourcemanager.monitor.capacity.preemption
+.total_preemption_per_round```). And we can do allocation together with preemption.
+This don't mean containers will be stopped immediately after preemption issued. Instead, RM can control delays between signal a container and kill a container. Such as grace
+termination of POD in K8s: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
+
+**2\. Do we want to do preemption for every scheduling logic, or we can do periodically?**
+
+In CS, we have preemption logic runs periodically, like every 1 sec or 3 sec.
+
+Since preemption logic involves some heavy logics, like calculating shares of queues/apps. And when doing accurate preemption, we may need to scan nodes for preemption candidate.
+Considering this, I propose to have preemption runs periodically. But it is important to note that, we need to try to use as much code as possible for
+allocation-inside-preemption, otherwise there will be too much duplicated logic and very hard to be maintained in the future.
+
+**3\. Preemption cost and function**
+
+We found it is helpful to add cost for preemption, such as container live time, priority, type of container. It could be a cost function (Which returns a numeric value) or it
+could be a comparator (which compare two allocations for preemption ask).
+
+## Pseudo code
+
+Logic of allocation (invoked every allocation cycle)
+
+```
+input:
+ - nAlloc, allocate N allocations for this allocation cycle.
+
+for partition:
+ askCandidates := findAskCandidates(nAlloc, preemption=false)
+
+ allocated, failed_to_allocated := tryAllocate(askCandidates);
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ nAlloc -= len(allocated)
+```
+
+Logic of preemption (invoked every preemption cycle)
+
+```
+// It has to be done for every preemption-policy because calculation is different.
+for preemption-policy:
+ preempt_results := policy.preempt()
+ for preempt_results:
+ send-preempt-result-to-cache-to-commit;
+ updated-missed-opportunity (allocated)
+```
+
+Inside preemption policy
+
+```
+inter-queue-preempt-policy:
+ calculate-preemption-quotas;
+
+ for partitions:
+ total_preempted := resource(0);
+
+ while total_preempted < partition-limited:
+ // queues will be sorted by allocating - preempting
+ // And ignore any key in preemption_mask
+ askCandidates := findAskCandidates(N, preemption=true)
+
+ preempt_results := tryAllocate(askCandidates, preemption=true);
+
+ total_preempted += sigma(preempt_result.allocResource)
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ update-preemption-mask(askCandidates.allocKeys - preempt_results.allocKeys)
+```
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/design/gang_scheduling.md b/versioned_docs/version-0.12.0/design/gang_scheduling.md
new file mode 100644
index 0000000..f475395
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/gang_scheduling.md
@@ -0,0 +1,605 @@
+---
+id: gang_scheduling
+title: Gang scheduling design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+# Gang Scheduling Implementation
+A new way of scheduling applications by taking into account the demand for resources the application expects it will generate over time.
+It guarantees the expected demand resources for the application by reserving the resources.
+
+There are two parts to this implementation:
+* Kubernetes Shim
+* Core and scheduling
+
+This document describes the implementation on the core side.
+
+## Document goals
+This document describes the following implementation design points:
+1. Define changes required for the shim to core communication (scheduler interface)
+2. Scheduler storage object changes
+3. Scheduler logic changes
+
+## Excluded design points
+Currently, the Kubernetes shim side implementation is not covered in this design document.
+
+Generalised preemption on the core side will _not_ be discussed in this design.
+
+## Generic flow
+The flow is triggered by a pod that is submitted which triggers the application creation.
+This first pod is in the case of a Spark application, the driver pod.
+In case the flow is triggered from the creation of an application CRD there will not be a first pod.
+This is however outside the core scheduling logic. From the core side there should be no difference between the two cases.
+More details are in the chapter on the [Scheduler logic changes](#scheduler-logic-changes).
+
+The flow of an application submitted. The numbers in the diagram correspond to the description below the diagram.
+
+
+
+Combined flow for the shim and core during startup of an application:
+* An application is submitted with TaskGroup(s) defined. (1)
+* The shim creates the application and passes the application to the core. (2)
+* The shim creates placeholder pods for each of the members of the TaskGroup(s) (3)
+* The pods are processed and passed to the core, as per the normal behaviour, as AllocationAsks for the application with the correct info set. (4)
+* The placeholder AllocationAsk’s are scheduled by the core as if they were normal AllocationAsk’s. (5)
+* All Allocations, even if they are the result of the placeholder AllocationAsks being allocated by the scheduler, are communicated back to the shim.
+* The original real pod is passed to the core as an AllocationAsk. (6)
+* After the real pod and all all the placeholder pods are scheduled the shim starts the real pod that triggered the application creation. (7)
+
+After the first, real, pod is started the following pods should all be handled in the same way (8):
+* A real pod is created on k8s.
+* The pod is processed and an AllocationAsk is created.
+* The scheduler processes the AllocationAsk (more detail below) and replaces a placeholder with the real allocation.
+
+## Application submit handling
+### Total placeholder size
+
+The application if requesting one or more TaskGroups should provide the total size of all the TaskGroup members it is going to request.
+The total resource size is required for the case that the application is scheduled in a queue with a resource limit set.
+
+The value is important for three cases:
+1. gang request is larger than the queue quota
+2. start of scheduling reservations
+3. resource pressure while scheduling reservations
+
+Further detail will be given below in [scheduling in queues with a quota set](#scheduling-in-queues-with-a-quota-set)
+
+The information passed on from the shim should be part of the AddApplicationRequest.
+Detailed information on the build up of the taskGroup(s), or the number of members are not relevant.
+The total resource requested by all taskGroup members is calculated using:
+
+
+
+This total placeholderAsk is added as an optional field to the AddApplicationRequest message.
+The calculation can be made by the shim based on the CRD or annotation provided in the pod description.
+
+If the placeholderAsk is larger than the queue quota set on the queue the application must be rejected.
+This rejection is based on the fact that we cannot in any way honor the request.
+For all other cases the application is accepted and will be scheduled as per normal.
+
+### Handling queue with a FAIR sort policy
+If an application is submitted to a queue that has a FAIR sort policy set it must be rejected.
+Queue sorting for the queue that the application runs in must be set to _FIFO_ or _StateAware_.
+
+Other queue policies cannot guarantee that there is only one _New_ application processed at a time.
+In the case of the _FAIR_ policy we could be allocating multiple _New_ applications at the same time making quota management impossible to enforce.
+The other side effect of using _FAIR_ as a policy could be that we get multiple applications with only a partial allocated guarantee.
+
+Auto-scaling can be triggered due to the fact that the core can not place the placeholders on any node.
+In case the queue would use the _FAIR_ sorting this could lead to other applications taking the scaled up nodes instead of the placeholders again breaking the gang.
+
+## Scheduling in queues with a quota set
+The main case already described above is handling a total placeholder request size that is larger than the quota set on the queue.
+When the application is submitted we can already assess that we cannot satisfy that requirement and reject the request.
+
+In the case that the total placeholder ask does fit in the queue we should not start scheduling until there are enough resources available in the queue to satisfy the total request.
+However this does not stop scheduling of other applications in the queue(s).
+Applications that are already running in the queue could ask for more resources.
+From an application perspective there is no limit set on the resource it can request.
+The gang defined on the application is a guaranteed number of resources, not a maximum number of resources the application can request.
+
+This is complicated by the fact that we have a queue hierarchy.
+There is the possibility that the quota is not set directly on the queue the application is running.
+It could be set on one of the parent queues.
+This case could become complex, and we need to make sure that we keep in mind that we could live lock the scheduling.
+
+In this first phase we should focus on the case that the gang resources requested are also the maximum number of resources the application will request.
+When we look at the queues we should focus on a single queue level with quotas.
+
+These two assumptions are correct for the spark use case without dynamic allocation using a dynamic mapping from a namespace to a queue.
+
+Furthermore, we assume that the quota set on the queue can be totally allocated.
+If the cluster does not have enough resources the cluster will scale up to the size needed to provide all queues with their full quota.
+
+The follow up should add further enhancements for deeper hierarchies and dynamic allocation support.
+This could also leverage preemption in certain use cases, like preempting allocations from applications over their total gang size.
+
+Further enhancements could be added by allowing specifying the time and application will wait for the placeholders to be allocated, or the time to start using the held resources.
+
+## Scheduler logic changes
+The scheduler logic change needs to account for two parts of cycle:
+* The placeholder asks and their allocation.
+* The allocation replacing the placeholder.
+
+The basic assumption is that all pods will generate a placeholder pod request to the core.
+This includes the pod that triggered the application creation if we do not use the application CRD.
+This assumption is needed to make sure that the scheduler core can behave in the same way for both ways of submitting the application.
+The placeholder pods must be communicated to the core before the real pod.
+
+Changes for the placeholder AllocationAsks are the first step.
+As part of the creation of the application the AllocationAsks get added.
+The addition of an AllocationsAsk normally will trigger the application state change as per the scheduling cycle.
+It moves the Application from a _New_ state to an _Accepted_ state. This is as per the current setup, and does not change.
+
+However, in the case that the AllocationAsk has the _placeholder_ flag set the allocation should not trigger a state change, the application stays in _Accepted_ state.
+AllocationAsks are processed until the application has no pending resources.
+AllocationAsks that do not have a _placeholder_ flag set should be ignored as a safety precaution.
+All resulting Allocations for the placeholder pods are confirmed to the shim as per the normal steps.
+This process continues until there are no more placeholder pods to be allocated.
+
+The shim at that point should create the AllocationAsk for the real pod(s) that it has buffered.
+The core cannot and must not assume that there is only one task group per application.
+The core is also not in the position to assume that it has received all AllocationAsks that belong to the task group if option 1 as described above is used by a shim.
+This is also why we have the assumption that every pod creates a placeholder request to the core.
+
+The second change is the replacement of the placeholder pods with the real pods.
+The shim creates an AllocationAsk with the _taskGroupName_ set but the _placeholder_ flag is not set.
+
+The process described here lines up with the process for generic pre-emption.
+An allocation is released by the core and then confirmed by the shim.
+For gang scheduling we have a simple one new to one release relation in the case of pre-emption we can use the same flow with a one new to multiple release relation.
+
+The scheduler processes the AllocationAsk as follows:
+1. Check if the application has an unreleased allocation for a placeholder allocation with the same _taskGroupName._ If no placeholder allocations are found a normal allocation cycle will be used to allocate the request.
+2. A placeholder allocation is selected and marked for release. A request to release the placeholder allocation is communicated to the shim. This must be an async process as the shim release process is dependent on the underlying K8s response which might not be instantaneous.
+ NOTE: no allocations are released in the core at this point in time.
+3. The core “parks” the processing of the real AllocationAsk until the shim has responded with a confirmation that the placeholder allocation has been released.
+ NOTE: locks are released to allow scheduling to continue
+4. After the confirmation of the release is received from the shim the “parked” AllocationAsk processing is finalised.
+5. The AllocationAsk is allocated on the same node as the placeholder used.
+ The removal of the placeholder allocation is finalised in either case. This all needs to happen as one update to the application, queue and node.
+ * On success: a new Allocation is created.
+ * On Failure: try to allocate on a different node, if that fails the AllocationAsk becomes unschedulable triggering scale up.
+6. Communicate the allocation back to the shim (if applicable, based on step 5)
+
+## Application completion
+Application completion has been a long standing issue.
+Currently, applications do not transition to a _completed_ state when done.
+The current states for the application are [documented here](./scheduler_object_states.md).
+However, at this point in time an application will not reach the _completed_ state and will be stuck in _waiting_.
+
+This provides a number of issues specifically around memory usage and cleanup of queues in long running deployments.
+
+### Definition
+Since we cannot rely on the application, running as pods on Kubernetes, to show that it has finished we need to define when we consider an application _completed_.
+At this point we are defining that an application is _completed_ when it has been in the _waiting_ state for a defined time period.
+An application enters the waiting state at the time that there are no active allocations (allocated resources > 0) and pending allocation asks (pending resources > 0).
+
+The transition to a _waiting_ state is already implemented.
+The time out of the _waiting_ state is new functionality.
+
+Placeholders are not considered active allocations.
+Placeholder asks are considered pending resource asks.
+These cases will be handled in the [Cleanup](#Cleanup) below.
+
+### Cleanup
+When we look at gang scheduling there is a further issue around unused placeholders, placeholder asks and their cleanup.
+Placeholders could be converted into real allocations at any time there are pending allocation asks or active allocations.
+
+Placeholder asks will all be converted into placeholder allocations before the real allocations are processed.
+
+Entry into the _waiting_ state is already handled.
+If new allocation asks are added to the application it will transition back to a _running_ state.
+At the time we entered the waiting state. there were no pending requests or allocated resources.
+There could be allocated placeholders.
+
+For the entry into the _waiting_ state the application must be clean.
+However, we can not guarantee that all placeholders will be used by the application during the time the application runs.
+Transitioning out of the _waiting_ state into the _completed_ state requires no (placeholder) allocations or asks at all.
+The second case that impact transitions is that not all placeholder asks are allocated, and the application thus never requests any real allocations.
+These two cases could prevent an application from transitioning out of the _accepted_, or the _waiting_ state.
+
+Processing in the core thus needs to consider two cases that will impact the transition out of specific states:
+1. Placeholder asks pending (exit from _accepted_)
+2. Placeholders allocated (exit from _waiting_)
+
+Placeholder asks pending:
+Pending placeholder asks are handled via a timeout.
+An application must only spend a limited time waiting for all placeholders to be allocated.
+This timeout is needed because an application’s partial placeholders allocation may occupy cluster resources without really using them.
+
+An application could be queued for an unknown time, waiting for placeholder allocation to start.
+The timeout for placeholder asks can thus not be linked to the creation of the application or the asks.
+The timeout must start at the time the first placeholder ask is allocated.
+
+The application cannot request real allocations until all placeholder asks are allocated.
+A placeholder ask is also tracked by the shim as it represents a pod.
+Releasing an ask in the core requires a message to flow between the core and shim to release that ask.
+However, in this case the timeout for allocating placeholder asks triggers an application failure.
+When the timeout is triggered and placeholder asks are pending the application will transition from the state it is in, which can only be _accepted_, to _killed_.
+
+The application state for this case can be summarised as:
+* Application status is _accepted_
+* Placeholder allocated resource is larger than zero, and less than the _placeholderAsk_ from the _AddApplicationRequest_
+* Pending resource asks is larger than zero
+
+Entering into the _killed_ state must move the application out of the queue automatically.
+
+The state change and placeholder allocation releases can be handled in a single UpdateResponse message. The message will have the following content:
+* _UpdatedApplication_ for the state change of the application
+* one or more _AllocationRelease_ messages, one for each placeholder, with the _TerminationType_ set to TIMEOUT
+* one or more AllocationAskRelease messages with the _TerminationType_ set to TIMEOUT
+
+The shim processes the AllocationAskRelease messages first, followed by the _AllocationResponse_ messages, and finally the _UpdatedApplication_ message. The application state change to the _killed_ state on the core side is only dependent on the removal of all placeholders pods, not on a response to the _UpdatedApplication _message.
+
+
+
+Combined flow for the shim and core during timeout of placeholder:
+* The core times out the placeholder allocation. (1)
+* The placeholder Allocations removal is passed to the shim. (2)
+* All placeholder Allocations are released by the shim, and communicated back to the core.
+* The placeholder AllocationAsks removal is passed to the shim. (3)
+* All placeholder AllocationAsks are released by the shim, and communicated back to the core.
+* After the placeholder Allocations and Asks are released the core moves the application to the killed state removing it from the queue (4).
+* The state change is finalised in the core and shim. (5)
+
+Allocated placeholders:
+Leftover placeholders need to be released by the core.
+The shim needs to be informed to remove them. This must be triggered on entry of the _completed_ state.
+After the placeholder release is requested by the core the state transition of the application can proceed.
+The core will process the _AllocationRelease_ messages for placeholder allocations that come back from the shim with the _TerminationType_ set to TIMEOUT as normal without triggering a state change.
+
+The state change and placeholder allocation releases can be handled in a single UpdateResponse message.
+The message will have the following content:
+* _UpdatedApplication_ for the state change of the application
+* zero or more _AllocationRelease_ messages, one for each placeholder, with the _TerminationType_ set to TIMEOUT
+
+The shim processes the _AllocationResponse_ messages first followed by the _UpdatedApplication_ message.
+The application state change to the _completed_ state on the core side is only dependent on the removal of all placeholders pods, not on a response to the _UpdatedApplication _message.
+
+Entering into the _completed_ state will move the application out of the queue automatically.
+This should also handle the case we discussed earlier around a possible delayed processing of requests from the shim as we can move back from _waiting_ to _running_ if needed.
+A _completed_ application should also not prevent the case that was discussed around cron like submissions using the same application ID for each invocation.
+A _completed_ application with the same application ID must not prevent the submission of a new application with the same ID.
+
+
+
+Combined flow for the shim and core during cleanup of an application:
+* A pod is released at the Kubernetes layer. (1)
+* The shim passes the release of the allocation on to the core. (2)
+* The core transitions the application to a waiting state if no pending or active allocations. (3)
+* The waiting state times out and triggers the cleanup. (4)
+* The placeholder Allocations removal is passed to the shim. (5)
+* All placeholder Allocations are released by the shim, and communicated back to the core.
+* After all placeholders are released the core moves the application to the completed state removing it from the queue (6).
+* The state change is finalised in the core and shim. (7)
+
+## Application recovery
+During application recovery the placeholder pods are recovered as any other pod on a node.
+These pods are communicated to the core by the shim as part of the node as an existing allocation.
+Existing allocations do not have a corresponding _AllocationAsk_ in the core. The core generates an _AllocationAsk_ based on the recovered information.
+
+For gang scheduling the _AllocationAsk_ contains the _taskGroupName_ and _placeholder_ flag.
+During recovery that same information must be part of the _Allocation_ message.
+This is due to the fact that the same message is used in two directions, from the RM to the scheduler and vice versa means we need to update the message and its processing.
+
+If the information is missing from the _Allocation_ message the recovered allocation will not be correctly tagged in the core.
+The recovered allocation will be seen as a regular allocation.
+This means it is skipped as part of the normal allocation cycle that replaces the placeholders.
+
+The logic change only requires that the recovery of existing allocations copies the fields from the interface message into the allocation object in the core.
+
+## Interface changes
+Multiple changes are needed to the communication between the shim and the core to support the gang information needed.
+
+An application must provide the total size of the placeholder requests to prevent accepting an application that can never run.
+
+The current object that is sent from the shim to the core for allocation requests is defined in the AllocationAsk.
+The Allocation, as the result message passed back from the scheduler core does not change. For recovery, which uses the same Allocation message, from the shim to the core, however must contain the gang related fields.
+Gang related fields must be added to both messages.
+
+The allocation release request and response request need to support bidirectional traffic and will need to undergo major changes.
+
+### AddApplication
+The AddApplicationRequest message requires a new field to communicate the total placeholder resource request that will be requested.
+The field is used to reject the application if it is impossible to satisfy the request.
+It can also be used to stop the core from scheduling any real pods for that application until all placeholder pods are processed.
+
+In patched message form that would look like:
+```
+message AddApplicationRequest {
+...
+ // The total amount of resources gang placeholders will request
+ Resource placeholderAsk = 7;
+...
+}
+```
+
+### AllocationAsk
+The first part of the change is the base information for the task group.
+This will require an additional optional attribute to be added.
+The content of this optional attribute is a name, a string, which will be mapped to the name of the task group.
+The field can be present on a real allocation and on a placeholder.
+
+Proposed name for the new field is: _taskGroupName_
+
+To distinguish normal AllocationAsks and placeholder AllocationAsks a flag must be added.
+The flag will never have more than two values and thus maps to a boolean. As the default value for a boolean is _false_ the field should show the fact that it is an AllocationAsk that represents a placeholder as true.
+
+Proposed name for the field is: _placeholder_
+
+In patched message form that would look like:
+```
+message AllocationAsk {
+...
+ // The name of the TaskGroup this ask belongs to
+ string taskGroupName = 10;
+ // Is this a placeholder ask (true) or a real ask (false), defaults to false
+ // ignored if the taskGroupName is not set
+ bool placeholder = 11;
+...
+}
+```
+
+The last part of the task group information that needs to be communicated is the size of the task group.
+This does not require a change in the interface as the current AllocationAsk object can support both possible options.
+
+Requests can be handled in two ways:
+1. Each member of the task group is passed to the core as a separate AllocationAsk with a maxAllocations, or the ask repeat, of 1
+2. The task group is considered one AllocationAsk with a maxAllocations set to the same value as minMember of the task group information.
+
+With option A the shim will need to generate multiple AllocationAsk objects and pass each to the core for scheduling.
+Each AllocationAsk is linked to one pod.
+Option B will only generate one AllocationAsk for all placeholder pods.
+Option B requires less code and has less overhead on the core side.
+However the logic on the shim side might be more complex as the returned allocation needs to be linked to just one pod.
+
+Proposal is to use option: A
+
+### Allocation
+Similar to the change for the _AllocationAsk_ the _Allocation_ requires additional optional attributes to be added.
+The new fields distinguish a normal Allocation and placeholder Allocations on recovery.
+The same rules apply to these fields as the ones added to the _AllocationAsk_.
+
+The content of this optional attribute is a name, a string, which will be mapped to the name of the task group.
+The field can be present on a real allocation and on a placeholder.
+
+Proposed name for the new field is: _taskGroupName_
+
+The flag will never have more than two values and thus maps to a boolean.
+As the default value for a boolean is _false_ the field should show the fact that it is an Allocation that represents a placeholder as true.
+
+Proposed name for the field is: _placeholder_
+
+In patched message form that would look like:
+```
+message Allocation {
+...
+ // The name of the TaskGroup this allocation belongs to
+ string taskGroupName = 11;
+ // Is this a placeholder allocation (true) or a real allocation (false), defaults to false
+ // ignored if the taskGroupName is not set
+ bool placeholder = 12;
+...
+}
+```
+
+### AllocationRelease Response and Request
+The name for the messages are based on the fact that the release is always triggered by the shim.
+In case of preemption and or gang scheduling the release is not triggered from the shim but from the core.
+That means the message name does not cover the usage. A response message might not have an associated request message.
+It could be used to indicate direction but that is in this case confusing.
+
+When a release is triggered from the core, for preemption or the placeholder allocation, a response is expected from the shim to confirm that the release has been processed.
+This response must be distinguished from a request to release the allocation initiated by the shim.
+A release initiated by the shim must be followed by a confirmation from the core to the shim that the message is processed.
+For releases initiated by the core no such confirmation message can or must be sent.
+In the current request message there is no way to indicate that it is a confirmation message.
+
+To fix the possible confusing naming the proposal is to merge the two messages into one message: _AllocationRelease_.
+
+The _AllocationReleaseRequest_ is indirectly part of the _UpdateRequest_ message as it is contained in the _AllocationReleasesRequest_.
+The _AllocationReleaseResponse_ is part of the _UpdateResponse_ message.
+The flow-on effect of the rename and merge of the two messages is a change in the two messages that contain them.
+The message changes for _UpdateResponse_ and _AllocationReleasesRequest_ are limited to type changes of the existing fields.
+
+| Message | Field ID | Old type | New type |
+| ------------------------- | -------- | ------------------------- | ----------------- |
+| UpdateResponse | 3 | AllocationReleaseResponse | AllocationRelease |
+| AllocationReleasesRequest | 1 | AllocationReleaseRequest | AllocationRelease |
+
+In patched message form that would look like:
+```
+message UpdateResponse {
+...
+ // Released allocation(s), allocations can be released by either the RM or scheduler.
+ // The TerminationType defines which side needs to act and process the message.
+ repeated AllocationRelease releasedAllocations = 3;
+...
+}
+
+message AllocationReleasesRequest {
+ // Released allocation(s), allocations can be released by either the RM or scheduler.
+ // The TerminationType defines which side needs to act and process the message.
+ repeated AllocationRelease releasedAllocations = 1;
+...
+}
+```
+
+The merged message _AllocationRelease_ will consist of:
+
+| Field name | Content type | Required |
+| --------------- | ----------------- | -------- |
+| partitionName | string | yes |
+| applicationID | string | no |
+| UUID | string | no |
+| terminationType | _TerminationType_ | yes |
+| message | string | no |
+
+Confirmation behaviour of the action should be triggered on the type of termination received.
+The core will confirm the release to the shim of all types that originate in the shim and vice versa.
+
+A confirmation or response uses the same _TerminationType_ as was set in the original message.
+An example of this is a pod that is removed from K8s will trigger an _AllocationRelease _message to be sent from the shim to the core with the TerminationType STOPPED_BY_RM. The core processes the request removing the allocation from the internal structures, and when all processing is done it responds to the shim with a message using the same _TerminationType_.
+The shim can ignore that or make follow up changes if needed.
+
+A similar process happens for a release that originates in the core.
+Example of the core sending an _AllocationRelease_ message to the shim using the _TerminationType_ PREEMPTED_BY_SCHEDULER.
+The shim handles that by releasing the pod identified and responds to the core that it has released the pod.
+On receiving the confirmation that the pod has been released the core can progress with the allocation and preemption processing.
+
+In patched message form that would look like:
+```
+message AllocationRelease {
+ enum TerminationType {
+ STOPPED_BY_RM = 0;
+ TIMEOUT = 1;
+ PREEMPTED_BY_SCHEDULER = 2;
+ PLACEHOLDER_REPLACED = 3;
+ }
+
+ // The name of the partition the allocation belongs to
+ string partitionName = 1;
+ // The application the allocation belongs to
+ string applicationID = 2;
+ // The UUID of the allocation to release, if not set all allocations are released for
+ // the applicationID
+ string UUID = 3;
+ // The termination type as described above
+ TerminationType terminationType = 4;
+ // human-readable message
+ string message = 5;
+}
+```
+### TerminationType
+The currently defined _TerminationType_ values and specification of the side that generates (Sender), and the side that actions and confirms processing (Receiver):
+
+| Value | Sender | Receiver |
+| ------------------------ | ------ | -------- |
+| STOPPED_BY_RM | shim | core |
+| TIMEOUT * | core | shim |
+| PREEMPTED_BY_SCHEDULER * | core | shim |
+
+* currently not handled by the shim, core or both
+
+When the placeholder allocation gets released the _AllocationReleaseResponse_ is used to communicate the release back from the core to the shim.
+The response contains an enumeration called _TerminationType_, and a human-readable message.
+For tracking and tracing purposes we should add a new _TerminationType_ specifically for the placeholder replacement. The shim must take action based on the type and confirm the allocation release to the core.
+
+It should provide enough detail so we do not have to re-use an already existing type, or the human-readable message.
+The human-readable format can still be used to provide further detail on which new allocation replaced the placeholder.
+
+Proposal is to add: _PLACEHOLDER_REPLACED_
+
+| Value | Sender | Receiver |
+| ---------------------| ------ | -------- |
+| PLACEHOLDER_REPLACED | shim | core |
+
+As part of the Scheduler Interface cleanup ([YUNIKORN-486](https://issues.apache.org/jira/browse/YUNIKORN-486)) the _TerminationType_ should be extracted from the _AllocationRelease_ and _AllocationaskRelease_ message.
+It is an enumeration that can be shared between multiple objects.
+[YUNIKORN-547](https://issues.apache.org/jira/browse/YUNIKORN-547) has been logged to handle this as it has an impact on the code outside of the scope of gang scheduling.
+
+
+### AllocationAskRelease Response and Request
+The allocation ask release right now can only be triggered by the shim.
+In order for the core to perform the cleanup when the placeholder allocation times out, we need to make this a bidirectional message.
+Similarly to the Allocation we would rename the _AllocationAskReleaseRequest_ to _AllocationAskRelease_, so we can use this message in both directions:
+```
+message AllocationReleasesRequest {
+...
+ // Released allocationask(s), allocationasks can be released by either the RM or
+ // scheduler. The TerminationType defines which side needs to act and process the
+ // message.
+ repeated AllocationAskRelease allocationAsksToRelease = 2;
+}
+```
+
+Similar processing logic based on the _TerminationType_ which is used for allocations should be used for ask releases.
+In patched message form that would look like:
+```
+message AllocationAskRelease {
+ enum TerminationType {
+ STOPPED_BY_RM = 0;
+ TIMEOUT = 1;
+ PREEMPTED_BY_SCHEDULER = 2;
+ PLACEHOLDER_REPLACED = 3;
+ }
+...
+ // The termination type as described above
+ TerminationType terminationType = 4;
+...
+}
+```
+
+Confirmation behaviour of the action should be triggered on the type of termination received.
+The core will confirm the release to the shim of all types that originate in the shim and vice versa.
+
+A confirmation or response uses the same _TerminationType_ as was set in the original message.
+
+## Scheduler storage object changes
+### AllocationAsk
+In line with the changes for the communication the objects in the scheduler also need to be modified to persist some of the detail communicated.
+The AllocationAsk that is used in the communication has an equivalent object inside the scheduler with the same name.
+This object needs to be able to store the new fields proposed above.
+
+Proposed new fields: _taskGroupName_ and _placeholder_.
+
+In the current interface specification a field called _executionTimeoutMilliSeconds_ is defined.
+This is currently not mapped to the object inside the scheduler and should be added.
+Time or Duration are stored as native go objects and do not include a size specifier.
+
+Proposed new field: _execTimeout_
+
+### Allocation
+After the allocation is made an Allocation object is created in the core to track the real allocation. This Allocation object is directly linked to the application and should show that the allocation is a placeholder and for which task group. This detail is needed to also enable the correct display of the resources used in the web UI.
+
+The propagation of the placeholder information could be achieved indirectly as the allocation object references an AllocationAsk. This would require a lookup of the AllocationAsk to assess the type of allocation. We could also opt to propagate the data into the Allocation object itself. This would remove the lookup and allow us to directly filter allocations based on the type and or task group information.
+
+From a scheduling and scheduler logic perspective the indirect reference is not really desirable due to the overhead of the lookups required. This means that the same fields added in the AllocationAsk are also added to the Allocation object.
+
+Proposed new fields: _taskGroupName_ and _placeholder_.
+
+To support the release of the allocation being triggered from the core tracking of the release action is required. The release is not final until the shim has confirmed that release. However during that time period the allocation may not be released again.
+
+Proposed new field: _released_
+
+At the point that we replace the placeholder with a real allocation we need to release an existing placeholder.
+The Allocation object allows us to specify a list of Allocations to release.
+This field was added earlier to support preemption.
+This same field will be reused for the placeholder release.
+
+### Application
+The AddApplicationRequest has a new field added that needs to be persisted in the object inside the scheduler.
+
+Proposed new field: _placeholderAsk_
+
+In the current interface specification a field called _executionTimeoutMilliSeconds_ is defined. This is currently not mapped to the object inside the scheduler and should be added. Time or Duration are stored as native go objects and do not include a size specifier.
+
+Proposed new field: _execTimeout_
+
+The application object should be able to track the placeholder allocations separately from the real allocations. The split of the allocation types on the application will allow us to show the proper state in the web UI.
+
+Proposed new field: _allocatedPlaceholder_
+
+
+### Queue & Node
+No changes at this point.
+The placeholder allocations should be counted as “real” allocations on the Queue and Node.
+By counting the placeholder as normal the quota for the queue is enforced as expected.
+The Node object needs to also show normal usage to prevent interactions with the autoscaler.
diff --git a/versioned_docs/version-0.12.0/design/k8shim.md b/versioned_docs/version-0.12.0/design/k8shim.md
new file mode 100644
index 0000000..4e9e7e7
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/k8shim.md
@@ -0,0 +1,86 @@
+---
+id: k8shim
+title: Kubernetes Shim Design
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Github repo: https://github.com/apache/incubator-yunikorn-k8shim
+
+Please read the [architecture](architecture.md) doc before reading this one, you will need to understand
+the 3 layer design of YuniKorn before getting to understand what is the Kubernetes shim.
+
+## The Kubernetes shim
+
+The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is responsible for translating the Kubernetes
+cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
+And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
+between the shim and the scheduler core is through the scheduler-interface.
+
+## The admission controller
+
+The admission controller runs in a separate pod, it runs a
+[mutation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook)
+and a [validation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook), where:
+
+1. The `mutation webhook` mutates pod spec by:
+ - Adding `schedulerName: yunikorn`
+ - By explicitly specifying the scheduler name, the pod will be scheduled by YuniKorn scheduler.
+ - Adding `applicationId` label
+ - When a label `applicationId` exists, reuse the given applicationId.
+ - When a label `spark-app-selector` exists, reuse the given spark app ID.
+ - Otherwise, assign a generated application ID for this pod, using convention: `yunikorn-<namespace>-autogen`. This is unique per namespace.
+ - Adding `queue` label
+ - When a label `queue` exists, reuse the given queue name. Note, if placement rule is enabled, values set in the label is ignored.
+ - Otherwise, adds `queue: root.default`
+ - Adding `disableStateAware` label
+ - If pod was assigned a generated applicationId by the admission controller, also set `disableStateAware: true`. This causes the generated application
+ to immediately transition from the `Starting` to `Running` state so that it will not block other applications.
+2. The `validation webhook` validates the configuration set in the configmap
+ - This is used to prevent writing malformed configuration into the configmap.
+ - The validation webhook calls scheduler [validation REST API](api/scheduler.md#configuration-validation) to validate configmap updates.
+
+### Admission controller deployment
+
+Currently, the deployment of the admission-controller is done as a `post-start` hook in the scheduler deployment, similarly, the
+uninstall is done as a `pre-stop` hook. See the related code [here](https://github.com/apache/incubator-yunikorn-release/blob/56e580af24ed3433e7d73d9ea556b19ad7b74337/helm-charts/yunikorn/templates/deployment.yaml#L80-L85).
+During the installation, it is expected to always co-locate the admission controller with the scheduler pod, this is done
+by adding the pod-affinity in the admission-controller pod, like:
+
+```yaml
+podAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: component
+ operator: In
+ values:
+ - yunikorn-scheduler
+ topologyKey: "kubernetes.io/hostname"
+```
+
+it also tolerates all the taints in case the scheduler pod has some toleration set.
+
+```yaml
+tolerations:
+- operator: "Exists"
+```
+
+
diff --git a/versioned_docs/version-0.12.0/design/namespace_resource_quota.md b/versioned_docs/version-0.12.0/design/namespace_resource_quota.md
new file mode 100644
index 0000000..90830b6
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/namespace_resource_quota.md
@@ -0,0 +1,183 @@
+---
+id: namespace_resource_quota
+title: Namespace Resource Quota
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+In K8s, user can setup namespace with [resource quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) to limit aggregated resource consumption in this namespace. The validation of namespace resource quotas is handled in api-server directly, therefore YuniKorn simply honors the quotas like the default scheduler.
+
+## Best practice
+
+It is not mandatory to setup YuniKorn queues with respect of namespaces.
+However, in practice, it makes more sense to do so.
+Namespace is often used to set a cap for resource consumptions per user-group/team,
+YuniKorn queue is also meant to divide cluster resource into multiple groups.
+Let's go through an example.
+
+### 1. Setup namespace
+
+Namespace: `advertisement`:
+```
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+ name: advertisement
+spec:
+ hard:
+ requests.cpu: "200m"
+ requests.memory: 2000Mi
+ limits.cpu: "200m"
+ limits.memory: 4000Mi
+```
+Create the namespace
+```
+kubectl create namespace advertisement
+kubectl create -f ./advertisement.yaml --namespace=advertisement
+kubectl get quota --namespace=advertisement
+kubectl describe quota advertisement --namespace=advertisement
+
+// output
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 0 200m
+limits.memory 0 4000Mi
+requests.cpu 0 200m
+requests.memory 0 2000Mi
+```
+
+### 2. Setup YuniKorn queues
+
+Queue: `advertisement`:
+```
+name: advertisement
+resources:
+ guaranteed:
+ vcore: 100
+ memory: 1000
+ max:
+ vcore: 200
+ memory: 2000
+```
+
+ensure `QueueMaxResource <= NamespaceResourceQuotaRequests`
+
+### 3. Mapping applications to queues & namespace
+
+In a pod spec
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00001"
+ queue: "root.advertisement"
+ name: task0
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "800M"
+ limits:
+ cpu: "100m"
+ memory: "1000M"
+```
+
+Check Quota
+
+```
+kubectl describe quota advertisement --namespace=advertisement
+
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 100m 200m
+limits.memory 1G 4000Mi
+requests.cpu 50m 200m
+requests.memory 800M 2000Mi
+```
+
+Now submit another application,
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00002"
+ queue: "root.advertisement"
+ name: task1
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "200m"
+ memory: "800M"
+ limits:
+ cpu: "200m"
+ memory: "1000M"
+```
+
+pod will not be able to submitted to api-server, because the requested cpu `200m` + used cpu `100m` = `300m` which exceeds the resource quota.
+
+```
+kubectl create -f pod_ns_adv_task1.yaml
+Error from server (Forbidden): error when creating "pod_ns_adv_task1.yaml": pods "task1" is forbidden: exceeded quota: advertisement, requested: limits.cpu=200m,requests.cpu=200m, used: limits.cpu=100m,requests.cpu=50m, limited: limits.cpu=200m,requests.cpu=200m
+```
+
+## Future Work
+
+For compatibility, we should respect namespaces and resource quotas.
+Resource quota is overlapped with queue configuration in many ways,
+for example the `requests` quota is just like queue's max resource. However,
+there are still a few features resource quota can do but queue cannot, such as
+
+1. Resource `limits`. The aggregated resource from all pods in a namespace cannot exceed this limit.
+2. Storage Resource Quota, e.g storage size, PVC number, etc.
+3. Object Count Quotas, e.g count of PVCs, services, configmaps, etc.
+4. Resource Quota can map to priority class.
+
+Probably we can build something similar to cover (3) in this list.
+But it would be hard to completely support all these cases.
+
+But currently, setting applications mapping to a queue as well as a corresponding namespace is over complex.
+Some future improvements might be:
+
+1. Automatically detects namespaces in k8s-shim and map them to queues. Behind the scenes, we automatically generates queue configuration based on namespace definition. Generated queues are attached under root queue.
+2. When new namespace added/updated/removed, similarly to (1), we automatically update queues.
+3. User can add more configuration to queues, e.g add queue ACL, add child queues on the generated queues.
+4. Applications submitted to namespaces are transparently submitted to corresponding queues.
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/design/pluggable_app_management.md b/versioned_docs/version-0.12.0/design/pluggable_app_management.md
new file mode 100644
index 0000000..d297ada
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/pluggable_app_management.md
@@ -0,0 +1,75 @@
+---
+id: pluggable_app_management
+title: Pluggable App Management
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## The Problem
+
+Currently, we schedule and group an application is based on a label on the pod.
+This generic way works for any type of workload. It does however give us a limited information on the lifecycle
+and application. On the K8s side, operators have been introduced to provide more detail on the application
+and help scheduling. We cannot use them currently and want to add that functionality.
+
+## K8s Operator Pattern
+
+[K8s operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/)
+is a pattern in K8s to manage applications, it's a handy way to manage application's lifecycle out-of-box on K8s.
+You define several CRDs and some controllers to monitor and mutate the state of the application based on the CRD definition.
+
+For example in [spark-k8s-operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator),
+it defines a CRD called `SparkApplication`, the controller watches the events of add/update/delete of this CRD
+and trigger corresponding actions on event notifications. The `SparkApplication` looks like
+[this example](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/examples/spark-pi.yaml). There
+are a lot more popular operators, such as [flink-k8s-operator](https://github.com/GoogleCloudPlatform/flink-on-k8s-operator),
+ [tf-operator](https://github.com/kubeflow/tf-operator), [pytorch-operator](https://github.com/kubeflow/pytorch-operator), etc.
+
+Use Spark as an example. YuniKorn is able to schedule resources for all pods in K8s, that seamlessly supports Spark. It
+works with [native Spark on K8s](https://spark.apache.org/docs/latest/running-on-kubernetes.html), or
+[spark on K8s with operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/design.md#architecture),
+you'll find the difference from the design architecture chart from the given link. To support native Spark on K8s,
+YuniKorn reads pods' spec and group Spark pods by a label-selector, based on `spark-app-selector`.
+The operator approach gives us more context about the Spark job, such as a better understanding about job state.
+But all these info requires us to look at `SparkApplication` CRD, currently, there is no neat way to
+add such functionality. That's why we need to design a flexible approach to support 3rd party operators
+(retrieving info from their CRDs), so we can easily integrate with other operators with small effort.
+
+## Design
+
+The key issue here is we need a app-management interface, that can be easily extended.
+It needs to be decoupled with existing scheduling logic. For each operator, we create a service to manage this type app's lifecycle,
+and communicate with the scheduling cache independently. The high-level design looks like below:
+
+
+
+Where
+- `AppManagementService` is a composite set of services that can be managed together.
+- `AppManager` is a specific app management service for a particular type of application. In each service, it has
+ access to K8s clients, such as informers, listers, in order to monitor CRD events. And it collects necessary info
+ and talk with scheduler cache through `AMProtocol`.
+- `APIProvider` encapsulate a set of useful APIs that can be shared, such as kube-client, pod/node/storage informers, etc.
+ Each of such informers, it can be shared with multiple app managers, to avoid the overhead.
+- `AMProtocol` defines the basic interaction contract between app manager and the scheduler cache, that helps the cache
+ to performs app lifecycle management without understanding what type of the application it is.
+
+In the upon chart, the AppManagementService has 2 services, the _general_ one is managing normal applications, that
+recognizes applications by pod labels; the _spark-k8s-operator_ one watches `SparkApplication` CRD and manage jobs'
+lifecycle defined by this CRD.
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/design/predicates.md b/versioned_docs/version-0.12.0/design/predicates.md
new file mode 100644
index 0000000..f750cbd
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/predicates.md
@@ -0,0 +1,80 @@
+---
+id: predicates
+title: Support K8s Predicates
+---
+
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+## Design
+
+Predicates are a set of pre-registered functions in K8s, the scheduler invokes these functions to check if a pod
+is eligible to be allocated onto a node. Common predicates are: node-selector, pod affinity/anti-affinity etc. To support
+these predicates in YuniKorn, we don't intend to re-implement everything on our own, but to re-use the core predicates
+code as much as possible.
+
+YuniKorn-core is agnostic about underneath RMs, so the predicates functions are implemented in K8s-shim as a `SchedulerPlugin`.
+SchedulerPlugin is a way to plug/extend scheduler capabilities. Shim can implement such plugin and register itself to
+yunikorn-core, so plugged function can be invoked in the scheduler core. Find all supported plugins in
+[types](https://github.com/apache/incubator-yunikorn-core/blob/master/pkg/plugins/types.go).
+
+## Workflow
+
+First, RM needs to register itself to yunikorn-core, it advertises what scheduler plugin interfaces are supported.
+E.g a RM could implement `PredicatePlugin` interface and register itself to yunikorn-core. Then yunikorn-core will
+call PredicatePlugin API to run predicates before making allocation decisions.
+
+
+Following workflow demonstrates how allocation looks like when predicates are involved.
+
+```
+pending pods: A, B
+shim sends requests to core, including A, B
+core starts to schedule A, B
+ partition -> queue -> app -> request
+ schedule A (1)
+ run predicates (3)
+ generate predicates metadata (4)
+ run predicate functions one by one with the metadata
+ success
+ proposal: A->N
+ schedule B (2)
+ run predicates (calling shim API)
+ generate predicates metadata
+ run predicate functions one by one with the metadata
+ success
+ proposal: B->N
+commit the allocation proposal for A and notify k8s-shim
+commit the allocation proposal for B and notify k8s-shim
+shim binds pod A to N
+shim binds pod B to N
+```
+
+(1) and (2) are running in parallel.
+
+(3) yunikorn-core calls a `schedulerPlugin` API to run predicates, this API is implemented on k8s-shim side.
+
+(4) K8s-shim generates metadata based on current scheduler cache, the metadata includes some intermittent states about nodes and pods.
+
+## Predicates White-list
+
+Intentionally, we only support a white-list of predicates. Majorly due to 2 reasons,
+* Predicate functions are time-consuming, it has negative impact on scheduler performance. To support predicates that are only necessary can minimize the impact. This will be configurable via CLI options;
+* The implementation depends heavily on K8s default scheduler code, though we reused some unit tests, the coverage is still a problem. We'll continue to improve the coverage when adding new predicates.
+
+the white-list currently is defined in [DefaultSchedulerPolicy](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/pkg/plugin/predicates/predictor.go).
diff --git a/versioned_docs/version-0.12.0/design/resilience.md b/versioned_docs/version-0.12.0/design/resilience.md
new file mode 100644
index 0000000..c68a708
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/resilience.md
@@ -0,0 +1,144 @@
+---
+id: resilience
+title: Resilience
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This is not a HA (High-availability) design, HA implies that a service can
+survive from a fatal software/hardware failure. That requires one or more
+standby instances providing same services to take over active instance on failures.
+Resilience here means for YuniKorn, we can restart it without losing its state.
+
+## The problem
+
+YuniKorn is designed as a stateless service, it doesn't persist its state, e.g
+applications/queues/allocations etc, to any persistent storage. All states are
+in memory only. This design ensures YuniKorn to be able to response requests with
+low latency, and deployment mode is simple. However, a restart (or recovery) will
+have the problem to lose state data. We need a decent way to reconstruct all
+previous states on a restart.
+
+## Design
+
+### Workflow
+
+Scheduler core has no notion of "state", which means it does not know if it is under recovering.
+It is too complex to maintain a series of `scheduler states` in both core and shim, because we must
+keep them in-sync. However, if we live under a simple assumption: **scheduler core only responses
+requests, the correction of requests is ensured by shim according its current state**.
+The design becomes much simpler. This way, the shim maintains a state machine like below. When
+it is under `running` state, it sends new requests to the scheduler core as long as a new one is found;
+when under `recovering` state, it collects previous allocations and send recovery messages to
+the scheduler core, and waiting for recovery to be accomplished.
+
+Shim scheduler state machine
+
+```
+ Register Recover Success
+New -----------> Registered -----------> Recovering ----------> Running
+ | Fail
+ --------> Failed
+```
+
+Following chart illustrate how yunikorn-core and shim works together on recovery.
+
+
+
+Restart (with recovery) process
+- yunikorn-shim registers itself with yunikorn-core
+- shim enters "recovering" state. Under "recovering" state, the shim only scans existing nodes and allocations, no new scheduling requests will be sent.
+ - shim scans existing nodes from api-server and added them to cache
+ - shim scans existing pods from api-server, filter out the pods that already assigned (scheduled to a node), and added that to cache (allocation in that node)
+ - shim sends update request to yunikorn-core with the info found in previous steps
+- yunikorn-core handles update requests, the steps should look like a replay of allocation process, including
+ - adding node
+ - adding applications
+ - adding allocations
+ - modifying queue resources
+ - update partition info
+- when all nodes are fully recovered, shim transits the state to "running"
+- shim notifies yunikorn-core that recovery is done, then yunikorn-core transits to "running" state.
+
+### How to determine recovery is complete?
+
+Shim queries K8s api-server to get how many nodes were available in this cluster. It tracks the recovering status of each node.
+Once all nodes are recovered, it can claim the recovery is completed. This approach requires us to add `recovering` and `recovered`
+states to nodes' state machine in the shim.
+
+### Node recovery
+
+In the shim layer, it maintains states for each node and pods running on this node. When start to recover nodes,
+all nodes initially are considered as under `recovering`. Only when all pods running on this node are fully recovered,
+the node can be considered as `recovered`.
+
+
+
+Like demonstrated on upon diagram,
+
+- Node0 is still recovering because pod0 is recovering.
+- Node1 is recovered (become schedulable) because all pods on this node have been recovered.
+- Node2 is lost, shim lost contact with this node. If after sometime this node comes back, shim should still try to recover this node.
+
+### Requests for recovery
+
+During recovery process, shim needs to collect all known information of applications, nodes and allocations from the underneath
+Resource Manager and use them for recovery.
+
+#### Applications
+
+Existing applications must be recovered first before allocations. Shim needs to scan all existing applications
+from nodes, and add applications info as a list of `AddApplicationRequest` in the `UpdateRequest`. This is same
+as the fresh application submission.
+
+```
+message AddApplicationRequest {
+ string applicationID = 1;
+ string queueName = 2;
+ string partitionName = 3;
+}
+```
+
+#### Nodes and allocations
+
+Once a shim is registered to the scheduler-core, subsequent requests are sent via `UpdateRequest#NewNodeInfo`
+(see more from [si.proto](https://github.com/apache/incubator-yunikorn-scheduler-interface/blob/master/si.proto)).
+The structure of the messages looks like,
+
+```
+message NewNodeInfo {
+ // nodeID
+ string nodeID = 1;
+ // optional node attributes
+ map<string, string> attributes = 2;
+ // total node resource
+ Resource schedulableResource = 3;
+ // existing allocations on this node
+ repeated Allocation existingAllocations = 4;
+}
+```
+Shim needs to scan all existing allocations on a node and wrap these info up as a `NewNodeInfo`, add that to a
+`UpdateRequest` and then send to scheduler-core.
+
+**Note**: the recovery of existing allocations depend on the existence of applications, which means applications must
+be recovered first. Since scheduler-core handles `UpdateRequest` one by one, it is required that all existing allocations
+in a `UpdateRequest` must from known applications or new applications embedded within the same `UpdateRequest`, which can be
+specified in `NewApplications` field. Scheduler-core ensures `NewApplications` are always processed first.
+
diff --git a/versioned_docs/version-0.12.0/design/scheduler_configuration.md b/versioned_docs/version-0.12.0/design/scheduler_configuration.md
new file mode 100644
index 0000000..54b616a
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/scheduler_configuration.md
@@ -0,0 +1,246 @@
+---
+id: scheduler_configuration
+title: Scheduler Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The Yunikorn core scheduler configuration has two separate areas that need to be configured. The scheduler service itself, things like web service ports etc, and the queue configuration. The split between the two types of configuration is proposed with two points in mind:
+* Separation of duty
+* Dynamic vs Static
+
+The scheduler configuration is mainly static. There is no need to change a web service port or a scheduling policy while the service is running. The queue configuration is far more dynamic and can change while the service is running.
+
+From a separation of duty we can allow an operator that manages the cluster to make changes to the scheduler queues. You would not want to allow that administrator to change the scheduler configuration itself.
+
+Separated from the core scheduler configuration we have one or more shim configurations. We currently cannot anticipate the deployment model of the scheduler and its shims. A shim, like the k8s-shim, might run in the same container or node but there is no guarantee it will. We also do not know the number of shims that will be used with one core scheduler. There is also still the possibility to have multiple instances of the same shim with one core scheduler.
+
+Shim configuration must be independent of the core scheduler configuration.
+## Scheduler Configuration
+Scheduler configuration covers all the configuration needed to start the scheduler and the dependent services. The configuration consists of a simple key value pair. All configuration to start the service must be part of this configuration.
+The scheduler configuration must exclude the queue related configuration.
+
+Scheduler configuration as currently identified
+* Bind host
+* Service port
+* Web bind host
+* Web service port
+* SSL config
+* Shims Configured
+* SchedulerACL
+
+Configuration to consider:
+* Assign multiple containers in one go: use case is bin packing, don’t spread an application over large number of nodes. Needs to become configurable.
+* Pre-emption related configuration:
+ * threshold: do not pre-empt from a queue if the cluster load is below a certain threshold.
+ * Interval: pause between pre-emption checks
+## Queue Configuration
+### Queue Definition
+On startup the scheduler will load the configuration for the queues from the provided configuration file after initialising the service. If there is no queue configuration provided the scheduler should start up with a simple default configuration which performs a well documented default behaviour.
+Based on the kubernetes definition this configuration could be a configMap <sup id="s1">[1](#f1)</sup> but not a CRD.
+
+The queue configuration is dynamic. Changing the queue configuration must not require a scheduler restart.
+Changes should be allowed by either calling the GO based API, the REST based API or by updating the configuration file. Changes made through the API must be persisted in the configuration file. Making changes through an API is not a high priority requirement and could be postponed to a later release.
+
+The queue configuration defines queues in a hierarchy: a tree. The base of the tree is the _root_ queue. The queue configuration must define a single _root_ queue. All queues that are defined in queue configuration are considered _managed_ queues.
+
+The root queue reflect the whole cluster. Resource settings on the root queue are not allowed. The resources available to the root queue are calculated based on the registered node resources in the cluster. If resources would be specified on the root limit the cluster would either be artificially limited to a specific size or expect resources to be available that are not there.
+
+Queues in the hierarchy in the tree are separated by the “.” dot character (ASCII 0x2E). This indirectly means that a queue name itself cannot contain a dot as it interferes with the hierarchy separator. Any queue name in the configuration that contains a dot will cause the configuration to be considered invalid. However we must allow placement rules to create a queue with a dot based input.
+
+Not all queues can be used to submit an application to. Applications can only be submitted to a queue which does not have a queue below it. These queues are defined as the _leaf_ queues of the tree. Queues that are not a _leaf_ and thus can contain other queues or child queues are considered _parent_ queues.
+
+Each queue must have exactly one _parent_ queue, besides the root queue. The root queue cannot have a _parent_ and will be automatically defined as a _parent_ queue type.
+A fully qualified queue name, case insensitive, must be unique in the hierarchy. A queue in the hierarchy can thus be only uniquely identified by its fully qualified path. This means that a queue with the same name is allowed at a different point in the hierarchy.
+Example:
+```
+root.companyA.development
+root.companyB.development
+root.production.companyA
+```
+In the example the queues _companyA_ and _companyB_ are _parent_ queues. Both _development_ queues are _leaf_ queues.
+The second instance of the _companyA_ queue is a _leaf_ queue which is not related to the first instance as it is defined at a different level in the hierarchy.
+
+The queue as defined in the configuration will be assigned a queue type. This can either be implicit based on how the queue is defined in the hierarchy or explicit by setting the optional _parent_ property as part of the queue definition. By default all queues will be assigned their type based on the configuration. There is only one case in which this should automatic process would need to be overridden and that is to mark a _leaf_ in the configuration as a _parent_. The use case is part [...]
+
+Access control lists provide a split between submission permission and administration permissions. Submission access to a queue allows an application to be submitted to the queue by the users or groups specified. The administration permissions allows submission to the queue plus the administrative actions. Administrative actions are currently limited to killing an application and moving an application to a different queue.
+
+Access control lists are checked recursively up to the root of the tree starting at the lowest point in the tree. In other words when the access control list of a queue does not allow access the parent queue is checked. The checks are repeated all the way up to the root of the queues.
+
+On each queue, except the root queue, the following properties can be set:
+* QueueType:
+ * Parent (boolean)
+* Resource settings:
+ * Guaranteed (resource)
+ * Maximum (resource)
+* Running Application limit:
+ * Maximum (integer)
+* Queue Permissions:
+ * SubmitACL (ACL)
+ * AdminACL (ACL)
+* Pre emption setting:
+ * PreEmptionAllowed (boolean)
+* Application sort algorithm:
+ * ApplicationSortPolicy (enumeration: fair, fifo)
+
+On the root queue only the following properties can be set:
+* Running Application limit:
+ * Maximum (integer)
+* Queue Permissions:
+ * SubmitACL (ACL)
+ * AdminACL (ACL)
+* Application sort algorithm:
+ * ApplicationSortPolicy (enumeration: fair, fifo)
+
+### User definition
+Applications are run by a user could run in one or more queues. The queues can have limits set on the resources that can be used. This does not limit the amount of resources that can be used by the user in the cluster.
+
+From an administrative perspective setting a limit of the resources that can be used by a specific user can be important. In this case a user is broadly defined as the identity that submits the application. This can be a service or a person, from a scheduling perspective there is no difference.
+User limits can prevent a take over of a queue or the cluster by a misbehaving user or application. From a multi tenancy perspective user limits also allows for sharing or subdivision of resources within the tenancy however that is defined.
+
+Adding user based limits will allow the cluster administrators to control the cluster wide resource usage of a user:
+* Running Application limit:
+ * Maximum (integer)
+* Resource setting:
+ * Maximum (resource)
+
+### Placement Rules definition
+Schedulers can place an application in a queue dynamically. This means that an application when submitted does not have to include a queue to run in.
+
+A placement rule will use the application details to place the application in the queue. The outcome of running a placement rule will be a fully qualified queue or a `fail`, which means execute the next rule in the list. Rules will be executed in the order that they are defined.
+
+During the evaluation of the rule the result could be a queue name that contains a dot. This is especially true for user and group names which are POSIX compliant. When a rule generates a partial queue name that contains a dot it must be replaced as it is the separator in the hierarchy. The replacement text will be `_dot_`
+
+The first rule that matches, i.e. returns a fully qualified queue name, will halt the execution of the rules. If the application is not placed at the end of the list of rules the application will be rejected. Rules can return queues that are not defined in the configuration only if the rule allows creation of queues.
+
+These queues created by the placement rules are considered _unmanaged_ queues as they are not managed by the administrator in the configuration. An administrator cannot influence the _unmanaged_ queue creation or deletion. The scheduler creates the queue when it is needed and removes the queue automatically when it is no longer used.
+
+Rules provide a fully qualified queue name as the result. To allow for deeper nesting of queues the parent of the queue can be set as part of the rule evaluation. The rule definition should allow a fixed configured fully qualified parent to be specified or it can call a second rule to generate the parent queue. By default a queue is generated as a child of the root queue.
+
+Example:
+Placing an application submitted by the user _user1_ whom is a member of the groups _user1_ and _companyA_ in a queue based on UserName:
+```
+Rule name: UserName
+ Parent: root.fixedparent
+Result: root.fixedparent.user1
+
+Rule name: UserName
+ Parent: SecondaryGroup
+ Filter:
+ Type: allow
+ Groups: company.*
+Result: root.companyA.user1
+
+Rule name: UserName
+Filter:
+ Users: user2,user3
+Result: denied placement
+```
+The default behaviour for placing an application in a queue, which would do the same as using the queue that is provided during submit, would be a rule that takes the provided queue with the create flag set to false.
+
+Access permissions will be enforced as part of the rule evaluation. For _managed_ queues this means that the ACL for the queue itself is checked. For an _unmanaged_ queue the parent queue ACL is the one that is checked. For the definition of the access control list and checks see the [Access Control Lists](#access-control-lists) chapter.
+
+Defining placement rules in the configuration requires the following information per rule:
+* Name:
+ * Name (string)
+* Parent
+ * Parent (string)
+* Create Flag:
+ * Create (boolean)
+* Filter:
+ * A regular expression or list of users/groups to apply the rule to.
+
+The filter can be used to allow the rule to be used (default behaviour) or deny the rule to be used. User or groups matching the filter will be either allowed or denied.
+The filter is defined as follow:
+* Type:
+ * Type (string) which can have no value (empty) or "allow" or "deny", case insensitive.
+* Users:
+ * A list of zero or more user names. If the list is exactly one long it will be interpreted as a regular expression.
+* Groups:
+ * A list of zero or more group names. If the list is exactly one long it will be interpreted as a regular expression.
+
+Proposed rules for placing applications would be:
+* Provided: returns the queue provided during the submission
+* UserName: returns the user name
+* PrimaryGroupName: returns the primary group of the user
+* SecondaryGroupName: returns the first secondary group of the user that matches
+* Fixed: returns the queue name configured in the rule
+* ApplicationType: returns the application type (if available)
+
+For _unmanaged_ queues in the current revision of the configuration you cannot provide any queue specific properties. However in the future we should consider propagating specific resource related settings from a _managed_ parent to the _unmanaged_ child, specifically:
+* Dynamic Resource settings:
+ * Guaranteed (resource)
+ * Maximum (resource)
+* Dynamic Running Application limit:
+ * Maximum (integer)
+
+### Configuration updates
+Updating the queue definition will allow updating the existing queue properties as well as adding and removing queues. A new queue definition will only become active if the configuration can be parsed. The change of the definition is an atomic change which applies all modification in one action.
+
+Updating the queue properties will not automatically trigger further action. This means that if the maximum number of resources of a queue or its parent is changed we leave the applications in the queue running as they are. The scheduler will adhere to the new property values which should see the convergence over time.
+
+A _managed_ queue will only be removed if it is removed from the configuration. Before we can remove a queue it must not be running applications. This means that when a _managed_ queue is removed from the configuration it must be empty or the system needs to allow the queue to drain. Forcing a _managed_ queue to be empty before we can remove it is not possible which means that _managed_ queues are removed in multiple steps:
+1. The queue is removed from the configuration
+1. The queue is marked as `draining`
+1. All managed queues that are `draining` and empty are removed
+
+Long running applications should be handled gracefully when removing a _managed_ queue. The scheduler should at least track and expose that a queue has been in a _draining_ state for an extended period of time. In the optimal case the application should be notified of the queue change to allow it to release resources. In all cases the queue administrators should be notified to allow them to take action. This action would currently be a manual move of the application to a different queue [...]
+
+_Unmanaged_ queues that are not defined in the queue definition are created by the scheduler automatically based on the placement rules. _Unmanaged_ queues have a lifespan independent of the configuration. Whenever an _unmanaged_ queue is empty it will get removed. The queue will automatically be created again when a new application is requesting it via triggering the placement rule.
+
+Removing an empty _managed_ or _unmanaged_ queue is handled by the same removal code which must run independent of the configuration updates and scheduling actions.
+
+Configurations can change over time. The impact of a fail over or restart must still be investigated.
+Base point to make: a changed configuration should not impact the currently running applications. Queues that no longer exist should be handled somehow.
+
+### Access Control Lists
+The scheduler ACL is independent of the queue ACLs. A scheduler administrator is not by default allowed to submit an application or administer the queues in the system.
+
+All ACL types should use the same definition pattern. We should allow at least POSIX user and group names which uses the portable filename character set <sup id="s2">[2](#f2)</sup>. However we should take into account that we could have domain specifiers based on the environment that the system runs in (@ sign as per HADOOP-12751).
+
+By default access control is enabled and access is denied. The only special case is for the core scheduler which automatically adds the system user, the scheduler process owner, to the scheduler ACL. The scheduler process owner is allowed to make sure that the process owner can use the API to call any administrative actions.
+
+Access control lists give access to the users and groups that have been specified in the list. They do not provide the possibility to explicitly remove or deny access to the users and groups specified in the list.
+
+The access control list is defined as:
+```
+ACL ::= “*” | userlist [ “ “ grouplist ]
+userlist ::= “” | user { “,” user }
+grouplist ::= “” | group { “,” group }
+```
+
+This definition specifies a wildcard of * which results in access for everyone. If the user list is empty and the group list is empty nobody will have access. This deny all ACL has two possible representations:
+* an empty access control list.
+* a single space.
+
+If there is no access control list is configured access is denied by default.
+## Shim Configuration
+The shim configuration is highly dependent on the shim implementation. The k8s shim differs from the YARN shim. Currently the k8s shim is configured via command line options but we should not depend on that.
+
+### K8s shim
+The full configuration of the K8s shim is still under development.
+
+### YARN shim
+The full configuration of the YARN shim is still under development.
+
+---
+<br/><b id="f1"></b>1: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#should-i-use-a-configmap-or-a-custom-resource. [↩](#s1)
+<br/><b id="f2"></b>2: The set of characters from which portable filenames are constructed. [↩](#s2)
+<br/>`A B C D E F G H I J K L M N O P Q R S T U V W X Y Z a b c d e f g h i j k l m n o p q r s t u v w x y z 0 1 2 3 4 5 6 7 8 9 . _ -`
diff --git a/versioned_docs/version-0.12.0/design/scheduler_core_design.md b/versioned_docs/version-0.12.0/design/scheduler_core_design.md
new file mode 100644
index 0000000..3b42a8f
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/scheduler_core_design.md
@@ -0,0 +1,401 @@
+---
+id: scheduler_core_design
+title: Scheduler Core Design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+:::caution
+The scheduler core design has changed. [YUNIKORN-317](https://issues.apache.org/jira/browse/YUNIKORN-317) was committed and has removed the scheduler cache.
+This document will not be maintained and is just for historical reference.
+See [scheduler cache removal design](cache_removal.md)
+:::
+
+Github repo: https://github.com/apache/incubator-yunikorn-core/
+
+Scheduler core encapsulates all scheduling algorithms, it collects resources from underneath resource management
+platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
+best spot for each request and then sends response allocations to the resource management platform.
+Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface).
+
+## Components:
+
+```
+
+ +---------------+ +--------------+
+ |K8s Shim | |YARN Shim |
+ +---------------+ +--------------+
+
+ +--------------+ +------------+
+ Scheduler- | GRPC Protocol| |Go API |
+ Interface: +--------------+ +------------+
+
++---------------------------------------------------------------------------+
+ +--------------------+
+ |Scheduler API Server|
+ +-------------+ +---------+----------+
+ |AdminService | |
+ +-------------+ |Write Ops +----------------+
+ +-------------+ V ++Scheduler |
+ |Configurator | +-------------------+ Allocate || And |
+ +-------------+ |Cache Event Handler+<-----------------| |
+ +----------> +-------------------+ Preempt ++Preemptor |
+ Update Cfg Handled by policies +----------------+
+ + (Stateless)
+ +------v--------+
+ |Scheduler Cache|
+ +---------------+
+ +---------------------------------------------+
+ |--------+ +------+ +----------+ +----------+ |
+ ||Node | |Queue | |Allocation| |Requests | |
+ |--------+ +------+ +----------+ +----------+ |
+ +---------------------------------------------+
+```
+
+### Scheduler API Server (RMProxy)
+
+Responsible for communication between RM and Scheduler, which implements scheduler-interface GRPC protocol,
+or just APIs. (For intra-process communication w/o Serde).
+
+### Scheduler Cache
+
+Caches all data related to scheduler state, such as used resources of each queues, nodes, allocations.
+Relationship between allocations and nodes, etc. Should not include in-flight data for resource allocation.
+For example to-be-preempted allocation candidates. Fair share resource of queues, etc.
+
+### Scheduler Cache Event Handler
+
+Handles all events which needs to update scheduler internal state. So all the write operations will be carefully handled.
+
+### Admin Service
+
+Handles request from Admin, which can also load configurations from storage and update scheduler policies.
+
+### Scheduler and Preemptor
+
+Handles Scheduler's internal state. (Which is not belong to scheduelr cache), such as internal reservations, etc.
+Scheduler and preemptor will work together, make scheduling or preemption decisions. All allocate/preempt request
+will be handled by event handler.
+
+## Scheduler's responsibility
+
+- According to resource usages between queues, sort queues, applications, and figure out order of application allocation. (This will be used by preemption as well).
+- It is possible that we cannot satisfy some of the allocation request, we need to skip them and find next request.
+- It is possible that some allocation request cannot be satisfied because of resource fragmentation. We need to reserve room for such requests.
+- Different nodes may belong to different disjoint partitions, we can make independent scheduler runs
+- Be able to config and change ordering policies for apps, queues.
+- Application can choose their own way to manage sort of nodes.
+
+## Preemption
+
+- It is important to know "who wants the resource", so we can do preemption based on allocation orders.
+- When do preemption, it is also efficient to trigger allocation op. Think about how to do it.
+- Preemption needs to take care about queue resource balancing.
+
+## Communication between Shim and Core
+
+YuniKorn-Shim (like https://github.com/apache/incubator-yunikorn-k8shim) communicates with core by
+using scheduler-interface (https://github.com/apache/incubator-yunikorn-scheduler-interface).
+Scheduler interface has Go API or GRPC. Currently, yunikorn-k8shim is using Go API to communicate with yunikorn-core
+to avoid extra overhead introduced by GRPC.
+
+**Shim (like K8shim) first need to register with core:**
+
+```go
+func (m *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerRequest, callback api.ResourceManagerCallback) (*si.RegisterResourceManagerResponse, error)
+```
+
+Which indicate ResourceManager's name, a callback function for updateResponse. The design of core is be able to do scheduling for multiple clusters (such as multiple K8s cluster) just with one core instance.
+
+**Shim interacts with core by invoking RMProxy's Update API frequently, which updates new allocation request, allocation to kill, node updates, etc.**
+
+```go
+func (m *RMProxy) Update(request *si.UpdateRequest) error
+```
+
+Response of update (such as new allocated container) will be received by registered callback.
+
+## Configurations & Semantics
+
+Example of configuration:
+
+- Partition is name space.
+- Same queues can under different partitions, but enforced to have same hierarchy.
+
+ Good:
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b c
+ ```
+
+ Good (c in partition y acl=""):
+
+ ```
+ partition=x partition=y
+ a a
+ / \ /
+ b c b
+ ```
+
+ Bad (c in different hierarchy)
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b d
+ /
+ c
+ ```
+
+ Bad (Duplicated c)
+
+ ```
+ partition=x
+ a
+ / \
+ b c
+ /
+ c
+
+ ```
+
+- Different hierarchies can be added
+
+ ```scheduler-conf.yaml
+ partitions:
+ - name: default
+ queues:
+ root:
+ configs:
+ acls:
+ childrens:
+ - a
+ - b
+ - c
+ - ...
+ a:
+ configs:
+ acls:
+ capacity: (capacity is not allowed to set for root)
+ max-capacity: ...
+ mapping-policies:
+ ...
+ - name: partition_a:
+ queues:
+ root:...
+ ```
+
+## How scheduler do allocation
+
+Scheduler runs a separate goroutine to look at asks and available resources, and do resource allocation. Here's allocation logic in pseudo code:
+
+Entry point of scheduler allocation is `scheduler.go: func (s *Scheduler) schedule()`
+
+```
+# First of all, YuniKorn has partition concept, a logical resource pool can consists
+# of one of multiple physical dis-joint partitions. It is similar to YARN's node
+# partition concept.
+
+for partition : partitions:
+ # YuniKorn can reserve allocations for picky asks (such as large request, etc.)
+ # Before doing regular allocation, YuniKorn look at reservedAllocations first.
+ for reservedAllocation : partition.reservedAllocations:
+ reservedAllocation.tryAllocate(..)
+
+ # After tried all reserved allocation, YuniKorn will go to regular allocation
+ partition.tryAllocate(..)
+
+ # If there's any allocation created, scheduler will create an AllocationProposal
+ # and send to Cache to "commit" the AllocationProposal
+```
+
+**Allocation by hierchical of queues**
+
+Inside `partition.tryAllocate`
+
+It recursively traverse from root queue and down to lower level, for each level, logic is inside `pkg/scheduler/scheduling_queue.go func (sq *SchedulingQueue) tryAllocate`
+
+Remember YuniKorn natively supports hierarchical of queues. For ParentQueue (which has sub queues under the parent queue), it uses queue's own sorting policy to sort subqueues and try to allocate from most preferred queue to least-preferred queue.
+
+For LeafQueue (which has applications inside the queue), it uses queue's own sorting policy to sort applications belong to the queue and allocate based on the sorted order.
+
+(All sorting policies can be configured differently at each level.)
+
+**Allocation by application**
+
+When it goes to Application, see (`scheduler_application.go: func (sa *SchedulingApplication) tryAllocate`), It first sort the pending resource requests belong to the application (based on requests' priority). And based on the selected request, and configured node-sorting policy, it sorts nodes belong to the partition and try to allocate resources on the sorted nodes.
+
+When application trying to allocate resources on nodes, it will invokes PredicatePlugin to make sure Shim can confirm the node is good. (For example K8shim runs predicates check for allocation pre-check).
+
+**Allocation completed by scheduler**
+
+Once allocation is done, scheduler will create an AllocationProposal and send to Cache to do further check, we will cover details in the upcoming section.
+
+## Flow of events
+
+Like mentioned before, all communications between components like RMProxy/Cache/Schedulers are done by using async event handler.
+
+RMProxy/Cache/Scheduler include local event queues and event handlers. RMProxy and Scheduler have only one queue (For example: `pkg/scheduler/scheduler.go: handleSchedulerEvent`), and Cache has two queues (One for events from RMProxy, and one for events from Scheduler, which is designed for better performance).
+
+We will talk about how events flowed between components:
+
+**Events for ResourceManager registration and updates:**
+
+```
+Update from ResourceManager -> RMProxy -> RMUpdateRequestEvent Send to Cache
+New ResourceManager registration -> RMProxy -> RegisterRMEvent Send to Cache
+```
+
+**Cache Handles RM Updates**
+
+There're many fields inside RM Update event (`RMUpdateRequestEvent`), among them, we have following categories:
+
+```
+1) Update for Application-related updates
+2) Update for New allocation ask and release.
+3) Node (Such as kubelet) update (New node, remove node, node resource change, etc.)
+```
+
+More details can be found at:
+
+```
+func (m *ClusterInfo) processRMUpdateEvent(event *cacheevent.RMUpdateRequestEvent)
+
+inside cluster_info.go
+```
+
+**Cache send RM updates to Scheduler**
+
+For most cases, Cache propagate updates from RM to scheduler directly (including Application, Node, Asks, etc.). And it is possible that some updates from RM is not valid (such as adding an application to a non-existed queue), for such cases, cache can send an event back to RMProxy and notify the ResourceManager. (See `RMApplicationUpdateEvent.RejectedApplications`)
+
+**Cache handles scheduler config**
+
+Cache also handles scheduler's config changes, see
+
+```go
+func (m *ClusterInfo) processRMConfigUpdateEvent(event *commonevents.ConfigUpdateRMEvent)
+```
+
+Similar to other RM updates, it propages news to scheduelr.
+
+**Scheduler do allocation**
+
+Once an AllocationProposal created by scheduler, scheduler sends `AllocationProposalBundleEvent` to Cache to commit.
+
+Cache look at AllocationProposal under lock, and commit these proposals. The reason to do proposal/commit is Scheduler can run in multi-threads which could cause conflict for resource allocation. This approach is inspired by Borg/Omega/YARN Global Scheduling.
+
+Cache checks more states such as queue resources, node resources (we cannot allocate more resource than nodes' available), etc. Once check is done, Cache updates internal data strcture and send confirmation to Scheduler to update the same, and scheduler sends allocated Allocation to RMProxy so Shim can do further options. For example, K8shim will `bind` an allocation (POD) to kubelet.
+
+```
+Job Add:
+--------
+RM -> Cache -> Scheduler (Implemented)
+
+Job Remove:
+-----------
+RM -> Scheduler -> Cache (Implemented)
+Released allocations: (Same as normal release) (Implemented)
+Note: Make sure remove from scheduler first to avoid new allocated created.
+
+Scheduling Request Add:
+-----------------------
+RM -> Cache -> Scheduler (Implemented)
+Note: Will check if requested job exists, queue exists, etc.
+When any request invalid:
+ Cache -> RM (Implemented)
+ Scheduler -> RM (Implemented)
+
+Scheduling Request remove:
+-------------------------
+RM -> Scheduler -> Cache (Implemented)
+Note: Make sure removed from scheduler first to avoid new container allocated
+
+Allocation remove (Preemption)
+-----------------
+Scheduler -> Cache -> RM (TODO)
+ (confirmation)
+
+Allocation remove (RM voluntarilly ask)
+---------------------------------------
+RM -> Scheduler -> Cache -> RM. (Implemented)
+ (confirmation)
+
+Node Add:
+---------
+RM -> Cache -> Scheduler (Implemented)
+Note: Inside Cache, update allocated resources.
+Error handling: Reject Node to RM (Implemented)
+
+Node Remove:
+------------
+Implemented in cache side
+RM -> Scheduler -> Cache (TODO)
+
+Allocation Proposal:
+--------------------
+Scheduler -> Cache -> RM
+When rejected/accepted:
+ Cache -> Scheduler
+
+Initial: (TODO)
+--------
+1. Admin configured partitions
+2. Cache initializes
+3. Scheduler copies configurations
+
+Relations between Entities
+-------------------------
+1. RM includes one or multiple:
+ - Partitions
+ - Jobs
+ - Nodes
+ - Queues
+
+2. One queue:
+ - Under one partition
+ - Under one RM.
+
+3. One job:
+ - Under one queue (Job with same name can under different partitions)
+ - Under one partition
+
+RM registration: (TODO)
+----------------
+1. RM send registration
+2. If RM already registered, remove old one, including everything belong to RM.
+
+RM termination (TODO)
+--------------
+Just remove the old one.
+
+Update of queues (TODO)
+------------------------
+Admin Service -> Cache
+
+About partition (TODO)
+-----------------------
+Internal partition need to be normalized, for example, RM specify node with partition = xyz.
+Scheduler internally need to normalize it to <rm-id>_xyz
+This need to be done by RMProxy
+
+```
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/design/scheduler_object_states.md b/versioned_docs/version-0.12.0/design/scheduler_object_states.md
new file mode 100644
index 0000000..e5a8b8f
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/scheduler_object_states.md
@@ -0,0 +1,127 @@
+---
+id: scheduler_object_states
+title: Scheduler Object States
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+The YuniKorn project uses state machines to track the states of different objects.
+This ranges from applications in the core to nodes in the k8shim.
+The state machines are independent and not shared between the resource managers and core.
+A resource manager shim, and the core can thus have an independent idea of the state of a similar object.
+
+## Core Scheduler
+State change are triggered by events that get processed.
+One event can cause a change for multiple states or no change at all.
+
+### Application State
+Applications have a complex state model.
+An application when created starts ain the new state.
+
+An application can have the following states:
+* New: A new application that is being submitted or created, from here the application transitions into the accepted state when it is ready for scheduling.
+The first ask to be added will trigger the transition.
+* Accepted: The application is ready and part of the scheduling cycle.
+On allocation of the first ask the application moves into a starting state.
+This state is part of the normal scheduling cycle.
+* Starting: The application has exactly one allocation confirmed this corresponds to one running container/pod.
+The application transitions to running if and when more allocations are added to the application.
+This state times out automatically to prevent applications that consist of just one allocation from getting stuck in this state.
+The current time out is set to 5 minutes, and cannot be changed.
+If after the timeout expires the application will auto transition to running.
+The state change on time out is independent of the number of allocations added.
+This state is part of the normal scheduling cycle.
+* Running: The state in which the application will spend most of its time.
+Containers/pods can be added to and removed from the application.
+This state is part of the normal scheduling cycle.
+* Completing: An application that has no pending requests or running containers/pod will be completing.
+This state shows that the application has not been marked completed yet but currently is not actively being scheduled.
+* Completed: An application is considered completed when it has been in the completing state for a defined time period.
+From this state the application can only move to the Expired state, and it cannot move back into any of scheduling states (Running or Completing)
+The current timeout is set to 30 seconds.
+* Expired: The completed application is tracked for a period of time, after that is expired and deleted from the scheduler.
+This is a final state and after this state the application cannot be tracked anymore.
+* Failing: An application marked for failing, what still has some allocations or asks what needs to be cleaned up before entering into the Failed state.
+ The application can be Failing when the partition it belongs to is removed or during gang scheduling, if the placeholder processing times out, and the application has no real allocations yet.
+* Failed: An application is considered failed when it was marked for failure and all the pending requests and allocations were already removed.
+This is a final state. The application cannot change state after entering.
+* Rejected: The application was rejected when it was added to the scheduler.
+This only happens when a resource manager tries to add a new application, when it gets created in a New state, and the scheduler rejects the creation.
+Applications can be rejected due ACLs denying access to a queue the application has specified, or a placement via placement rules has failed.
+This is a final state. The application cannot change state after entering.
+
+The events that can trigger a state change:
+* Reject: rejecting the application by the scheduler (source: core scheduler)
+* Run: progress an application to the next active state (source: core scheduler)
+* Complete: mark an application as idle or complete (source: core scheduler)
+* Fail: fail an application (source: resource manager or core scheduler)
+* Expire: progress the application to the expired state and remove it from the scheduler (source: core scheduler)
+
+Here is a diagram that shows the states with the event that causes the state to change:
+
+
+### Object State
+<!-- fix the draining to stopped transition -->
+The object state is used by the following objects:
+* queues
+* partitions
+
+The object states are as follows:
+* Active: The object is active and used during the scheduling cycle.
+This is the starting and normal state of an object.
+An active object transitions to draining when it is removed.
+* Stopped: The object is stopped and no longer actively scheduled.
+The object if empty is ready to be removed from the scheduler.
+The object can transition back into active state if it gets re-started.
+* Draining: Before an object can be removed it needs to be cleaned up.
+The cleanup starts with placing the object in the draining state.
+In this state it does not accept additions or changes but is still actively being scheduled.
+This allows for a graceful shutdown, cleanup and removal of the object.
+This is the final state.
+
+The events that can trigger a state change:
+* Start: make the object active (source: core scheduler)
+* Stop: make the object inactive (source: core scheduler)
+* Remove: mark an object for removal (source: core scheduler)
+
+Here is a diagram that shows the states with the event that causes the state to change:
+
+
+### Node
+<!-- should start using object state -->
+Node objects in the core are not using a state machine but do have a state.
+A node can have one of two states: `schedulable` or `not schedulable`.
+There is no complex state model or complex transition logic.
+The scheduler can either use the node or not.
+
+The node status changes based on the status provided by the resource manager (shim) that owns the node.
+
+## K8Shim Resource Manager
+
+### Application
+
+
+### Task
+
+
+### Node
+
+
+### Scheduler
+
diff --git a/versioned_docs/version-0.12.0/design/state_aware_scheduling.md b/versioned_docs/version-0.12.0/design/state_aware_scheduling.md
new file mode 100644
index 0000000..f92f93c
--- /dev/null
+++ b/versioned_docs/version-0.12.0/design/state_aware_scheduling.md
@@ -0,0 +1,112 @@
+---
+id: state_aware_scheduling
+title: Batch Workloads Ordering with StateAware Policy
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## The problem
+A common pattern while processing data is that the application can be divided into multiple stages.
+Another way to look at this is the fact that processing needs to be kicked off and that the first step is to start a driver or manager for the application.
+Later stages might depend on the previous stages.
+When running applications in a size limited environment this could lead to a resource exhaustion when submitting multiple applications at the same time.
+These first stages might consume all available resources leaving no room for the next stage(s) to start.
+Often this issue is caused by having a high number of applications start simultaneous and trying to get resources in parallel.
+### Example issue
+When submitting numerous Spark applications in a short amount of time the drivers will all be started shortly after each other.
+The drivers consume all available resources in a queue or in the whole cluster.
+After starting the drivers they will request resources for the executors.
+Since the queue or cluster has no resources left the executors will not be started.
+The driver cannot progress.
+The only way that progress would be made is if and when one of the drivers finishes or fails and frees up resources for executors to be started.
+
+## Design
+### Design goals
+1. Prevent resource exhaustion by first stage allocations
+1. Improve chance for jobs to get minimal required resources over others
+
+### None goals
+1. This is NOT an implementation of Gang scheduling.
+1. No change to the currently supported FAIR or FIFO scheduling algorithms
+1. Fix resource quota usage outside of the core scheduler for submitted but waiting applications
+
+### Possible solutions
+Other batch schedulers like the YARN schedulers use a limit on the number of simultaneous running applications.
+They use either resource constraints on the driver or management stage or set a hard limit of the number of applications that can run in a queue.
+The draw back of that solution is that it does not work well in a cluster that can scale up or down automatically in a cloud environment.
+To work around that percentage based limits could be set on the consumed resources for the driver or management stage.
+This does not alleviate the fact that driver or management stages can be of any size, large and or small, which complicates the percentage scenario further as it does not give a predictable behaviour.
+
+A different solution would be to assume a specific behaviour of the applications.
+Using that assumption a limit on the applications could be set based on the state it is in.
+The spark driver and executor behaviour is the most usual use case.
+This would provide a way to limit scheduling to existing applications and only drip feed new applications into the list of applications to schedule when there are resources available.
+
+### Algorithm
+The algorithm described here is based on the drip feed of new applications into the applications to schedule based on the states of all applications.
+Scheduling is based on the applications in a queue.
+The algorithm will be applied at a queue level.
+This is not a cluster wide setup.
+
+What we want to achieve is the following behaviour: only schedule one (1) application that is in its early stage(s) (called a starting state) at the same time.
+Only consider another new application if and when the previous application has transitioned out of the starting state.
+Applications will always be allocated resources on a first in first out basis based on submission time.
+That means that an application that is newly added and in its starting phase will only get resources if applications in the later stages do not need any resources.
+
+This algorithm will be implemented as an application sorting policy on a queue.
+This allows specific queues to limit parallel application startup while other queues with different work loads can schedule without or with different limitations.
+
+### Fallback mechanism
+A fallback mechanism has to be built into the algorithm.
+Not all applications will request more than one allocation.
+The other case that has to be accounted for could be a misbehaving or a slow application.
+Having an application stuck in the starting state could cause a scheduler livelock and starvation of other applications.
+
+The fall back mechanism proposed is as simple as a time limit on the starting state.
+This means that any application auto progresses out of the starting state.
+The time limit will be set to five (5) minutes hard coded as a starting point and will not be made configurable.
+
+The other fallback mechanism considered was making the number of allocations for the starting state configurable.
+This option provided a number of issues which made it difficult to implement.
+One of the main stumbling blocks is the fact that it requires the application submitter to specify the value.
+It also does not guarantee that the application will leave the starting state either and does not fix the livelock issue.
+If an application was submitted with five required allocation but due to issues during the run never asked for more than four then the livelock would still occur.
+
+Setting a default of zero (0) would also not work as it would bypass the starting state.
+It would make the sorting policy an opt-in instead of an opt-out.
+Setting a default of one (1) does not give us much enhancement to what we currently propose.
+It makes the sorting policy an opt-out but does not give the cluster administrator any control over the scheduling behaviour.
+Weighing those against each other the proposal is to not make this configurable.
+
+### Example run
+Using Spark applications as an example: a new application can only be scheduled if the previous application has at least one (1) executor allocated.
+
+
+
+Assume we have the following Spark apps: App1 & App2 as in the diagram above. The applications were submitted in that order: App1 first, then App2. They were both submitted to the same queue.
+
+1. Both applications are in the queue waiting for the first allocation: accepted by the scheduler. App1 has requested driver D1 and App2 has requested driver D2.
+1. The scheduler sorts the application and allows 1 accepted application to be scheduled (no starting applications yet): App1 as the oldest applications with an outstanding request is scheduled.
+App1 is allocated its driver (D1) and progresses to starting.
+App2 request for a driver is ignored as the scheduler is starting App1 (only 1 application in starting or accepted state is scheduled).
+1. App1 requests executors E11 and E12. The scheduler assigns E11 and E12. At this point the application state changes to running when it has at least 1 executor allocated.
+1. App2 has been waiting to get the driver allocated. Since there are no applications in a starting state the scheduler looks at App2 which is in an accepted state. App2 moves from the accepted state to starting when the driver is allocated.
+1. App2 requests its executor E21. The application state changes to running when E21 is allocated.
+
+This process would repeat itself for any new application submitted.
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/developer_guide/build.md b/versioned_docs/version-0.12.0/developer_guide/build.md
new file mode 100644
index 0000000..0db8de0
--- /dev/null
+++ b/versioned_docs/version-0.12.0/developer_guide/build.md
@@ -0,0 +1,184 @@
+---
+id: build
+title: Build and Run
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn always works with a container orchestrator system. Currently, a Kubernetes shim [yunikorn-k8shim](https://github.com/apache/incubator-yunikorn-k8shim)
+is provided in our repositories, you can leverage it to develop YuniKorn scheduling features and integrate with Kubernetes.
+This document describes resources how to setup dev environment and how to do the development.
+
+## Development Environment setup
+
+Read the [environment setup guide](developer_guide/env_setup.md) first to setup Docker and Kubernetes development environment.
+
+## Build YuniKorn for Kubernetes
+
+Prerequisite:
+- Go 1.12+
+
+You can build the scheduler for Kubernetes from [yunikorn-k8shim](https://github.com/apache/incubator-yunikorn-k8shim) project.
+The build procedure will build all components into a single executable that can be deployed and running on Kubernetes.
+
+Start the integrated build process by pulling the `yunikorn-k8shim` repository:
+```bash
+mkdir $HOME/yunikorn/
+cd $HOME/yunikorn/
+git clone https://github.com/apache/incubator-yunikorn-k8shim.git
+```
+At this point you have an environment that will allow you to build an integrated image for the YuniKorn scheduler.
+
+### A note on Go modules and git version
+Go use git to fetch module information.
+Certain modules cannot be retrieved if the git version installed on the machine used to build is old.
+A message similar to the one below will be logged when trying to build for the first time.
+```text
+go: finding modernc.org/mathutil@v1.0.0
+go: modernc.org/golex@v1.0.0: git fetch -f origin refs/heads/*:refs/heads/* refs/tags/*:refs/tags/* in <location>: exit status 128:
+ error: RPC failed; result=22, HTTP code = 404
+ fatal: The remote end hung up unexpectedly
+```
+Update git to a recent version to fix this issue.
+Git releases later than 1.22 are known to work.
+
+### Build Docker image
+
+Building a docker image can be triggered by following command.
+
+```
+make image
+```
+
+The image with the build in configuration can be deployed directly on kubernetes.
+Some sample deployments that can be used are found under [deployments](https://github.com/apache/incubator-yunikorn-k8shim/tree/master/deployments/scheduler) directory.
+For the deployment that uses a config map you need to set up the ConfigMap in kubernetes.
+How to deploy the scheduler with a ConfigMap is explained in the [scheduler configuration deployment](developer_guide/deployment.md) document.
+
+The image build command will first build the integrated executable and then create the docker image.
+Currently, there are some published docker images under [this docker hub repo](https://hub.docker.com/r/apache/yunikorn), you are free to fetch and use.
+The default image tags are not suitable for deployments to an accessible repository as it uses a hardcoded user and would push to Docker Hub with proper credentials.
+You *must* update the `TAG` variable in the `Makefile` to push to an accessible repository.
+When you update the image tag be aware that the deployment examples given will also need to be updated to reflect the same change.
+
+### Inspect the docker image
+
+The docker image built from previous step has embedded some important build info in image's metadata. You can retrieve
+these info with docker `inspect` command.
+
+```
+docker inspect apache/yunikorn:scheduler-latest
+```
+
+This info includes git revisions (last commit SHA) for each component, to help you understand which version of the source code
+was shipped by this image. They are listed as docker image `labels`, such as
+
+```
+"Labels": {
+ "BuildTimeStamp": "2019-07-16T23:08:06+0800",
+ "Version": "0.1",
+ "yunikorn-core-revision": "dca66c7e5a9e",
+ "yunikorn-k8shim-revision": "bed60f720b28",
+ "yunikorn-scheduler-interface-revision": "3df392eded1f"
+}
+```
+
+### Dependencies
+
+The dependencies in the projects are managed using [go modules](https://blog.golang.org/using-go-modules).
+Go Modules require at least Go version 1.11 to be installed on the development system.
+
+If you want to modify one of the projects locally and build with your local dependencies you will need to change the module file.
+Changing dependencies uses mod `replace` directives as explained in the [Update dependencies](#Updating dependencies).
+
+The YuniKorn project has four repositories three of those repositories have a dependency at the go level.
+These dependencies are part of the go modules and point to the github repositories.
+During the development cycle it can be required to break the dependency on the committed version from github.
+This requires making changes in the module file to allow loading a local copy or a forked copy from a different repository.
+
+#### Affected repositories
+The following dependencies exist between the repositories:
+
+| repository| depends on |
+| --- | --- |
+| yunikorn-core | yunikorn-scheduler-interface |
+| yunikorn-k8shim | yunikorn-scheduler-interface, yunikorn-core |
+| yunikorn-scheduler-interface | none |
+| yunikorn-web | yunikorn-core |
+
+The `yunikorn-web` repository has no direct go dependency on the other repositories. However any change to the `yunikorn-core` webservices can affect the web interface.
+
+#### Making local changes
+
+To make sure that the local changes will not break other parts of the build you should run:
+- A full build `make` (build target depends on the repository)
+- A full unit test run `make test`
+
+Any test failures should be fixed before proceeding.
+
+#### Updating dependencies
+
+The simplest way is to use the `replace` directive in the module file. The `replace` directive allows you to override the import path with a new (local) path.
+There is no need to change any of the imports in the source code. The change must be made in the `go.mod` file of the repository that has the dependency.
+
+Using `replace` to use of a forked dependency, such as:
+```
+replace github.com/apache/incubator-yunikorn-core => example.com/some/forked-yunikorn
+```
+
+There is no requirement to fork and create a new repository. If you do not have a repository you can use a local checked out copy too.
+Using `replace` to use of a local directory as a dependency:
+```
+replace github.com/apache/incubator-yunikorn-core => /User/example/local/checked-out-yunikorn
+```
+and for the same dependency using a relative path:
+```
+replace github.com/apache/incubator-yunikorn-core => ../checked-out-yunikorn
+```
+Note: if the `replace` directive is using a local filesystem path, then the target must have the `go.mod` file at that location.
+
+Further details on the modules' wiki: [When should I use the 'replace' directive?](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
+
+## Build the web UI
+
+Example deployments reference the [YuniKorn web UI](https://github.com/apache/incubator-yunikorn-web).
+The YuniKorn web UI has its own specific requirements for the build. The project has specific requirements for the build follow the steps in the README to prepare a development environment and build how to build the projects.
+The scheduler is fully functional without the web UI.
+
+## Locally run the integrated scheduler
+
+When you have a local development environment setup you can run the scheduler in your local kubernetes environment.
+This has been tested in a Docker desktop with 'Docker for desktop' and Minikube. See the [environment setup guide](developer_guide/env_setup.md) for further details.
+
+```
+make run
+```
+It will connect with the kubernetes cluster using the users configured configuration located in `$HOME/.kube/config`.
+
+You can also use the same approach to run the scheduler locally but connecting to a remote kubernetes cluster,
+as long as the `$HOME/.kube/config` file is pointing to that remote cluster.
+
+
+## Verify external interface changes with e2e tests
+
+Yunikorn has an external REST interface which is validated by end-to-end tests. However, the tests exist in the k8shim repository.
+Whenever a change is made to the external interface, make sure that it is validated by running e2e tests or adjust the test cases accordingly.
+
+How to run the tests locally is described [here](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/test/e2e/README.md).
diff --git a/versioned_docs/version-0.12.0/developer_guide/deployment.md b/versioned_docs/version-0.12.0/developer_guide/deployment.md
new file mode 100644
index 0000000..5d27d5e
--- /dev/null
+++ b/versioned_docs/version-0.12.0/developer_guide/deployment.md
@@ -0,0 +1,133 @@
+---
+id: deployment
+title: Deploy to Kubernetes
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The easiest way to deploy YuniKorn is to leverage our [helm charts](https://hub.helm.sh/charts/yunikorn/yunikorn),
+you can find the guide [here](get_started/get_started.md). This document describes the manual process to deploy YuniKorn
+scheduler and it is majorly for developers.
+
+## Build docker image
+
+Under project root of the `yunikorn-k8shim`, run the command to build an image using the map for the configuration:
+```
+make image
+```
+
+This command will build an image. The image will be tagged with a default version and image tag.
+
+**Note** the default build uses a hardcoded user and tag. You *must* update the `IMAGE_TAG` variable in the `Makefile` to push to an appropriate repository.
+
+**Note** the latest yunikorn images in docker hub are not updated anymore due to ASF policy. Hence, you should build both scheduler image and web image locally before deploying them.
+
+## Setup RBAC
+
+The first step is to create the RBAC role for the scheduler, see [yunikorn-rbac.yaml](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/deployments/scheduler/yunikorn-rbac.yaml)
+```
+kubectl create -f scheduler/yunikorn-rbac.yaml
+```
+The role is a requirement on the current versions of kubernetes.
+
+## Create the ConfigMap
+
+This must be done before deploying the scheduler. It requires a correctly setup kubernetes environment.
+This kubernetes environment can be either local or remote.
+
+- download configuration file if not available on the node to add to kubernetes:
+```
+curl -o queues.yaml https://raw.githubusercontent.com/apache/incubator-yunikorn-k8shim/master/conf/queues.yaml
+```
+- create ConfigMap in kubernetes:
+```
+kubectl create configmap yunikorn-configs --from-file=queues.yaml
+```
+- check if the ConfigMap was created correctly:
+```
+kubectl describe configmaps yunikorn-configs
+```
+
+**Note** if name of the ConfigMap is changed the volume in the scheduler yaml file must be updated to reference the new name otherwise the changes to the configuration will not be picked up.
+
+## Attach ConfigMap to the Scheduler Pod
+
+The ConfigMap is attached to the scheduler as a special volume. First step is to specify where to mount it in the pod:
+```yaml
+ volumeMounts:
+ - name: config-volume
+ mountPath: /etc/yunikorn/
+```
+Second step is to link the mount point back to the configuration map created in kubernetes:
+```yaml
+ volumes:
+ - name: config-volume
+ configMap:
+ name: yunikorn-configs
+```
+
+Both steps are part of the scheduler yaml file, an example can be seen at [scheduler.yaml](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/deployments/scheduler/scheduler.yaml)
+for reference.
+
+## Deploy the Scheduler
+
+The scheduler can be deployed with following command.
+```
+kubectl create -f deployments/scheduler/scheduler.yaml
+```
+
+The deployment will run 2 containers from your pre-built docker images in 1 pod,
+
+* yunikorn-scheduler-core (yunikorn scheduler core and shim for K8s)
+* yunikorn-scheduler-web (web UI)
+
+The pod is deployed as a customized scheduler, it will take the responsibility to schedule pods which explicitly specifies `schedulerName: yunikorn` in pod's spec. In addition to the `schedulerName`, you will also have to add a label `applicationId` to the pod.
+```yaml
+ metadata:
+ name: pod_example
+ labels:
+ applicationId: appID
+ spec:
+ schedulerName: yunikorn
+```
+
+Note: Admission controller abstracts the addition of `schedulerName` and `applicationId` from the user and hence, routes all traffic to YuniKorn. If you use helm chart to deploy, it will install admission controller along with the scheduler.
+
+## Access to the web UI
+
+When the scheduler is deployed, the web UI is also deployed in a container.
+Port forwarding for the web interface on the standard ports can be turned on via:
+
+```
+POD=`kubectl get pod -l app=yunikorn -o jsonpath="{.items[0].metadata.name}"` && \
+kubectl port-forward ${POD} 9889 9080
+```
+
+`9889` is the default port for Web UI, `9080` is the default port of scheduler's Restful service where web UI retrieves info from.
+Once this is done, web UI will be available at: http://localhost:9889.
+
+## Configuration Hot Refresh
+
+YuniKorn supports to load configuration changes automatically from attached configmap. Simply update the content in the configmap,
+that can be done either via Kubernetes dashboard UI or commandline. _Note_, changes made to the configmap might have some
+delay to be picked up by the scheduler.
+
+
+
diff --git a/versioned_docs/version-0.12.0/developer_guide/env_setup.md b/versioned_docs/version-0.12.0/developer_guide/env_setup.md
new file mode 100644
index 0000000..c45d77e
--- /dev/null
+++ b/versioned_docs/version-0.12.0/developer_guide/env_setup.md
@@ -0,0 +1,156 @@
+---
+id: env_setup
+title: Dev Environment Setup
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+There are several ways to setup a local development environment for Kubernetes, the two most common ones are `Minikube` ([docs](https://kubernetes.io/docs/setup/minikube/)) and `docker-desktop`.
+`Minikube` provisions a local Kubernetes cluster on several Virtual Machines (via VirtualBox or something similar). `docker-desktop` on the other hand, sets up Kubernetes cluster in docker containers.
+
+## Local Kubernetes cluster using Docker Desktop
+
+In this tutorial, we will base all the installs on Docker Desktop.
+Even in this case we can use a lightweight [minikube](#local-kubernetes-cluster-with-minikube) setup which gives the same functionality with less impact.
+
+### Installation
+
+Download and install [Docker-Desktop](https://www.docker.com/products/docker-desktop) on your laptop. Latest version has an embedded version of Kubernetes so no additional install is needed.
+Just simply follow the instruction [here](https://docs.docker.com/docker-for-mac/#kubernetes) to get Kubernetes up and running within docker-desktop.
+
+Once Kubernetes is started in docker desktop, you should see something similar below:
+
+
+
+This means that:
+1. Kubernetes is running.
+1. the command line tool `kubctl` is installed in the `/usr/local/bin` directory.
+1. the Kubernetes context is set to `docker-desktop`.
+
+### Deploy and access dashboard
+
+After setting up the local Kubernetes you need to deploy the dashboard using the following steps:
+1. follow the instructions in [Kubernetes dashboard doc](https://github.com/kubernetes/dashboard) to deploy the dashboard.
+1. start the Kubernetes proxy in the background from a terminal to get access on the dashboard on the local host:
+ ```shell script
+ kubectl proxy &
+ ```
+1. access the dashboard at the following URL: [clickable link](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login)
+
+### Access local Kubernetes cluster
+
+The dashboard as deployed in the previous step requires a token or config to sign in. Here we use the token to sign in. The token is generated automatically and can be retrieved from the system.
+
+1. retrieve the name of the dashboard token:
+ ```shell script
+ kubectl -n kube-system get secret | grep kubernetes-dashboard-token
+ ```
+2. retrieve the content of the token, note that the token name ends with a random 5 character code and needs to be replaced with the result of step 1. As an example:
+ ```shell script
+ kubectl -n kube-system describe secret kubernetes-dashboard-token-tf6n8
+ ```
+3. copy the token value which is part of the `Data` section with the tag `token`.
+4. select the **Token** option in the dashboard web UI:<br/>
+ 
+5. paste the token value into the input box and sign in:<br/>
+ 
+
+## Local Kubernetes cluster with Minikube
+Minikube can be added to an existing Docker Desktop install. Minikube can either use the pre-installed hypervisor or use a hypervisor of choice. These instructions use [HyperKit](https://github.com/moby/hyperkit) which is embedded in Docker Desktop.
+
+If you want to use a different hypervisor then HyperKit make sure that you follow the generic minikube install instructions. Do not forget to install the correct driver for the chosen hypervisor if required.
+The basic instructions are provided in the [minikube install](https://kubernetes.io/docs/tasks/tools/install-minikube/) instructions.
+
+Check hypervisor Docker Desktop should have already installed HyperKit. In a terminal run: `hyperkit` to confirm. Any response other than `hyperkit: command not found` confirms that HyperKit is installed and on the path. If it is not found you can choose a different hypervisor or fix the Docker Desktop install.
+
+### Installing Minikube
+1. install minikube, you can either use brew or directly via these steps:
+ ```shell script
+ curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-amd64
+ chmod +x minikube
+ sudo mv minikube /usr/local/bin
+ ```
+1. install HyperKit driver (required), you can either use brew or directly via these steps:
+ ```shell script
+ curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-hyperkit
+ sudo install -o root -g wheel -m 4755 docker-machine-driver-hyperkit /usr/local/bin/
+ ```
+1. update the minikube config to default to the HyperKit install `minikube config set vm-driver hyperkit`
+1. change docker desktop to use minikube for Kubernetes:<br/>
+ 
+
+### Deploy and access the cluster
+After the installation is done you can start a new cluster.
+1. start the minikube cluster: `minikube start --kubernetes-version v1.14.2`
+1. start the minikube dashboard: `minikube dashboard &`
+
+### Build impact
+When you create images make sure that the build is run after pointing it to the right environment.
+Without setting the enviromnent minikube might not find the docker images when deploying the scheduler.
+1. make sure minikube is started
+1. in the terminal where you wll run the build execute: `eval $(minikube docker-env)`
+1. run the image build from the yunikorn-k8shim repository root: `make image`
+1. deploy the scheduler as per the normal instructions.
+
+## Debug code locally
+
+Note, this instruction requires you have GoLand IDE for development.
+
+In GoLand, go to yunikorn-k8shim project. Then click "Run" -> "Debug..." -> "Edit Configuration..." to get the pop-up configuration window.
+Note, you need to click "+" to create a new profile if the `Go Build` option is not available at the first time.
+
+
+
+The highlighted fields are the configurations you need to add. These include:
+
+- Run Kind: package
+- Package path: point to the path of `pkg/shim` package
+- Working directory: point to the path of the `conf` directory, this is where the program loads configuration file from
+- Program arguments: specify the arguments to run the program, such as `-kubeConfig=/path/to/.kube/config -interval=1s -clusterId=mycluster -clusterVersion=0.1 -name=yunikorn -policyGroup=queues -logEncoding=console -logLevel=-1`.
+Note, you need to replace `/path/to/.kube/config` with the local path to the kubeconfig file. And if you want to change or add more options, you can run `_output/bin/k8s-yunikorn-scheduler -h` to find out.
+
+Once the changes are done, click "Apply", then "Debug". You will need to set proper breakpoints in order to debug the program.
+
+## Access remote Kubernetes cluster
+
+This setup assumes you have already installed a remote Kubernetes cluster.
+For a generic view on how to access a multiple cluster and integrate it follow the [accessing multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) documentation from Kubernetes.
+
+Or follow these simplified steps:
+1. get the Kubernetes `config` file from remote cluster, copy it to the local machine and give it a unique name i.e. `config-remote`
+1. save the `KUBECONFIG` environment variable (if set)
+ ```shell script
+ export KUBECONFIG_SAVED=$KUBECONFIG
+ ```
+1. add the new file to the environment variable
+ ```shell script
+ export KUBECONFIG=$KUBECONFIG:config-remote
+ ```
+1. run the command `kubectl config view` to check that both configs can be accessed
+1. switch context using `kubectl config use-context my-remote-cluster`
+1. confirm that the current context is now switched to the remote cluster config:
+ ```text
+ kubectl config get-contexts
+ CURRENT NAME CLUSTER AUTHINFO NAMESPACE
+ docker-for-desktop docker-for-desktop-cluster docker-for-desktop
+ * my-remote-cluster kubernetes kubernetes-admin
+ ```
+
+More docs can be found [here](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)
diff --git a/versioned_docs/version-0.12.0/developer_guide/openshift_development.md b/versioned_docs/version-0.12.0/developer_guide/openshift_development.md
new file mode 100644
index 0000000..8d21171
--- /dev/null
+++ b/versioned_docs/version-0.12.0/developer_guide/openshift_development.md
@@ -0,0 +1,182 @@
+---
+id: openshift_development
+title: Development in CodeReady Containers
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn is tested against OpenShift and developers can set up their local environment to test patches against OpenShift.
+Our recommended local environment uses CodeReady containers.
+
+## Set up a running CRC cluster
+
+1. Download CodeReady Container binaries
+
+ Select your OS from the dropdown list then click on "Download" (On a Mac, you'll download crc-macos-amd64.tar.xz; on Linux, crc-linux-amd64.tar.xz).
+ You'll be asked to connect using your Red Hat login. If you don't have one, just click on "Create one now". You do *not* need a Red Hat subscription for this.
+
+ Once logged in, download CodeReady Containers binary and the pull secret.
+
+1. Unzip the tar file.
+
+ ```bash
+ tar -xvzf crc-macos-amd64.tar.xz
+ ```
+
+1. Move the crc binary under your path. Like
+
+ ```bash
+ sudo cp `pwd`/crc-macos-$CRCVERSION-amd64/crc /usr/local/bin
+ ```
+
+1. Configure CRC in accordance with your hardware capabilities.
+
+ ```bash
+ crc config set memory 16000
+ crc config set cpus 12
+ crc setup
+ ```
+1. Start the CRC and open the console.
+
+ ```bash
+ crc start --pull-secret-file pull-secret.txt
+ crc console
+ ```
+
+## Testing a patch
+
+The following steps assume you have a running CRC cluster in your laptop. Note that these steps are not tested against a remote CRC cluster.
+
+1. Access your environment through the `oc` command.
+
+ Type in the `crc oc-env` command to a shell.
+ ```bash
+ $ crc oc-env
+ export PATH="/Users/<user>/.crc/bin/oc:$PATH"
+ # Run this command to configure your shell:
+ # eval $(crc oc-env)
+ ```
+ So you need to type in this to access the `oc` comamnd:
+ ```
+ eval $(crc oc-env)
+ ```
+
+1. Log in to `oc`. After the CRC has started it will display a similar message:
+
+ ```
+ To access the cluster, first set up your environment by following 'crc oc-env' instructions.
+ Then you can access it by running 'oc login -u developer -p developer https://api.crc.testing:6443'.
+ To login as an admin, run 'oc login -u kubeadmin -p duduw-yPT9Z-hsUpq-f3pre https://api.crc.testing:6443'.
+ To access the cluster, first set up your environment by following 'crc oc-env' instructions.
+ ```
+
+ Use the `oc login -u kubeadmin ...` command.
+
+1. Get the URL of the local OpenShift cluster's internal private Docker repository by typing the command below.
+
+ ```bash
+ $ oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}'
+ default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+ By default it should be `default-route-openshift-image-registry.apps-crc.testing`. Change the steps above, if the displayed URL is different.
+
+1. Prepare the Docker images.
+
+ You can read more about this at the bottom, in the *Using custom images* section.
+
+1. Prepare the helm chart.
+
+ If you want to use custom Docker images, replace the images in the chart's `values.yaml` config file.
+
+ Note that if you manually pushed the Docker image to the `default-route-openshift-image-registry.apps-crc.testing` docker registry directly you need to have valid certs to access it.
+ On OpenShift there's service for this: `image-registry.openshift-image-registry.svc`, which is easier to use.
+
+ For example, if you want to override all of the three Docker images you should use the following configs:
+ ```yaml
+ image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn
+ tag: scheduler-latest
+ pullPolicy: Always
+
+ admission_controller_image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn
+ tag: admission-latest
+ pullPolicy: Always
+
+ web_image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn-web
+ tag: latest
+ pullPolicy: Always
+ ```
+
+ You can find it in the yunikorn-release repo's helm chart directory.
+
+1. Install the helm charts.
+
+ ```bash
+ helm install yunikorn . -n yunikorn
+ ```
+
+## Using custom images
+
+### Podman
+
+1. Log in into Podman using the following command.
+
+ ```bash
+ podman login --tls-verify=false -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+1. Build the image in the repository e.g. in shim using the generic `make image` command.
+
+1. Verify that the image is present in the repository.
+
+ ```bash
+ podman images
+ REPOSITORY TAG IMAGE ID CREATED SIZE
+ localhost/apache/yunikorn admission-latest 19eb41241d64 About a minute ago 53.5 MB
+ localhost/apache/yunikorn scheduler-latest e60e09b424d9 About a minute ago 543 MB
+ ```
+
+## Directly pushing OS Image Registry
+
+1. Create the images that you wish to replace.
+
+ You can either build new images locally or use official (maybe mix both).
+ * For the -shim and -web images checkout the repository (optionally make your changes) and type the following command:
+ ```bash
+ make clean image REGISTRY=default-route-openshift-image-registry.apps-crc.testing/<project>/<name>:<tag>
+ ```
+ Note that in OpenShift a project is equivalent a Kubernetes namespace. The `yunikorn` project/namespace is recommended.
+ * Using an official image is possible by, retagging it with by the `docker tag` command.
+ ```bash
+ docker tag apache/yunikorn:scheduler-latest default-route-openshift-image-registry.apps-crc.testing/yunikorn/yunikorn:scheduler-latest
+ ```
+
+1. Login to the Docker repository.
+ ```bash
+ docker login -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+1. Push the Docker images to the internal Docker repository
+ ```
+ docker push default-route-openshift-image-registry.apps-crc.testing/yunikorn/yunikorn:scheduler-latest
+ ```
diff --git a/versioned_docs/version-0.12.0/get_started/core_features.md b/versioned_docs/version-0.12.0/get_started/core_features.md
new file mode 100644
index 0000000..8f22589
--- /dev/null
+++ b/versioned_docs/version-0.12.0/get_started/core_features.md
@@ -0,0 +1,73 @@
+---
+id: core_features
+title: Features
+keywords:
+ - feature
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The main features of YuniKorn include:
+
+## App-aware scheduling
+One of the key differences of YuniKorn is, it does app-aware scheduling. In default K8s scheduler, it simply schedules
+pod by pod, without any context about user, app, queue. However, YuniKorn recognizes users, apps, queues, and it considers
+a lot more factors, e.g resource, ordering etc, while making scheduling decisions. This gives us the possibility to do
+fine-grained controls on resource quotas, resource fairness and priorities, which are the most important requirements
+for a multi-tenancy computing system.
+
+## Hierarchy Resource Queues
+
+Hierarchy queues provide an efficient mechanism to manage cluster resources. The hierarchy of the queues can logically
+map to the structure of an organization. This gives fine-grained control over resources for different tenants. The YuniKorn
+UI provides a centralised view to monitor the usage of resource queues, it helps you to get the insight how the resources are
+used across different tenants. What's more, By leveraging the min/max queue capacity, it can define how elastic it can be
+in terms of the resource consumption for each tenant.
+
+## Job Ordering and Queuing
+Applications can be properly queued in working-queues, the ordering policy determines which application can get resources first.
+The policy can be various, such as simple `FIFO`, `Fair`, `StateAware` or `Priority` based. Queues can maintain the order of applications,
+and based on different policies, the scheduler allocates resources to jobs accordingly. The behavior is much more predictable.
+
+What's more, when the queue max-capacity is configured, jobs and tasks can be properly queued up in the resource queue.
+If the remaining capacity is not enough, they can be waiting in line until some resources are released. This simplifies
+the client side operation. Unlike the default scheduler, resources are capped by namespace resource quotas,
+and that is enforced by the quota-admission-controller, if the underneath namespace has no enough quota, pods cannot be
+created. Client side needs complex logic, e.g retry by condition, to handle such scenarios.
+
+## Resource fairness
+In a multi-tenant environment, a lot of users are sharing cluster resources. To avoid tenants from competing resources
+and potential get starving. More fine-grained fairness needs to achieve fairness across users, as well as teams/organizations.
+With consideration of weights or priorities, some more important applications can get high demand resources that stand over its share.
+This is often associated with resource budget, a more fine-grained fairness mode can further improve the expense control.
+
+## Resource Reservation
+
+YuniKorn automatically does reservations for outstanding requests. If a pod could not be allocated, YuniKorn will try to
+reserve it on a qualified node and tentatively allocate the pod on this reserved node (before trying rest of nodes).
+This mechanism can avoid this pod gets starved by later submitted smaller, less-picky pods.
+This feature is important in the batch workloads scenario because when a large amount of heterogeneous pods is submitted
+to the cluster, it's very likely some pods can be starved even they are submitted much earlier.
+
+## Throughput
+Throughput is a key criterion to measure scheduler performance. It is critical for a large scale distributed system.
+If throughput is bad, applications may waste time on waiting for scheduling, and further impact service SLAs.
+When the cluster gets bigger, it also means the requirement of higher throughput. The [performance evaluation based on Kube-mark](performance/evaluate_perf_function_with_kubemark.md)
+reveals some perf numbers.
diff --git a/versioned_docs/version-0.12.0/get_started/get_started.md b/versioned_docs/version-0.12.0/get_started/get_started.md
new file mode 100644
index 0000000..aa526d0
--- /dev/null
+++ b/versioned_docs/version-0.12.0/get_started/get_started.md
@@ -0,0 +1,73 @@
+---
+id: user_guide
+title: Get Started
+slug: /
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Before reading this guide, we assume you either have a Kubernetes cluster, or a local Kubernetes dev environment, e.g MiniKube.
+It is also assumed that `kubectl` is on your path and properly configured.
+Follow this [guide](developer_guide/env_setup.md) on how to setup a local Kubernetes cluster using docker-desktop.
+
+## Install
+
+The easiest way to get started is to use our Helm Charts to deploy YuniKorn on an existing Kubernetes cluster.
+It is recommended to use Helm 3 or later versions.
+
+```shell script
+helm repo add yunikorn https://apache.github.io/incubator-yunikorn-release
+helm repo update
+kubectl create namespace yunikorn
+helm install yunikorn yunikorn/yunikorn --namespace yunikorn
+```
+
+By default, the helm chart will install the scheduler, web-server and the admission-controller in the cluster.
+When `admission-controller` is installed, it simply routes all traffic to YuniKorn. That means the resource scheduling
+is delegated to YuniKorn. You can disable it by setting `embedAdmissionController` flag to false during the helm install.
+
+Further configuration options for installing YuniKorn via Helm are available in the [YuniKorn Helm hub page](https://hub.helm.sh/charts/yunikorn/yunikorn).
+
+If you don't want to use helm charts, you can find our step-by-step
+tutorial [here](developer_guide/deployment.md).
+
+## Uninstall
+
+Run the following command to uninstall YuniKorn:
+```shell script
+helm uninstall yunikorn --namespace yunikorn
+```
+
+## Access the Web UI
+
+When the scheduler is deployed, the web UI is also deployed in a container.
+Port forwarding for the web interface on the standard port can be turned on via:
+
+```
+kubectl port-forward svc/yunikorn-service 9889:9889 -n yunikorn
+```
+
+`9889` is the default port for web UI.
+Once this is done, web UI will be available at: `http://localhost:9889`.
+
+
+
+YuniKorn UI provides a centralised view for cluster resource capacity, utilization, and all application info.
+
diff --git a/versioned_docs/version-0.12.0/performance/evaluate_perf_function_with_kubemark.md b/versioned_docs/version-0.12.0/performance/evaluate_perf_function_with_kubemark.md
new file mode 100644
index 0000000..f2c7090
--- /dev/null
+++ b/versioned_docs/version-0.12.0/performance/evaluate_perf_function_with_kubemark.md
@@ -0,0 +1,103 @@
+---
+id: evaluate_perf_function_with_kubemark
+title: Evaluate YuniKorn function & performance with Kubemark
+keywords:
+ - performance
+ - throughput
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+All the following tests are done with [Kubemark](https://github.com/kubernetes/kubernetes/blob/release-1.3/docs/devel/kubemark-guide.md#starting-a-kubemark-cluster),
+a tool helps us to simulate large K8s cluster and run experimental workloads.
+There were 18 bare-metal servers being used to simulate 2000/4000 nodes for these tests.
+
+## Scheduler Throughput
+
+When running Big Data batch workloads, e.g Spark, on K8s, scheduler throughput becomes to be one of the main concerns.
+In YuniKorn, we have done lots of optimizations to improve the performance, such as a fully async event-driven system
+and low-latency sorting policies. The following chart reveals the scheduler throughput (by using Kubemark simulated
+environment, and submitting 50,000 pods), comparing to the K8s default scheduler.
+
+
+
+The charts record the time spent until all pods are running on the cluster
+
+| | THROUGHPUT (pods/sec) | THROUGHPUT (pods/sec) |
+|----------------------- |:---------------------: |:---------------------: |
+| ENVIRONMENT (# nodes) | Default Scheduler | YuniKorn |
+| 2000 | 263 | 617 |
+| 4000 | 141 | 373 |
+
+## Resource Fairness between queues
+
+Each of YuniKorn queues has its guaranteed and maximum capacity. When we have lots of jobs submitted to these queues,
+YuniKorn ensures each of them gets its fair share. When we monitor the resource usage of these queues, we can clearly
+see how fairness was enforced:
+
+
+
+We set up 4 heterogeneous queues on this cluster, and submit different workloads against these queues.
+From the chart, we can see the queue resources are increasing nearly in the same trend, which means the resource
+fairness across queues is honored.
+
+## Node sorting policies
+
+There are 2 node sorting policies available in YuniKorn, with regarding the pod distributing flavors. One is *FAIR*,
+which tries best to evenly distribute pods to nodes; the other one is *BIN-PACKING*, which tries best to bin pack pods
+to less number of nodes. The former one is suitable for the Data Center scenarios, it helps to balance the stress of
+cluster nodes; the latter one is suitable to be used on Cloud, it can minimize the number of instances when working
+with auto-scaler, in order to save cost.
+
+### FAIR Policy
+
+We group nodes into 10 buckets, each bucket represents for the number of nodes that has a similar resource
+utilization (a range). To help you understand the chart, imagine the buckets have the following values at a certain
+point of time:
+
+| BUCKET | RESOURCE UTILIZATION RANGE | VALUE |
+|:--------: |:--------------------------: |:-----: |
+| bucket-0 | 0% - 10% | 100 |
+| bucket-1 | 10% - 20% | 300 |
+| ... | | |
+| bucket-9 | 90% - 100% | 0 |
+
+This means at the given time, this cluster has 100 nodes whose utilization is in the range 0% to 10%;
+it has 300 nodes whose utilization is in the range 10% - 20%, and so on… Now, we run lots of workloads and
+collect metrics, see the below chart:
+
+
+
+We can see all nodes have 0% utilization, and then all of them move to bucket-1, then bucket-2 … and eventually
+all nodes moved to bucket-9, which means all capacity is used. In another word, nodes’ resource has been used in
+a fairness manner.
+
+### BIN-PACKING
+
+This is When the bin-packing policy is enabled, we can see the following pattern:
+
+
+
+On the contrary, all nodes are moving between 2 buckets, bucket-0 and bucket-9. Nodes in bucket-0 (0% - 10%)
+are decreasing in a linear manner, and nodes in bucket-9 (90% - 100%) are increasing with the same curve.
+In other words, node resources are being used up one by one.
+
+
+
diff --git a/versioned_docs/version-0.12.0/performance/metrics.md b/versioned_docs/version-0.12.0/performance/metrics.md
new file mode 100644
index 0000000..9dedbec
--- /dev/null
+++ b/versioned_docs/version-0.12.0/performance/metrics.md
@@ -0,0 +1,109 @@
+---
+id: metrics
+title: Scheduler Metrics
+keywords:
+ - metrics
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn leverages [Prometheus](https://prometheus.io/) to record metrics. The metrics system keeps tracking of
+scheduler's critical execution paths, to reveal potential performance bottlenecks. Currently, there are three categories
+for these metrics:
+
+- scheduler: generic metrics of the scheduler, such as allocation latency, num of apps etc.
+- queue: each queue has its own metrics sub-system, tracking queue status.
+- event: record various changes of events in YuniKorn.
+
+all metrics are declared in `yunikorn` namespace.
+### Scheduler Metrics
+
+| Metrics Name | Metrics Type | Description |
+| --------------------- | ------------ | ------------ |
+| containerAllocation | Counter | Total number of attempts to allocate containers. State of the attempt includes `allocated`, `rejected`, `error`, `released`. Increase only. |
+| applicationSubmission | Counter | Total number of application submissions. State of the attempt includes `accepted` and `rejected`. Increase only. |
+| applicationStatus | Gauge | Total number of application status. State of the application includes `running` and `completed`. |
+| totalNodeActive | Gauge | Total number of active nodes. |
+| totalNodeFailed | Gauge | Total number of failed nodes. |
+| nodeResourceUsage | Gauge | Total resource usage of node, by resource name. |
+| schedulingLatency | Histogram | Latency of the main scheduling routine, in seconds. |
+| nodeSortingLatency | Histogram | Latency of all nodes sorting, in seconds. |
+| appSortingLatency | Histogram | Latency of all applications sorting, in seconds. |
+| queueSortingLatency | Histogram | Latency of all queues sorting, in seconds. |
+| tryNodeLatency | Histogram | Latency of node condition checks for container allocations, such as placement constraints, in seconds, in seconds. |
+
+### Queue Metrics
+
+| Metrics Name | Metrics Type | Description |
+| ------------------------- | ------------- | ----------- |
+| appMetrics | Counter | Application Metrics, record the total number of applications. State of the application includes `accepted`,`rejected` and `Completed`. |
+| usedResourceMetrics | Gauge | Queue used resource. |
+| pendingResourceMetrics | Gauge | Queue pending resource. |
+| availableResourceMetrics | Gauge | Used resource metrics related to queues etc. |
+
+### Event Metrics
+
+| Metrics Name | Metrics Type | Description |
+| ------------------------ | ------------ | ----------- |
+| totalEventsCreated | Gauge | Total events created. |
+| totalEventsChanneled | Gauge | Total events channeled. |
+| totalEventsNotChanneled | Gauge | Total events not channeled. |
+| totalEventsProcessed | Gauge | Total events processed. |
+| totalEventsStored | Gauge | Total events stored. |
+| totalEventsNotStored | Gauge | Total events not stored. |
+| totalEventsCollected | Gauge | Total events collected. |
+
+## Access Metrics
+
+YuniKorn metrics are collected through Prometheus client library, and exposed via scheduler restful service.
+Once started, they can be accessed via endpoint http://localhost:9080/ws/v1/metrics.
+
+## Aggregate Metrics to Prometheus
+
+It's simple to setup a Prometheus server to grab YuniKorn metrics periodically. Follow these steps:
+
+- Setup Prometheus (read more from [Prometheus docs](https://prometheus.io/docs/prometheus/latest/installation/))
+
+- Configure Prometheus rules: a sample configuration
+
+```yaml
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['docker.for.mac.host.internal:9080']
+```
+
+- start Prometheus
+
+```shell script
+docker pull prom/prometheus:latest
+docker run -p 9090:9090 -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus
+```
+
+Use `docker.for.mac.host.internal` instead of `localhost` if you are running Prometheus in a local docker container
+on Mac OS. Once started, open Prometheus web UI: http://localhost:9090/graph. You'll see all available metrics from
+YuniKorn scheduler.
+
diff --git a/versioned_docs/version-0.12.0/performance/performance_tutorial.md b/versioned_docs/version-0.12.0/performance/performance_tutorial.md
new file mode 100644
index 0000000..2d34025
--- /dev/null
+++ b/versioned_docs/version-0.12.0/performance/performance_tutorial.md
@@ -0,0 +1,452 @@
+---
+id: performance_tutorial
+title: Benchmarking Tutorial
+keywords:
+ - performance
+ - tutorial
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Overview
+
+The YuniKorn community continues to optimize the performance of the scheduler, ensuring that YuniKorn satisfies the performance requirements of large-scale batch workloads. Thus, the community has built some useful tools for performance benchmarking that can be reused across releases. This document introduces all these tools and steps to run them.
+
+## Hardware
+
+Be aware that performance result is highly variable depending on the underlying hardware. All results published in the doc can only be used as references. We encourage each individual to run similar tests on their own environments in order to get a result based on your own hardware. This doc is just for demonstration purpose.
+
+A list of servers being used in this test are (Huge thanks to [National Taichung University of Education](http://www.ntcu.edu.tw/newweb/index.htm), [Kuan-Chou Lai](http://www.ntcu.edu.tw/kclai/) for providing these servers for running tests):
+
+| Manchine Type | CPU | Memory | Download/upload(Mbps) |
+| --------------------- | --- | ------ | --------------------- |
+| HP | 16 | 36G | 525.74/509.86 |
+| HP | 16 | 30G | 564.84/461.82 |
+| HP | 16 | 30G | 431.06/511.69 |
+| HP | 24 | 32G | 577.31/576.21 |
+| IBM blade H22 | 16 | 38G | 432.11/4.15 |
+| IBM blade H22 | 16 | 36G | 714.84/4.14 |
+| IBM blade H22 | 16 | 42G | 458.38/4.13 |
+| IBM blade H22 | 16 | 42G | 445.42/4.13 |
+| IBM blade H22 | 16 | 32G | 400.59/4.13 |
+| IBM blade H22 | 16 | 12G | 499.87/4.13 |
+| IBM blade H23 | 8 | 32G | 468.51/4.14 |
+| WS660T | 8 | 16G | 87.73/86.30 |
+| ASUSPRO D640MB_M640SA | 4 | 8G | 92.43/93.77 |
+| PRO E500 G6_WS720T | 16 | 8G | 90/87.18 |
+| WS E500 G6_WS720T | 8 | 40G | 92.61/89.78 |
+| E500 G5 | 8 | 8G | 91.34/85.84 |
+| WS E500 G5_WS690T | 12 | 16G | 92.2/93.76 |
+| WS E500 G5_WS690T | 8 | 32G | 91/89.41 |
+| WS E900 G4_SW980T | 80 | 512G | 89.24/87.97 |
+
+The following steps are needed for each server, otherwise the large scale testing may fail due to the limited number of users/processes/open-files.
+
+### 1. Set /etc/sysctl.conf
+```
+kernel.pid_max=400000
+fs.inotify.max_user_instances=50000
+fs.inotify.max_user_watches=52094
+```
+### 2. Set /etc/security/limits.conf
+
+```
+* soft nproc 4000000
+* hard nproc 4000000
+root soft nproc 4000000
+root hard nproc 4000000
+* soft nofile 50000
+* hard nofile 50000
+root soft nofile 50000
+root hard nofile 50000
+```
+---
+
+## Deploy workflow
+
+Before going into the details, here are the general steps used in our tests:
+
+- [Step 1](#Kubernetes): Properly configure Kubernetes API server and controller manager, then add worker nodes.
+- [Step 2](#Setup-Kubemark): Deploy hollow pods,which will simulate worker nodes, name hollow nodes. After all hollow nodes in ready status, we need to cordon all native nodes, which are physical presence in the cluster, not the simulated nodes, to avoid we allocated test workload pod to native nodes.
+- [Step 3](#Deploy-YuniKorn): Deploy YuniKorn using the Helm chart on the master node, and scale down the Deployment to 0 replica, and [modify the port](#Setup-Prometheus) in `prometheus.yml` to match the port of the service.
+- [Step 4](#Run-tests): Deploy 50k Nginx pods for testing, and the API server will create them. But since the YuniKorn scheduler Deployment has been scaled down to 0 replica, all Nginx pods will be stuck in pending.
+- [Step 5](../user_guide/trouble_shooting.md#restart-the-scheduler): Scale up The YuniKorn Deployment back to 1 replica, and cordon the master node to avoid YuniKorn allocating Nginx pods there. In this step, YuniKorn will start collecting the metrics.
+- [Step 6](#Collect-and-Observe-YuniKorn-metrics): Observe the metrics exposed in Prometheus UI.
+---
+
+## Setup Kubemark
+
+[Kubemark](https://github.com/kubernetes/kubernetes/tree/master/test/kubemark) is a performance testing tool which allows users to run experiments on simulated clusters. The primary use case is the scalability testing. The basic idea is to run tens or hundreds of fake kubelet nodes on one physical node in order to simulate large scale clusters. In our tests, we leverage Kubemark to simulate up to a 4K-node cluster on less than 20 physical nodes.
+
+### 1. Build image
+
+##### Clone kubernetes repo, and build kubemark binary file
+
+```
+git clone https://github.com/kubernetes/kubernetes.git
+```
+```
+cd kubernetes
+```
+```
+KUBE_BUILD_PLATFORMS=linux/amd64 make kubemark GOFLAGS=-v GOGCFLAGS="-N -l"
+```
+
+##### Copy kubemark binary file to the image folder and build kubemark docker image
+
+```
+cp _output/bin/kubemark cluster/images/kubemark
+```
+```
+IMAGE_TAG=v1.XX.X make build
+```
+After this step, you can get the kubemark image which can simulate cluster node. You can upload it to Docker-Hub or just deploy it locally.
+
+### 2. Install Kubermark
+
+##### Create kubemark namespace
+
+```
+kubectl create ns kubemark
+```
+
+##### Create configmap
+
+```
+kubectl create configmap node-configmap -n kubemark --from-literal=content.type="test-cluster"
+```
+
+##### Create secret
+
+```
+kubectl create secret generic kubeconfig --type=Opaque --namespace=kubemark --from-file=kubelet.kubeconfig={kubeconfig_file_path} --from-file=kubeproxy.kubeconfig={kubeconfig_file_path}
+```
+### 3. Label node
+
+We need to label all native nodes, otherwise the scheduler might allocate hollow pods to other simulated hollow nodes. We can leverage Node selector in yaml to allocate hollow pods to native nodes.
+
+```
+kubectl label node {node name} tag=tagName
+```
+
+### 4. Deploy Kubemark
+
+The hollow-node.yaml is down below, there are some parameters we can configure.
+
+```
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: hollow-node
+ namespace: kubemark
+spec:
+ replicas: 2000 // the node number you want to simulate
+ selector:
+ name: hollow-node
+ template:
+ metadata:
+ labels:
+ name: hollow-node
+ spec:
+ nodeSelector: // leverage label to allocate to native node
+ tag: tagName
+ initContainers:
+ - name: init-inotify-limit
+ image: docker.io/busybox:latest
+ imagePullPolicy: IfNotPresent
+ command: ['sysctl', '-w', 'fs.inotify.max_user_instances=200'] // set as same as max_user_instance in actual node
+ securityContext:
+ privileged: true
+ volumes:
+ - name: kubeconfig-volume
+ secret:
+ secretName: kubeconfig
+ - name: logs-volume
+ hostPath:
+ path: /var/log
+ containers:
+ - name: hollow-kubelet
+ image: 0yukali0/kubemark:1.20.10 // the kubemark image you build
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 4194
+ - containerPort: 10250
+ - containerPort: 10255
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ command:
+ - /kubemark
+ args:
+ - --morph=kubelet
+ - --name=$(NODE_NAME)
+ - --kubeconfig=/kubeconfig/kubelet.kubeconfig
+ - --alsologtostderr
+ - --v=2
+ volumeMounts:
+ - name: kubeconfig-volume
+ mountPath: /kubeconfig
+ readOnly: true
+ - name: logs-volume
+ mountPath: /var/log
+ resources:
+ requests: // the resource of hollow pod, can modify it.
+ cpu: 20m
+ memory: 50M
+ securityContext:
+ privileged: true
+ - name: hollow-proxy
+ image: 0yukali0/kubemark:1.20.10 // the kubemark image you build
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ command:
+ - /kubemark
+ args:
+ - --morph=proxy
+ - --name=$(NODE_NAME)
+ - --use-real-proxier=false
+ - --kubeconfig=/kubeconfig/kubeproxy.kubeconfig
+ - --alsologtostderr
+ - --v=2
+ volumeMounts:
+ - name: kubeconfig-volume
+ mountPath: /kubeconfig
+ readOnly: true
+ - name: logs-volume
+ mountPath: /var/log
+ resources: // the resource of hollow pod, can modify it.
+ requests:
+ cpu: 20m
+ memory: 50M
+ tolerations:
+ - effect: NoExecute
+ key: node.kubernetes.io/unreachable
+ operator: Exists
+ - effect: NoExecute
+ key: node.kubernetes.io/not-ready
+ operator: Exists
+```
+
+once done editing, apply it to the cluster:
+
+```
+kubectl apply -f hollow-node.yaml
+```
+
+---
+
+## Deploy YuniKorn
+
+#### Install YuniKorn with helm
+
+We can install YuniKorn with Helm, please refer to this [doc](https://yunikorn.apache.org/docs/#install).
+We need to tune some parameters based on the default configuration. We recommend to clone the [release repo](https://github.com/apache/incubator-yunikorn-release) and modify the parameters in `value.yaml`.
+
+```
+git clone https://github.com/apache/incubator-yunikorn-release.git
+cd helm-charts/yunikorn
+```
+
+#### Configuration
+
+The modifications in the `value.yaml` are:
+
+- increased memory/cpu resources for the scheduler pod
+- disabled the admission controller
+- set the app sorting policy to FAIR
+
+please see the changes below:
+
+```
+resources:
+ requests:
+ cpu: 14
+ memory: 16Gi
+ limits:
+ cpu: 14
+ memory: 16Gi
+```
+```
+embedAdmissionController: false
+```
+```
+configuration: |
+ partitions:
+ -
+ name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ queues:
+ -
+ name: sandbox
+ properties:
+ application.sort.policy: fair
+```
+
+#### Install YuniKorn with local release repo
+
+```
+Helm install yunikorn . --namespace yunikorn
+```
+
+---
+
+## Setup Prometheus
+
+YuniKorn exposes its scheduling metrics via Prometheus. Thus, we need to set up a Prometheus server to collect these metrics.
+
+### 1. Download Prometheus release
+
+```
+wget https://github.com/prometheus/prometheus/releases/download/v2.30.3/prometheus-2.30.3.linux-amd64.tar.gz
+```
+```
+tar xvfz prometheus-*.tar.gz
+cd prometheus-*
+```
+
+### 2. Configure prometheus.yml
+
+```
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['docker.for.mac.host.internal:9080']
+ // 9080 is internal port, need port forward or modify 9080 to service's port
+```
+
+### 3. Launch Prometheus
+```
+./prometheus --config.file=prometheus.yml
+```
+
+---
+## Run tests
+
+Once the environment is setup, you are good to run workloads and collect results. YuniKorn community has some useful tools to run workloads and collect metrics, more details will be published here.
+
+---
+
+## Collect and Observe YuniKorn metrics
+
+After Prometheus is launched, YuniKorn metrics can be easily collected. Here is the [docs](metrics.md) of YuniKorn metrics. YuniKorn tracks some key scheduling metrics which measure the latency of some critical scheduling paths. These metrics include:
+
+ - **scheduling_latency_seconds:** Latency of the main scheduling routine, in seconds.
+ - **app_sorting_latency_seconds**: Latency of all applications sorting, in seconds.
+ - **node_sorting_latency_seconds**: Latency of all nodes sorting, in seconds.
+ - **queue_sorting_latency_seconds**: Latency of all queues sorting, in seconds.
+ - **container_allocation_attempt_total**: Total number of attempts to allocate containers. State of the attempt includes `allocated`, `rejected`, `error`, `released`. Increase only.
+
+you can select and generate graph on Prometheus UI easily, such as:
+
+
+
+
+---
+
+## Performance Tuning
+
+### Kubernetes
+
+The default K8s setup has limited concurrent requests which limits the overall throughput of the cluster. In this section, we introduced a few parameters that need to be tuned up in order to increase the overall throughput of the cluster.
+
+#### kubeadm
+
+Set pod-network mask
+
+```
+kubeadm init --pod-network-cidr=10.244.0.0/8
+```
+
+#### CNI
+
+Modify CNI mask and resources.
+
+```
+ net-conf.json: |
+ {
+ "Network": "10.244.0.0/8",
+ "Backend": {
+ "Type": "vxlan"
+ }
+ }
+```
+```
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "200Mi"
+ limits:
+ cpu: "100m"
+ memory: "200Mi"
+```
+
+
+#### Api-Server
+
+In the Kubernetes API server, we need to modify two parameters: `max-mutating-requests-inflight` and `max-requests-inflight`. Those two parameters represent the API request bandwidth. Because we will generate a large amount of pod request, we need to increase those two parameters. Modify `/etc/kubernetes/manifest/kube-apiserver.yaml`:
+
+```
+--max-mutating-requests-inflight=3000
+--max-requests-inflight=3000
+```
+
+#### Controller-Manager
+
+In the Kubernetes controller manager, we need to increase the value of three parameters: `node-cidr-mask-size`, `kube-api-burst` and `kube-api-qps`. `kube-api-burst` and `kube-api-qps` control the server side request bandwidth. `node-cidr-mask-size` represents the node CIDR. it needs to be increased as well in order to scale up to thousands of nodes.
+
+
+Modify `/etc/kubernetes/manifest/kube-controller-manager.yaml`:
+
+```
+--node-cidr-mask-size=21 //log2(max number of pods in cluster)
+--kube-api-burst=3000
+--kube-api-qps=3000
+```
+
+#### kubelet
+
+In single worker node, we can run 110 pods as default. But to get higher node resource utilization, we need to add some parameters in Kubelet launch command, and restart it.
+
+Modify start arg in `/etc/systemd/system/kubelet.service.d/10-kubeadm.conf`, add `--max-Pods=300` behind the start arg and restart
+
+```
+systemctl daemon-reload
+systemctl restart kubelet
+```
+
+---
+
+## Summary
+
+With Kubemark and Prometheus, we can easily run benchmark testing, collect YuniKorn metrics and analyze the performance. This helps us to identify the performance bottleneck in the scheduler and further eliminate them. The YuniKorn community will continue to improve these tools in the future, and continue to gain more performance improvements.
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/performance/profiling.md b/versioned_docs/version-0.12.0/performance/profiling.md
new file mode 100644
index 0000000..4050ccd
--- /dev/null
+++ b/versioned_docs/version-0.12.0/performance/profiling.md
@@ -0,0 +1,122 @@
+---
+id: profiling
+title: Profiling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Use [pprof](https://github.com/google/pprof) to do CPU, Memory profiling can help you understand the runtime status of YuniKorn scheduler. Profiling instruments have been
+added to YuniKorn rest service, we can easily retrieve and analyze them from HTTP
+endpoints.
+
+## CPU profiling
+
+At this step, ensure you already have YuniKorn running, it can be either running from
+local via a `make run` command, or deployed as a pod running inside of K8s. Then run
+
+```
+go tool pprof http://localhost:9080/debug/pprof/profile
+```
+
+The profile data will be saved on local file system, once that is done, it enters into
+the interactive mode. Now you can run profiling commands, such as
+
+```
+(pprof) top
+Showing nodes accounting for 14380ms, 44.85% of 32060ms total
+Dropped 145 nodes (cum <= 160.30ms)
+Showing top 10 nodes out of 106
+ flat flat% sum% cum cum%
+ 2130ms 6.64% 6.64% 2130ms 6.64% __tsan_read
+ 1950ms 6.08% 12.73% 1950ms 6.08% __tsan::MetaMap::FreeRange
+ 1920ms 5.99% 18.71% 1920ms 5.99% __tsan::MetaMap::GetAndLock
+ 1900ms 5.93% 24.64% 1900ms 5.93% racecall
+ 1290ms 4.02% 28.67% 1290ms 4.02% __tsan_write
+ 1090ms 3.40% 32.06% 3270ms 10.20% runtime.mallocgc
+ 1080ms 3.37% 35.43% 1080ms 3.37% __tsan_func_enter
+ 1020ms 3.18% 38.62% 1120ms 3.49% runtime.scanobject
+ 1010ms 3.15% 41.77% 1010ms 3.15% runtime.nanotime
+ 990ms 3.09% 44.85% 990ms 3.09% __tsan::DenseSlabAlloc::Refill
+```
+
+you can type command such as `web` or `gif` to get a graph that helps you better
+understand the overall performance on critical code paths. You can get something
+like below:
+
+
+
+Note, in order to use these
+options, you need to install the virtualization tool `graphviz` first, if you are using Mac, simply run `brew install graphviz`, for more info please refer [here](https://graphviz.gitlab.io/).
+
+## Memory Profiling
+
+Similarly, you can run
+
+```
+go tool pprof http://localhost:9080/debug/pprof/heap
+```
+
+this will return a snapshot of current heap which allows us to check memory usage.
+Once it enters the interactive mode, you can run some useful commands. Such as
+top can list top memory consumption objects.
+```
+(pprof) top
+Showing nodes accounting for 83.58MB, 98.82% of 84.58MB total
+Showing top 10 nodes out of 86
+ flat flat% sum% cum cum%
+ 32MB 37.84% 37.84% 32MB 37.84% github.com/apache/incubator-yunikorn-core/pkg/cache.NewClusterInfo
+ 16MB 18.92% 56.75% 16MB 18.92% github.com/apache/incubator-yunikorn-core/pkg/rmproxy.NewRMProxy
+ 16MB 18.92% 75.67% 16MB 18.92% github.com/apache/incubator-yunikorn-core/pkg/scheduler.NewScheduler
+ 16MB 18.92% 94.59% 16MB 18.92% github.com/apache/incubator-yunikorn-k8shim/pkg/dispatcher.init.0.func1
+ 1.04MB 1.23% 95.81% 1.04MB 1.23% k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypeWithName
+ 0.52MB 0.61% 96.43% 0.52MB 0.61% github.com/gogo/protobuf/proto.RegisterType
+ 0.51MB 0.61% 97.04% 0.51MB 0.61% sync.(*Map).Store
+ 0.50MB 0.6% 97.63% 0.50MB 0.6% regexp.onePassCopy
+ 0.50MB 0.59% 98.23% 0.50MB 0.59% github.com/json-iterator/go.(*Iterator).ReadString
+ 0.50MB 0.59% 98.82% 0.50MB 0.59% text/template/parse.(*Tree).newText
+```
+
+you can also run `web`, `pdf` or `gif` command to get the graph for heap.
+
+## Download profiling samples and analyze it locally
+
+We have included essential go/go-tool binaries in scheduler docker image, you should be able to do some basic profiling
+analysis inside of the docker container. However, if you want to dig into some issues, it might be better to do the analysis
+locally. Then you need to copy the samples file to local environment first. The command to copy files is like following:
+
+```
+kubectl cp ${SCHEDULER_POD_NAME}:${SAMPLE_PATH_IN_DOCKER_CONTAINER} ${LOCAL_COPY_PATH}
+```
+
+for example
+
+```
+kubectl cp yunikorn-scheduler-cf8f8dd8-6szh5:/root/pprof/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+once you get the file in your local environment, then you can run the `pprof` command for analysis.
+
+```
+go tool pprof /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+## Resources
+
+* pprof Document https://github.com/google/pprof/tree/master/doc.
diff --git a/versioned_docs/version-0.12.0/user_guide/acls.md b/versioned_docs/version-0.12.0/user_guide/acls.md
new file mode 100644
index 0000000..c1fd7f2
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/acls.md
@@ -0,0 +1,119 @@
+---
+id: acls
+title: ACLs
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+:::info
+User information is passed to the core scheduler from the kubernetes shim using the methodology defined [here](usergroup_resolution)
+:::
+
+## Usage
+Access Control Lists are generic for YuniKorn.
+They can be used in multiple places in YuniKorn.
+The current use case is limited to queue ACLs.
+
+Access control lists give access to the users and groups that have been specified in the list.
+They do not provide the possibility to explicitly remove or deny access to the users and groups specified in the list.
+
+If there is no access control list is configured access is *denied* by default.
+
+## Syntax
+The access control list is defined as:
+```
+ACL ::= “*” | userlist [ “ “ grouplist ]
+userlist ::= “” | user { “,” user }
+grouplist ::= “” | group { “,” group }
+```
+
+This definition specifies a wildcard of * which results in access for everyone.
+
+If the user list is empty and the group list is empty nobody will have access.
+This deny all ACL has two possible representations:
+* an empty access control list. (implicit)
+* a single space. (explicit)
+
+## Example config
+
+### Simple examples
+An ACL that allows access to just the user `sue`
+```yaml
+ adminacl: sue
+```
+Nobody else will get access, this is just for `sue`.
+`john` and `bob` will be denied access.
+
+An ACL that allows access to the user `sue` and the members of the group `dev`.
+```yaml
+ adminacl: sue dev
+```
+The user `sue` gets access based on her explicit mention in the user part of the ACL.
+Even though she is not a member of the group `dev`. Her group membership is irrelevant.
+
+The user named `john` whom is a member of the group `dev` will be allowed access based on his group membership.
+A third user called `bob` whom is not mentioned explicitly and is not a member of the `dev` group will be denied access.
+
+An ACL that allows access to the members of the groups `dev` and `test`.
+```yaml
+ adminacl: " dev,test"
+```
+The ACL must start with a space to indicate that there is no user list.
+If the ACL is not correctly quoted the space is dropped by the yaml parser.
+Since the user list is empty none of the users will get access unless they are a member of either the `dev` or `test` group.
+
+Looking at the same three users as before:
+The user `sue` is not a member of either group and is denied access.
+The user named `john` whom is a member of the group `dev` will be allowed access based on his group membership.
+`bob` is not a member of the `dev` group but is a member of `test` and will be allowed access.
+
+### Escaping and quotation marks
+ACLs are currently implemented in the queue configuration which uses a yaml file.
+This places some limitations on the how to escape the values.
+Incorrectly quoted values will cause a yaml parse error or could lead to the incorrect interpretation of the value.
+
+The following points need to be taken into account:
+1. The wildcard entry must be quoted in the yaml config.
+1. A simple list of users with or without it being followed by a list of groups does not need quoting but may be quoted.
+1. An ACL without a user list and just one or more groups must be quoted to find the starting space:
+
+Correctly quoted ACL example
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: test
+ submitacl: "*"
+ adminacl: sue dev,test
+ - name: product
+ submitacl: " product"
+```
+
+## Access check
+The access check follows the pattern:
+* check if the ACL is the wildcard
+* check if the user is in the user list
+* check if any of the groups the user is a member of is part of the group list
+
+If a check matches the ACL allows access and checking is stopped.
+If none of the checks match the ACL denies access.
+
+## User and Group information
+For User & Group resolution, please follow instructions defined [here](usergroup_resolution)
diff --git a/versioned_docs/version-0.12.0/user_guide/gang_scheduling.md b/versioned_docs/version-0.12.0/user_guide/gang_scheduling.md
new file mode 100644
index 0000000..47b5722
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/gang_scheduling.md
@@ -0,0 +1,285 @@
+---
+id: gang_scheduling
+title: Gang Scheduling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## What is Gang Scheduling
+
+When Gang Scheduling is enabled, YuniKorn schedules the app only when
+the app’s minimal resource request can be satisfied. Otherwise, apps
+will be waiting in the queue. Apps are queued in hierarchy queues,
+with gang scheduling enabled, each resource queue is assigned with the
+maximum number of applications running concurrently with min resource guaranteed.
+
+
+
+## Enable Gang Scheduling
+
+There is no cluster-wide configuration needed to enable Gang Scheduling.
+The scheduler actively monitors the metadata of each app, if the app has included
+a valid taskGroups definition, it will be considered as gang scheduling desired.
+
+:::info Task Group
+A task group is a “gang” of tasks in an app, these tasks are having the same resource profile
+and the same placement constraints. They are considered as homogeneous requests that can be
+treated as the same kind in the scheduler.
+:::
+
+### Prerequisite
+
+For the queues which runs gang scheduling enabled applications, the queue sorting policy needs to be set either
+`FIFO` or `StateAware`. To configure queue sorting policy, please refer to doc: [app sorting policies](user_guide/sorting_policies.md#Application_sorting).
+
+:::info Why FIFO based sorting policy?
+When Gang Scheduling is enabled, the scheduler proactively reserves resources
+for each application. If the queue sorting policy is not FIFO based (StateAware is FIFO based sorting policy),
+the scheduler might reserve partial resources for each app and causing resource segmentation issues.
+:::
+
+### App Configuration
+
+On Kubernetes, YuniKorn discovers apps by loading metadata from individual pod, the first pod of the app
+is required to enclosed with a full copy of app metadata. If the app doesn’t have any notion about the first or second pod,
+then all pods are required to carry the same taskGroups info. Gang scheduling requires taskGroups definition,
+which can be specified via pod annotations. The required fields are:
+
+| Annotation | Value |
+|----------------------------------------------- |--------------------- |
+| yunikorn.apache.org/task-group-name | Task group name, it must be unique within the application |
+| yunikorn.apache.org/task-groups | A list of task groups, each item contains all the info defined for the certain task group |
+| yunikorn.apache.org/schedulingPolicyParameters | Optional. A arbitrary key value pairs to define scheduling policy parameters. Please read [schedulingPolicyParameters section](#scheduling-policy-parameters) |
+
+#### How many task groups needed?
+
+This depends on how many different types of pods this app requests from K8s. A task group is a “gang” of tasks in an app,
+these tasks are having the same resource profile and the same placement constraints. They are considered as homogeneous
+requests that can be treated as the same kind in the scheduler. Use Spark as an example, each job will need to have 2 task groups,
+one for the driver pod and the other one for the executor pods.
+
+#### How to define task groups?
+
+The task group definition is a copy of the app’s real pod definition, values for fields like resources, node-selector
+and toleration should be the same as the real pods. This is to ensure the scheduler can reserve resources with the
+exact correct pod specification.
+
+#### Scheduling Policy Parameters
+
+Scheduling policy related configurable parameters. Apply the parameters in the following format in pod's annotation:
+
+```yaml
+annotations:
+ yunikorn.apache.org/schedulingPolicyParameters: "PARAM1=VALUE1 PARAM2=VALUE2 ..."
+```
+
+Currently, the following parameters are supported:
+
+`placeholderTimeoutInSeconds`
+
+Default value: *15 minutes*.
+This parameter defines the reservation timeout for how long the scheduler should wait until giving up allocating all the placeholders.
+The timeout timer starts to tick when the scheduler *allocates the first placeholder pod*. This ensures if the scheduler
+could not schedule all the placeholder pods, it will eventually give up after a certain amount of time. So that the resources can be
+freed up and used by other apps. If non of the placeholders can be allocated, this timeout won't kick-in. To avoid the placeholder
+pods stuck forever, please refer to [troubleshooting](trouble_shooting.md#gang-scheduling) for solutions.
+
+` gangSchedulingStyle`
+
+Valid values: *Soft*, *Hard*
+
+Default value: *Soft*.
+This parameter defines the fallback mechanism if the app encounters gang issues due to placeholder pod allocation.
+See more details in [Gang Scheduling styles](#gang-scheduling-styles) section
+
+More scheduling parameters will added in order to provide more flexibility while scheduling apps.
+
+#### Example
+
+The following example is a yaml file for a job. This job launches 2 pods and each pod sleeps 30 seconds.
+The notable change in the pod spec is *spec.template.metadata.annotations*, where we defined `yunikorn.apache.org/task-group-name`
+and `yunikorn.apache.org/task-groups`.
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: gang-scheduling-job-example
+spec:
+ completions: 2
+ parallelism: 2
+ template:
+ metadata:
+ labels:
+ app: sleep
+ applicationId: "gang-scheduling-job-example"
+ queue: root.sandbox
+ annotations:
+ yunikorn.apache.org/task-group-name: task-group-example
+ yunikorn.apache.org/task-groups: |-
+ [{
+ "name": "task-group-example",
+ "minMember": 2,
+ "minResource": {
+ "cpu": "100m",
+ "memory": "50M"
+ },
+ "nodeSelector": {},
+ "tolerations": []
+ }]
+ spec:
+ schedulerName: yunikorn
+ restartPolicy: Never
+ containers:
+ - name: sleep30
+ image: "alpine:latest"
+ command: ["sleep", "30"]
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50M"
+```
+
+When this job is submitted to Kubernetes, 2 pods will be created using the same template, and they all belong to one taskGroup:
+*“task-group-example”*. YuniKorn will create 2 placeholder pods, each uses the resources specified in the taskGroup definition.
+When all 2 placeholders are allocated, the scheduler will bind the the real 2 sleep pods using the spot reserved by the placeholders.
+
+You can add more than one taskGroups if necessary, each taskGroup is identified by the taskGroup name,
+it is required to map each real pod with a pre-defined taskGroup by setting the taskGroup name. Note,
+the task group name is only required to be unique within an application.
+
+### Enable Gang scheduling for Spark jobs
+
+Each Spark job runs 2 types of pods, driver and executor. Hence, we need to define 2 task groups for each job.
+The annotations for the driver pod looks like:
+
+```yaml
+Annotations:
+ yunikorn.apache.org/schedulingPolicyParameters: “placeholderTimeoutSeconds=30”
+ yunikorn.apache.org/taskGroupName: “spark-driver”
+ yunikorn.apache.org/taskGroup: “
+ TaskGroups: [
+ {
+ Name: “spark-driver”
+ minMember: 1,
+ minResource: {
+ Cpu: 1,
+ Memory: 2Gi
+ },
+ Node-selector: ...
+ Tolerations: ...
+ },
+ {
+ Name: “spark-executor”,
+ minMember: 10,
+ minResource: {
+ Cpu: 1,
+ Memory: 2Gi
+ }
+ }
+ ]
+ ”
+```
+
+:::note
+Spark driver and executor pod has memory overhead, that needs to be considered in the taskGroup resources.
+:::
+
+For all the executor pods,
+
+```yaml
+Annotations:
+ # the taskGroup name should match to the names
+ # defined in the taskGroups field
+ yunikorn.apache.org/taskGroupName: “spark-executor”
+```
+
+Once the job is submitted to the scheduler, the job won’t be scheduled immediately.
+Instead, the scheduler will ensure it gets its minimal resources before actually starting the driver/executors.
+
+## Gang scheduling Styles
+
+There are 2 gang scheduling styles supported, Soft and Hard respectively. It can be configured per app-level to define how the app will behave in case the gang scheduling fails.
+
+- `Hard style`: when this style is used, we will have the initial behavior, more precisely if the application cannot be scheduled according to gang scheduling rules, and it times out, it will be marked as failed, without retrying to schedule it.
+- `Soft style`: when the app cannot be gang scheduled, it will fall back to the normal scheduling, and the non-gang scheduling strategy will be used to achieve the best-effort scheduling. When this happens, the app transits to the Resuming state and all the remaining placeholder pods will be cleaned up.
+
+**Default style used**: `Soft`
+
+**Enable a specific style**: the style can be changed by setting in the application definition the ‘gangSchedulingStyle’ parameter to Soft or Hard.
+
+#### Example
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: gang-app-timeout
+spec:
+ completions: 4
+ parallelism: 4
+ template:
+ metadata:
+ labels:
+ app: sleep
+ applicationId: gang-app-timeout
+ queue: fifo
+ annotations:
+ yunikorn.apache.org/task-group-name: sched-style
+ yunikorn.apache.org/schedulingPolicyParameters: "placeholderTimeoutInSeconds=60 gangSchedulingStyle=Hard"
+ yunikorn.apache.org/task-groups: |-
+ [{
+ "name": "sched-style",
+ "minMember": 4,
+ "minResource": {
+ "cpu": "1",
+ "memory": "1000M"
+ },
+ "nodeSelector": {},
+ "tolerations": []
+ }]
+ spec:
+ schedulerName: yunikorn
+ restartPolicy: Never
+ containers:
+ - name: sleep30
+ image: "alpine:latest"
+ imagePullPolicy: "IfNotPresent"
+ command: ["sleep", "30"]
+ resources:
+ requests:
+ cpu: "1"
+ memory: "1000M"
+
+```
+
+## Verify Configuration
+
+To verify if the configuration has been done completely and correctly, check the following things:
+1. When an app is submitted, verify the expected number of placeholders are created by the scheduler.
+If you define 2 task groups, 1 with minMember 1 and the other with minMember 5, that means we are expecting 6 placeholder
+gets created once the job is submitted.
+2. Verify the placeholder spec is correct. Each placeholder needs to have the same info as the real pod in the same taskGroup.
+Check field including: namespace, pod resources, node-selector, and toleration.
+3. Verify the placeholders can be allocated on correct type of nodes, and verify the real pods are started by replacing the placeholder pods.
+
+## Troubleshooting
+
+Please see the troubleshooting doc when gang scheduling is enabled [here](trouble_shooting.md#gang-scheduling).
diff --git a/versioned_docs/version-0.12.0/user_guide/placement_rules.md b/versioned_docs/version-0.12.0/user_guide/placement_rules.md
new file mode 100644
index 0000000..5f2c64d
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/placement_rules.md
@@ -0,0 +1,354 @@
+---
+id: placement_rules
+title: App Placement Rules
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The basics for the placement rules are described in the [scheduler configuration design document](design/scheduler_configuration.md#placement-rules-definition).
+Multiple rules can be chained to form a placement policy.
+[Access control lists](user_guide/acls.md) and rule filters are defined per rule and enforced per rule.
+This document explains how to build a policy, including the rule usage, that is part of the scheduler with examples.
+
+## Configuration
+Rules are defined per partition as part of the scheduler queue configuration.
+The order that the rules are defined in is the order in which they are executed.
+If a rule matches the policy will stop executing the remaining rules.
+
+A matching rule generates a fully qualified queue name.
+This means that the name returned starts at the _root_ queue.
+There is no limit on the number of levels in the queue hierarchy that can be generated.
+
+When a rule is executed the result of rules that have been executed is unknown and not taken into account.
+Similar for rule that have not been executed yet: rules cannot influence other rules except when they are configured as the [parent](#parent-parameter) rule.
+
+If the policy does not generate a queue name and no more rules are left the application will be rejected.
+
+Basic structure for the rule placement definition in the configuration:
+```yaml
+placementrules:
+ - name: <name of the 1st rule>
+ - name: <name of the 2nd rule>
+```
+Each rule can take a predefined set of parameters in the configuration.
+The name of the rules that can be used are given in the [rule](#rules) description.
+A rule name is not case sensitive in the configuration.
+Rule name must follow the following naming convention:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _
+
+A rule that is not known, i.e. the name does not map to the rules defined below, will cause a initialisation error of the placement manager.
+Rules can also throw a parse error or an error during the initialisation if the parameters are incorrect.
+A rule set with an error can never become active.
+
+A placement manager is considered initialised if it has an active rule set.
+When the configuration is reloaded a new rule set will be created to replace the active rule set.
+In the case that a new rule set loaded contains an error, i.e. is broken, the placement manager ignores the new rule set.
+This means that the placement manager stays in a the state it was in when a broken rule set is loaded.
+If the placement manager keeps using the existing active rule set in the case that it was already initialised.
+A message will be logged about the broken and ignored configuration.
+
+Dots "." in the rule result are replaced by the string "\_dot_".
+A dot is replaced because it is used as the hierarchy separator in the fully qualified queue name.
+Replacing the dot occurs before the full queue hierarchy is build and the result is qualified.
+This means that we allow user name and or tag values to contain dots without the dots affecting the queue hierarchy.
+For queues in the configuration that as an example must map to username with a dot you must specify them as follows:
+A user rule with the user `user.name` will generate the queue name `root.user_dot_name` as output.
+If that "user queue" must be added to the configuration the `user_dot_name` name should be used.
+
+### Create parameter
+The create parameter is a boolean flag that defines if a queue that is generated by the rule may be created if it does not exist.
+There is no guarantee that the queue will be created because the existing queues might prevent the queue creation.
+If the queue generated by the rule does not exist and the flag is not set to _true_ the result of the rule will be a fail.
+
+Basic yaml entry for a rule with `create` flag:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ create: true
+```
+
+The default value is _false_.
+Allowed values: _true_ or _false_, any other value will cause a parse error.
+
+### Parent parameter
+The parent parameter allows specifying a rule that generates a parent queue for the current rule.
+Parent rules can be nested, a parent rule _may_ contain another parent rule.
+There is no enforced limit of parent rules that can be nested.
+
+A parent rule is treated as if it was a rule specified at the top level of the list and thus has the same parameters and requirements as a any other rule in the placement definition.
+The exception is that using a parent rule on a rule that already generates a fully qualified queue is considered a configuration error.
+This error can only occur on the rule of type [fixed](#fixed-rule), see the rule specification for more details.
+
+NOTE: the rule execution traverses down the list of parent rules and executes the last one in the list first.
+This means that the last parent rule will generate the queue directly below the root.
+See the examples for details.
+
+Basic yaml entry for a rule with a `parent` rule:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ parent:
+ name: <name of the parent rule>
+```
+
+The default value is _no_ parent.
+
+### Filter parameter
+The filter on a rule allows filtering the users that the rule applies to.
+A filter is a complex configuration object.
+
+The _users_ and _groups_ that can be configured can be one of two types:
+* a regular expression
+* a list of users or groups.
+
+If the entry for users or groups contains more than 1 entry in the yaml it is always considered a list of either users or groups.
+Duplicate entries in the lists are ignored and do not cause an error.
+Specifying a regular expression beside other list elements is not allowed.
+
+User and group names follow the standard linux user and group conventions.
+For user names:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _ . @ -
+* the last character may also be a $
+
+For group names:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _ -
+
+If the list is exactly one entry it can be either a single user or group or a regular expression.
+When an entry contains a character that is not allowed in a user or group name the entry is considered a regular expression.
+The regular expression must compile as specified.
+A regular expression that does not compile is ignored.
+
+Specifically for the group regular expression: matching is performed one group at a time not the against the list of groups.
+
+Basic `filter` yaml entry:
+```yaml
+filter:
+ type: deny
+ users:
+ - <user name or regexp>
+ - <user name>
+ groups:
+ - <group name or regexp>
+ - <group name>
+```
+
+The default value is _no_ filter.
+
+### Value parameter
+This is a generic value that can be used to pass to a rule to implement or alter its behaviour.
+The value It is used by the [fixed](#fixed-rule) and the [tag](#tag-rule) rule.
+The value is a single value in string form and is not interpreted or manipulated by the system.
+
+Basic yaml entry for a rule with a `value` set:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ value: "any string"
+```
+
+The default is _no_ value.
+
+## Access Control List
+Access control lists are not defined in the rules but they impact the outcome of the placement policy.
+Two access control lists can be defined on a queue:
+1. Submit ACL: _submitacl_
+1. Administration ACL: _adminacl_
+
+The placement rule will only match if the ACL of the queue allows submit access via either ACL.
+The administrative queue ACL also provides _submit_ access.
+If the queue does not exist or does not have an ACL set, the ACL of the parent queue is checked.
+This recursive check is repeated until an ACL provides access or after the ACL of the root queue is checked.
+
+For more detail on the ACL syntax check the [ACL documentation](user_guide/acls.md).
+
+## Rules
+### Provided Rule
+Name to be used in the configuration: *provided*
+
+Returns the queue provided during the submission of the application.
+The behaviour of the this rule is to fully qualify the queue provided by the application if the queue is not fully qualified.
+If a parent rule is set and the queue provided in the application submission is fully qualified then the parent rule will not be executed.
+
+Supported parameters:
+* create
+* parent
+* filter
+
+Example: create the queue passed in by the user if it does not exist under the users name
+```yaml
+placementrules:
+ - name: provided
+ create: true
+ parent:
+ name: user
+ create: true
+```
+Application submit request by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.developer.my_special_queue` (parent rule set the user name)
+
+Application submit request by the user `developer`, queue in the application on submit: `root.dev_queue`<br/>
+Result: `root.dev_queue` (parent rule ignored)
+
+### User Name Rule
+Name to be used in the configuration: *user*
+
+Returns the queue based on the user name that is part of the submitted application.
+
+Supported parameters:
+* create
+* parent
+* filter
+
+Example: submit to a queue based on the user name, do not create the queue if the queue does not exist:
+```yaml
+placementrules:
+ - name: user
+ create: false
+```
+
+Application submit request by the user `finance.test`, queue does exist:<br/>
+Result: `root.finance_dot_test` (note the replacement of the dot)
+
+Application submit request by the user `developer`, queue does not exist:<br/>
+Result: failed, next rule executed
+
+### Fixed Rule
+Name to be used in the configuration: *fixed*
+
+Returns the name configured in the rule parameter _value_.
+The value configured must be a legal queue name or queue hierarchy.
+The name does not have to be a fully qualified queue name.
+The hierarchy in the name uses a dot as a separator for the queue names at the different levels in the hierarchy.
+The fixed rule can only fail if the queue configured does not exist and the create flag is not set as it will always return the configured queue.
+
+If the configured value is a fully qualified queue name configuring a parent rule will be considered an _error_.
+
+Supported parameters:
+* value (required)
+* create
+* parent
+* filter
+
+Example: a fixed queue returned always, without the `create` flag set and thus requires the queue to be defined as a leaf queue in the configuration.
+```yaml
+placementrules:
+ - name: fixed
+ value: last_resort
+```
+
+Application submit request by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.last_resort`
+
+### Tag Rule
+Name to be used in the configuration: *tag*
+Retrieves the queue name from the applications tags.
+The tag name that is checked for its value is configured in the rule using the `value`.
+Configuring a tag rule without a `value` set is an error, however an application does not have to have the tag set.
+
+If the tag is not set on the application the rule fails.
+If the tag value returned from the application is a fully qualified queue name the parent rule, if configured, will not be executed.
+
+Supported parameters:
+* value (required)
+* create
+* parent
+* filter
+
+Example: place an application based on the kubernetes `namespace` which gets sets automatically in the application when submitted:
+```yaml
+placementrules:
+ - name: tag
+ value: namespace
+ create: true
+```
+
+Application submit request for a kubernetes based application in the namespace `default` by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.default`
+
+Application submit request for a kubernetes based application in the namespace `testing` by the user `developer`<br/>
+Result: `root.testing`
+
+Application submit request for a non kubernetes based application by the user `developer`<br/>
+Result: failed, next rule executed
+
+## Complex examples
+In this complex example we chain three rules:
+1. a `user` rule, with a parent rule `tag` using the kubernetes namespace, to be used only for users that are part of and "dev" group.
+1. a `tag` rule using the kubernetes namespace, with a parent rule `fixed` using the existing queue `root.namespaces`.
+1. a `fixed` rule to place everything that reaches this point in the `root.default` queue.
+
+Example:
+```yaml
+placementrules:
+ - name: user
+ create: true
+ filter:
+ type: allow
+ groups:
+ - dev*
+ parent:
+ - name: tag
+ value: namespace
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ - name: fixed
+ value: root.namespaces
+ filter:
+ type: allow
+ users:
+ - john
+ - name: fixed
+ value: root.default
+```
+
+Application submit request for a kubernetes based application in the namespace `testing` by the user `john`<br/>
+Result: `root.namespaces.testing` (matched in rule 2)
+
+Application submit request for a kubernetes based application in the namespace `newapp` by the user `sarah` with groups membership `sarah, test_app, dev_app`<br/>
+Result: `root.newapp.sarah` (matched in rule 1)
+
+Application submit request for a kubernetes based application in the namespace `testapp` by the user `bob` with groups membership `bob`<br/>
+Result: `root.deault` (matched in rule 3)
+
+In this second example we chain two rules:
+1. a `fixed` rule to place everything in the `root.production` queue
+1. a `user` rule, with the create flag set
+
+In this case however we have ACLs set up on the `root.production` queue to only allow two specific user to use this queue.
+So even if the rule matches unless the user is either `john` or `bob` the application will not be placed in the `production` queue.
+All other users will match the second rule and use their own queue, which is created if it does not exist.
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: production
+ submitacl: john,bob
+ placementrules:
+ - name: fixed
+ value: root.production
+ - name: user
+ create: true
+```
diff --git a/versioned_docs/version-0.12.0/user_guide/queue_config.md b/versioned_docs/version-0.12.0/user_guide/queue_config.md
new file mode 100644
index 0000000..26dd0f7
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/queue_config.md
@@ -0,0 +1,353 @@
+---
+id: queue_config
+title: Partition and Queue Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The basis for the queue configuration is given in the [configuration design document](design/scheduler_configuration.md).
+
+This document provides the generic queue configuration.
+It references both the [Access Control Lists](user_guide/acls.md) and [Placement rules](user_guide/placement_rules.md) documentation.
+
+This document explains how to create the partition and queue configuration for the scheduler with examples.
+
+The scheduler relies on the shim to reliably provide user information as part of the application submission.
+The current shim identifies the user and the groups the user belongs to using the methodology provided in [User & Group Resolution](usergroup_resolution)
+
+## Configuration
+The configuration file for the scheduler that is described here only provides the configuration for the partitions and queues.
+
+By default we use the file called `queues.yaml` in our deployments.
+The filename can be changed via the command line flag `policyGroup` of the scheduler.
+Changing the filename must be followed by corresponding changes in the deployment details, either the `configmap` or the file included in the docker container.
+
+The example file for the configuration is located in the scheduler core's [queues.yaml](https://github.com/apache/incubator-yunikorn-core/blob/master/config/queues.yaml).
+
+## Partitions
+Partitions are the top level of the scheduler configuration.
+There can be more than one partition defined in the configuration.
+
+Basic structure for the partition definition in the configuration:
+```yaml
+partitions:
+ - name: <name of the 1st partition>
+ - name: <name of the 2nd partition>
+```
+The default name for the partition is `default`.
+The partition definition contains the full configuration for the scheduler for a particular shim.
+Each shim uses its own unique partition.
+
+The partition must have at least the following keys defined:
+* name
+* [queues](#queues)
+
+The queues configuration is explained below.
+
+Optionally the following keys can be defined for a partition:
+* [placementrules](#placement-rules)
+* [limits](#limits)
+* nodesortpolicy
+* preemption
+
+Placement rules and limits are explained in their own chapters
+
+The `nodesortpolicy` defines the way the nodes are sorted for the partition.
+Details on the values that can be used are in the [sorting policy](sorting_policies.md#node-sorting) documentation.
+
+The preemption key can currently have only one sub key: _enabled_.
+This boolean value defines the preemption behaviour for the whole partition.
+
+The default value for _enabled_ is _false_.
+Allowed values: _true_ or _false_, any other value will cause a parse error.
+
+Example `partition` yaml entry with _preemption_ flag set and a `nodesortpolicy` of _fair_:
+```yaml
+partitions:
+ - name: <name of the partition>
+ nodesortpolicy: fair
+ preemption:
+ enabled: true
+```
+NOTE:
+Currently the Kubernetes unique shim does not support any other partition than the `default` partition..
+This has been logged as an [jira](https://issues.apache.org/jira/browse/YUNIKORN-22) for the shim.
+
+### Queues
+
+YuniKorn manages resources by leveraging resource queues. The resource queue has the following characters:
+- queues can have **hierarchical** structure
+- each queue can be preset with **min/max capacity** where min-capacity defines the guaranteed resource and the max-capacity defines the resource limit (aka resource quota)
+- tasks must be running under a certain leaf queue
+- queues can be **static** (loading from configuration file) or **dynamical** (internally managed by YuniKorn)
+- queue level **resource fairness is** enforced by the scheduler
+- a job can only run under a specific queue
+
+:::info
+The difference between YuniKorn queue and [Kubernetes namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/):
+Kubernetes namespace provides the scope for the Kubernetes resources, including the security context (i.e who can access the objects), and resource
+boundary when [resource quota](https://kubernetes.io/docs/concepts/policy/resource-quotas/) is defined (i.e how many resources can be used by the objects).
+On the other hand, YuniKorn queue is only used how many resources can be used by a group of jobs, and in which order. It provides
+a more fine-grained control on resource sharing across multiple tenants with considering of resource fairness, job ordering, etc. In most of the cases,
+YuniKorn queue can be used to replace the namespace resource quota, in order to provide more scheduling features.
+:::
+
+The _queues_ entry is the main configuration element.
+It defines a hierarchical structure for the queues.
+
+It can have a `root` queue defined but it is not a required element.
+If the `root` queue is not defined the configuration parsing will insert the root queue for consistency.
+The insertion of the root queue is triggered by:
+* If the configuration has more than one queue defined at the top level a root queue is inserted.
+* If there is only one queue defined at the top level and it is not called `root` a root queue is inserted.
+
+The defined queue or queues will become a child queue of the inserted `root` queue.
+
+Basic `queues` yaml entry with sub queue:
+```yaml
+queues:
+- name: <name of the queue>
+ queues:
+ - name: <name of the queue>
+```
+
+Supported parameters for the queues:
+* name
+* parent
+* queues
+* properties
+* adminacl
+* submitacl
+* [resources](#resources)
+* [limits](#limits)
+
+Each queue must have a _name_.
+The name of a queue must be unique at the level that the queue is defined.
+Since the queue structure is fully hierarchical queues at different points in the hierarchy may have the same name.
+As an example: the queue structure `root.testqueue` and `root.parent.testqueue` is a valid structure.
+A queue cannot contain a dot "." character as that character is used to separate the queues in the hierarchy.
+If the name is not unique for the queue in the configuration or contains a dot a parsing error is generated and the configuration is rejected.
+
+Queues in the structure will automatically get a type assigned.
+The type of the queue is based on the fact that the queue has children or sub queues.
+The two types of queues are:
+* parent
+* leaf
+
+Applications can only be assigned to a _leaf_ queue.
+A queue that has a child or sub queue in the configuration is automatically a _parent_ queue.
+If a queue does not have a sub the queue in the configuration it is a _leaf_ queue, unless the `parent` parameter is set to _true_.
+Trying to override a _parent_ queue type in the configuration will cause a parsing error of the configuration.
+
+Sub queues for a parent queue are defined under the `queues` entry.
+The `queues` entry is a recursive entry for a queue level and uses the exact same set of parameters.
+
+The `properties` parameter is a simple key value pair list.
+The list provides a simple set of properties for the queue.
+There are no limitations on the key or value values, anything is allowed.
+Currently, the property list is only used in the scheduler to define a [sorting order](sorting_policies.md#application-sorting) for a leaf queue.
+Future expansions, like the option to turn on or off preemption on a queue or other sorting policies, would use this same property construct without the need to change the configuration.
+
+Access to a queue is set via the `adminacl` for administrative actions and for submitting an application via the `submitacl` entry.
+ACLs are documented in the [Access control lists](acls.md) document.
+
+Queue resource limits are set via the `resources` parameter.
+User and group limits are set via the `limits` parameter.
+As both entries are complex configuration entries they are explained in [resources](#resources) and [limits](#limits) below.
+
+An example configuration of a queue `root.namespaces` as a _parent_ queue with limits:
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: namespaces
+ parent: true
+ resources:
+ guaranteed:
+ {memory: 1000, vcore: 10}
+ max:
+ {memory: 10000, vcore: 100}
+```
+
+### Placement rules
+
+The placement rules are defined and documented in the [placement rule](placement_rules.md) document.
+
+Each partition can have only one set of placement rules defined.
+If no rules are defined the placement manager is not started and each application *must* have a queue set on submit.
+
+### Limits
+Limits define a set of limit objects for a partition or queue.
+It can be set on either the partition or on a queue at any level.
+```yaml
+limits:
+ - limit: <description>
+ - limit: <description>
+```
+
+A limit object is a complex configuration object.
+It defines one limit for a set of users and or groups.
+Multiple independent limits can be set as part of one `limits` entry on a queue or partition.
+Users and or groups that are not part of the limit setting will not be limited.
+
+A sample limits entry:
+```yaml
+limits:
+ - limit: <description>
+ users:
+ - <user name or "*"">
+ - <user name>
+ groups:
+ - <group name or "*"">
+ - <group name>
+ maxapplications: <1..maxint>
+ maxresources:
+ <resource name 1>: <0..maxint>
+ <resource name 2>: <0..maxint>
+```
+
+Limits are applied recursively in the case of a queue limit.
+This means that a limit on the `root` queue is an overall limit in the cluster for the user or group.
+A `root` queue limit is thus also equivalent with the `partition` limit.
+
+The limit object parameters:
+* limit
+* users
+* groups
+* maxapplications
+* maxresources
+
+The _limit_ parameter is an optional description of the limit entry.
+It is not used for anything but making the configuration understandable and readable.
+
+The _users_ and _groups_ that can be configured can be one of two types:
+* a star "*"
+* a list of users or groups.
+
+If the entry for users or groups contains more than one (1) entry it is always considered a list of either users or groups.
+The star "*" is the wildcard character and matches all users or groups.
+Duplicate entries in the lists are ignored and do not cause a parsing error.
+Specifying a star beside other list elements is not allowed.
+
+_maxapplications_ is an unsigned integer value, larger than 1, which allows you to limit the number of running applications for the configured user or group.
+Specifying a zero maximum applications limit is not allowed as it would implicitly deny access.
+Denying access must be handled via the ACL entries.
+
+The _maxresources_ parameter can be used to specify a limit for one or more resources.
+The _maxresources_ uses the same syntax as the [resources](#resources) parameter for the queue.
+Resources that are not specified in the list are not limited.
+A resource limit can be set to 0.
+This prevents the user or group from requesting the specified resource even though the queue or partition has that specific resource available.
+Specifying an overall resource limit of zero is not allowed.
+This means that at least one of the resources specified in the limit must be greater than zero.
+
+If a resource is not available on a queue the maximum resources on a queue definition should be used.
+Specifying a limit that is effectively zero, _maxapplications_ is zero and all resource limits are zero, is not allowed and will cause a parsing error.
+
+A limit is per user or group.
+It is *not* a combined limit for all the users or groups together.
+
+As an example:
+```yaml
+limit: "example entry"
+maxapplications: 10
+users:
+- sue
+- bob
+```
+In this case both the users `sue` and `bob` are allowed to run 10 applications.
+
+### Resources
+The resources entry for the queue can set the _guaranteed_ and or _maximum_ resources for a queue.
+Resource limits are checked recursively.
+The usage of a leaf queue is the sum of all assigned resources for that queue.
+The usage of a parent queue is the sum of the usage of all queues, leafs and parents, below the parent queue.
+
+The root queue, when defined, cannot have any resource limit set.
+If the root queue has any limit set a parsing error will occur.
+The maximum resource limit for the root queue is automatically equivalent to the cluster size.
+There is no guaranteed resource setting for the root queue.
+
+Maximum resources when configured place a hard limit on the size of all allocations that can be assigned to a queue at any point in time.
+A maximum resource can be set to 0 which makes the resource not available to the queue.
+Guaranteed resources are used in calculating the share of the queue and during allocation.
+It is used as one of the inputs for deciding which queue to give the allocation to.
+Preemption uses the _guaranteed_ resource of a queue as a base which a queue cannot go below.
+
+Basic `resources` yaml entry:
+```yaml
+resources:
+ guaranteed:
+ <resource name 1>: <0..maxint>
+ <resource name 2>: <0..maxint>
+ max:
+ <resource name 1>: <0..maxint>
+ <resource name 2>: <0..maxint>
+```
+Resources that are not specified in the list are not limited, for max resources, or guaranteed in the case of guaranteed resources.
+
+### Child Template
+
+The parent queue can provide a template to define the behaviour of dynamic leaf queues below it. A parent queue having no child template inherits the child template
+from its parent if that parent has one defined. A child template can be defined at any level in the queue hierarchy on a queue that is of the type parent.
+
+The supported configuration in template are shown below.
+1. application sort policy
+2. max resources
+3. guaranteed resources
+4. max applications
+
+As an example:
+```yaml
+ partitions:
+ - name: default
+ placementrules:
+ - name: provided
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ childtemplate:
+ maxapplications: 10
+ properties:
+ application.sort.policy: stateaware
+ resources:
+ guaranteed:
+ vcore: 1000
+ memory: 1000
+ max:
+ vcore: 20000
+ memory: 600000
+ queues:
+ - name: parent
+ parent: true
+ childtemplate:
+ resources:
+ max:
+ vcore: 21000
+ memory: 610000
+ - name: notemplate
+ parent: true
+```
+In this case, `root.parent.sales` will directly use the child template of parent queue `root.parent`. By contrast,
+`root.notemplate.sales` will use the child template set on the queue `root` since its parent queue `root.notemplate` inherits the child template from the queue `root`.
+
+[DEPRECATED] Please migrate to template if your cluster is relying on old behavior that dynamic leaf queue can inherit
+`application.sort.policy` from parent (introduced by [YUNIKORN-195](https://issues.apache.org/jira/browse/YUNIKORN-195)). The old behavior will get removed in the future release.
diff --git a/versioned_docs/version-0.12.0/user_guide/resource_quota_mgmt.md b/versioned_docs/version-0.12.0/user_guide/resource_quota_mgmt.md
new file mode 100644
index 0000000..8890fdb
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/resource_quota_mgmt.md
@@ -0,0 +1,324 @@
+---
+id: resource_quota_management
+title: Resource Quota Management
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Quota configuration and rules
+YuniKorn can offer a finer grained resource quota management setup compared to the simple namespace resource quota provided by Kubernetes.
+
+On Kubernetes a pod must fit into the namespace quota when the pod is submitted.
+If the pod does not fit in the namespace quota the pod is rejected.
+The client must implement a retry-mechanism and re-submit the pod if it needs the pod to be scheduled.
+
+Contrary to quotas in Kubernetes YuniKorn does not enforce quotas on submission but only on actively consumed resources.
+To explain the difference: when using YuniKorn for quota enforcement a new pod submitted to Kubernetes is always accepted.
+Yunikorn will queue the pod without counting the queued pod's resources towards the consumed quota.
+When YuniKorn tries to schedule the pod it checks at scheduling time if the pod fits in the quota configured for the queue the pod is assigned to.
+If at that point the pod does not fit in the quota the pod is skipped and not counted in the resource consumption.
+This means that until a scheduling attempt of a pod is successful a pod it is not consuming resources in the YuniKorn quota system.
+
+Resource quotas in YuniKorn are linked to the queue and its place in the queue hierarchy.
+The base of the queue structure, the `root` queue, does not allow setting a quota as it reflects the current size of the cluster.
+Node additions and removals update the `root` queue quota automatically.
+
+Beside the `root` queue the quotas can be set, and is enforced, at any point in the hierarchy.
+Every queue can have a quota set. The quota is enforced recursively throughout the hierarchy.
+This means that a child queue can never use more resources than the **configured** quota of the parent queue.
+Setting a quota on a child queue larger than its parent queue's quota would thus not have any effect and is handled as a configuration error.
+
+In the hierarchy there are some further rules that need to be considered.
+If a parent queue has multiple children the sum of the **usage** of all children combined can never exceed the quota **configured** on the parent.
+However, from a configuration perspective this does not mean that the sum of the **configured** quotas for all children must be smaller than the parent quota.
+
+
+
+As an example the `root.parent` queue has a quota of 900.
+It contains three child queues, two with a quota set.
+The `root.parent.child1` has no quota set and will thus be limited to the `root.parent` quota.
+The two other queues `root.parent.child2` and `root.parent.child3` each have a quota of 750 set.
+During normal operation the total usage of the 3 child queues together will be 900.
+The applications running in each child queue have a demand of more than 1000 each.
+
+Distribution in that case could be any of:
+* all 900 used by just the `child1` queue
+* spread out evenly over the 3 queues (300 by each)
+* `child2` maxed out using 750, and the left over 150 used by `child3`
+
+The exact distribution between the queues will fluctuate and is dependent on the scheduling policies.
+
+## Converting Kubernetes resources and quotas
+Resource support for pods is limited to the resources specified as part of the _requests_ specification:
+* _cpu_ is mapped to _vcore_ with the value in milli cpu.
+* _memory_ is mapped to _memory_ with the value in MB (1 MB = 10^6 B = 1 000 000 B).
+* all other resources are mapped as provided.
+
+Extended resource as per the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) are supported.
+
+Example pod with a single container:
+```yaml
+apiVersion: v1
+kind: Pod
+spec:
+ containers:
+ - name: container-1
+ resources:
+ requests:
+ cpu: "250m"
+ memory: "1Gi"
+ hugepages-1Gi: "1"
+```
+The above specification will set pod resources request for scheduling in YuniKorn to:
+* _vcore_ -> 250
+* _memory_ -> 1074
+* _hugepages-1Gi_ -> 1
+
+Two remarks:
+Multiple container specifications will be aggregated into one total pod resource request automatically.
+All memory is reported in MB with unit conversions applied where needed.
+
+In the case that static queue definitions are used for a queue there is no limit on the type of resource that can be specified in a quota.
+Quota annotations on namespaces, used as part of the automatic queue creation, are limited to the equivalent _cpu_ and _memory_ resources.
+See the [setup](#Namespace-quota) below for the annotations on the namespace for quotas.
+
+## Kubernetes and YuniKorn quota interaction
+The recommendation is to turn off, not configure, the Kubernetes Namespace quotas.
+Using only YuniKorn queue quotas provides a more flexible setup and allows queueing of workloads.
+
+In a setup that has both YuniKorn and Kubernetes quotas turned on consider the following points:
+* Two separate configurations need to be maintained.
+ This increases the maintenance burden, and the possibility of configuration mistakes.
+* Both quotas will be enforced.
+
+Having both quotas turned on can lead to unexpected behaviour.
+The main issue is the fact that the Kubernetes namespace quota is enforced on submit.
+There are three combinations of quota configuration that are possible.
+The 3 combinations could have two effects when used in combination with the YuniKorn quota.
+
+1. Both quotas are _equal_: workloads will not be queued, the full configured quota can be used.
+ - Maximum usage and queueing will be limited to the set quota
+2. Kubernetes quota is _lower_ than YuniKorn: the YuniKorn quota will never be reached and workloads will not be queued.
+ - Maximum usage will be limited to the Kubernetes quota.
+3. Kubernetes quota is _higher_ than YuniKorn: YuniKorn will limit the usage to the quota set in YuniKorn.
+ The Kubernetes quota will be enforced on submit and thus set the limit for the workload that can be queued on top of the YuniKorn quota.
+ - Maximum usage will be limited to the YuniKorn quota.
+ - Workload queueing will be limited to the Kubernetes quota.
+
+:::note
+The following configuration examples are just to demonstrate the format needed
+to create a queue hierarchy with quotas set.
+:::
+
+## Static queue definition
+
+### Goal
+A preconfigured hierarchy of queues with a maximum and guaranteed capacity.
+The users can only submit applications to the leaf queues.
+This approach manages the resource capacity for each of the queues, which is suitable to the scenarios that queues do not change too often.
+
+### Configuration
+Apply the following configuration to YuniKorn's configmap to:
+* setup 3 queues under `root`
+* each queue has a specific guaranteed and maximum capacity
+* anyone can submit to any queue
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ queues:
+ - name: advertisement
+ resources:
+ guaranteed:
+ memory: 500000
+ vcore: 50000
+ max:
+ memory: 800000
+ vcore: 80000
+ - name: search
+ resources:
+ guaranteed:
+ memory: 400000
+ vcore: 40000
+ max:
+ memory: 600000
+ vcore: 60000
+ - name: sandbox
+ resources:
+ guaranteed:
+ memory: 100000
+ vcore: 10000
+ max:
+ memory: 100000
+ vcore: 10000
+```
+
+### Run a workload
+In order to run applications in specific queues, you will need to set the following labels in all pod specs.
+All pods with the same `applicationID` label are considered ti be one application.
+In the below example the application `my-test-app` will run in the queue `root.sandbox`:
+
+```yaml
+labels:
+ app: my-test-app
+ applicationId: "my-test-app-01"
+ queue: root.sandbox
+```
+
+## Namespace to queue mapping
+
+### Goal
+Automatically map a Kubernetes `namespace` to a queue in YuniKorn.
+The user creates the required namespaces in Kubernetes.
+The YuniKorn k8s shim and core scheduler automatically pass the required information and map the namespace to a queue, creating the queue if it does not exist.
+The resource quota will be managed by YuniKorn instead of using the Kubernetes namespace quota.
+This does require the namespaces to be setup without Kubernetes quota enforcement and tags as per the [setup](#Namespace-quota) below.
+
+### Configuration
+Apply the following configuration to YuniKorn's configmap:
+
+```yaml
+partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ properties:
+ application.sort.policy: stateaware
+```
+
+This configuration places an application based on the `tag` rule.
+The tag selected is the `namespace` tag which is automatically added by the k8s shim to all applications that get created.
+The `create` flag is set to true which will trigger the creation of the queue with the same name as the namespace if it does not exist.
+
+Applications within the automatically created child queues will be sorted based sorting policy set on the parent queue.
+In this case the property `application.sort.policy` is in this configuration set to `stateaware`.
+This is a simple app sorting policy applicable for batch jobs, you can find more document [here](sorting_policies.md#StateAwarePolicy).
+
+You can change the configuration using the helm charts during the installation by overwriting the configuration in the
+[helm chart template](https://github.com/apache/incubator-yunikorn-release/blob/master/helm-charts/yunikorn/values.yaml#L71-L81).
+
+### Namespace quota
+Namespaces in Kubernetes contain the quota information.
+If a quota is set on a namespace Kubernetes will automatically enforce the quota.
+In the case that YuniKorn is used for quota enforcement no quota must be set on the namespace.
+
+To allow specifying a quota on the namespace the following annotations should be set in the namespace object:
+```yaml
+yunikorn.apache.org/namespace.max.cpu: "64"
+yunikorn.apache.org/namespace.max.memory: "100Gi"
+```
+YuniKorn will parse these annotations and set the maximum capacity of the queue mapped to this namespace.
+The values specified follow the standard Kubernetes formatting and unit specification.
+Currently, we only support mapping memory and cpu not other resource types.
+
+The example above will limit the queue mapped to the annotated namespace to 64 CPUs and 100GB memory.
+
+### Run a workload
+
+Applications, and the pods that are part of the application, can be submitted without specific labels.
+YuniKorn will automatically add the required tags.
+The configured placement rule will create the queue, if required, and add the application to the queue.
+
+For example, if an application is submitted to namespace `development`, then the application will run in the `root.development` queue.
+
+## Parent queue mapping for namespaces
+
+### Goal
+Though the tag placement rule using the `namespace` tag is capable of placing an application in a queue this might not be enough in all setups.
+In some cases, multi tenancy for example, namespaces need to be grouped together.
+Administrators could annotate namespaces which allows dynamic placement of applications based on multiple annotations if placement rules were setup.
+YuniKorn cannot and does not just add all annotations from a namespace to an application.
+
+To help support this grouping case a parent queue can be tagged on a namespace.
+
+### Configuration
+The configuration for this functionality consists of two pieces:
+1. the mapping rule
+1. the namespace annotation
+
+First we set the following configuration to YuniKorn's configmap:
+
+```yaml
+partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ - name: tag
+ value: namespace.parentqueue
+ queues:
+ - name: root
+ queues:
+ - name: production
+ - name: development
+```
+
+The configuration used for the namespace to queue mapping is the same as [above](#Namespace-to-queue-mapping).
+As an extension to the placement rule a `parent` rule is added to support the grouping.
+The parent rule is used to generate the parent, or the queue above, in the hierarchy.
+The rule uses the tag `namespace.parentqueue` from the application to generate the parent queue name.
+The `namespace.parentqueue` tag is automatically added by the Kubernetes shim but does require a namespace annotation (see below).
+
+In the example rule configuration given the `create` flag is not set on the parent rule.
+This means that the parent queue must exist in the configuration otherwise the application submit will fail.
+For the example configuration this means supported values for the parent are thus limited to `production` and `development`.
+
+Quotas cannot be set on the parent queue using any of these mappings.
+The quota linked to the namespace is set on the namespace queue not the parent as per the namespace mapping provided earlier.
+
+Parent queue quotas must always be set directly in the configuration.
+This requires the `create` flag to be set to `false` on the parent rule.
+
+### Namespace parent queue
+Contrary to the namespace name itself, and inline with the quota settings, the namespaces need to be annotated to use the parent queue mapping.
+Namespace names must be unique in Kubernetes which is not affected by this annotation.
+The same annotation value may be used for multiple namespaces:
+```yaml
+yunikorn.apache.org/parentqueue: root.production
+```
+
+The example annotation above will map the parent queue to the existing `root.production` queue.
+Note that the rule will fully qualify the name if needed, you can thus omit the `root.` part in the annotation.
+If the annotation starts with `root.` the system assumes it is a fully qualified queue name.
+
+To complete the picture here is an image that shows the mapping from Kubernetes namespaces to queues in YuniKorn.
+It uses the annotations on the namespaces in Kubernetes as described, and the example configuration for the mapping rules.
+The `finance` and `sales` namespaces become queues grouped under the parent queue `production`.
+The namespaces `dev` and `test` are placed under the `development` parent queue.
+
+
+
+### Run a workload
+Applications, and the pods that are part of the application, can be submitted without specific labels or changes.
+YuniKorn will add the tags, the placement rules will do the rest.
+The configured placement rule will create the queues, if required, and add the application to the queue.
+
+Since the namespace `finance` is annotated with the example value, and the rules are in place.
+Applications in the `finance` namespace will run in the `root.production.finance` queue that is created dynamically.
diff --git a/versioned_docs/version-0.12.0/user_guide/sorting_policies.md b/versioned_docs/version-0.12.0/user_guide/sorting_policies.md
new file mode 100644
index 0000000..c5f4bfc
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/sorting_policies.md
@@ -0,0 +1,185 @@
+---
+id: sorting_policies
+title: Sorting Policies
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+The scheduler uses policies allow changing the scheduling behaviour without code changes.
+Policies can be set for:
+* [Applications](#application-sorting)
+* [Nodes](#node-sorting)
+* [Requests](#request-sorting)
+
+## Application sorting
+The application sorting policy is set for each queue via the config.
+A sorting policy setting is only effective on a `leaf` queue.
+Each `leaf` queue can use a different policy.
+
+A sorting policy only specifies the order in which the applications are sorted within a queue.
+That order is crucial in specifying which application is considered first when assigning resources.
+Sorting policies do _not_ affect the number of applications that are scheduled or active in the queue at the same time.
+All applications that have pending resource requests can and will be scheduled in a queue unless specifically filtered out.
+Even when applications are sorted using a first in first out policy multiple applications will run in a queue in parallel.
+
+A `parent` queue will always use the fair policy to sort the child queues.
+
+The following configuration entry sets the application sorting policy to `fifo` for the queue `root.sandbox`:
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: sandbox
+ properties:
+ application.sort.policy: fifo
+```
+
+The only applications that are considered during scheduling must have outstanding requests.
+A filter is applied _while_ sorting the applications to remove all that do not have outstanding requests.
+
+### FifoSortPolicy
+Short description: first in first out, based on application create time
+Config value: fifo (default)
+Behaviour:
+Before sorting the applications are filtered and must have pending resource requests.
+
+After filtering the applications left are sorted based on the application create time stamp only, no other filtering is applied.
+Since applications can only be added while the system is locked there can never be two applications with the exact same time stamp.
+
+The result is that the oldest application that requests resources gets resources.
+Younger applications will be given resources when all the current requests of older applications have been fulfilled.
+
+### FairSortPolicy
+Short description: fair based on usage
+Config value: fair
+Behaviour:
+Before sorting the applications are filtered and must have pending resource requests.
+
+After filtering the applications left are sorted based on the application usage.
+The usage of the application is defined as all confirmed and unconfirmed allocations for the applications.
+All resources defined on the application will be taken into account when calculating the usage.
+
+The result is that the resources available are spread equally over all applications that request resources.
+
+### StateAwarePolicy
+Short description: limit of one (1) application in Starting or Accepted state
+Config value: stateaware
+Behaviour:
+This sorting policy requires an understanding of the application states.
+Applications states are described in the [application states](design/scheduler_object_states.md#application-state) documentation.
+
+Before sorting applications the following filters are applied to all applications in the queue:
+The first filter is based on the application state.
+The following applications pass through the filter and generate the first intermediate list:
+* all applications in the state _running_
+* _one_ (1) application in the _starting_ state
+* if there are _no_ applications in the _starting_ state _one_ (1) application in the _accepted_ state is added
+
+The second filter takes the result of the first filter as an input.
+The preliminary list is filtered again: all applications _without_ a pending request are removed.
+
+After filtering based on status and pending requests the applications that remain are sorted.
+The final list is thus filtered twice with the remaining applications sorted on create time.
+
+To recap the _staring_ and _accepted_ state interactions:
+The application in the _accepted_ state is only added if there is no application in the _starting_ state.
+The application in the _starting_ state does not have to have pending requests.
+Any application in the _starting_ state will prevent _accepted_ applications from being added to the filtered list.
+
+For further details see the [Example run](design/state_aware_scheduling.md#example-run) in the design document.
+
+The result is that already running applications that request resources will get resources first.
+A drip feed of one new applications is added to the list of running applications to be allocated after all running applications.
+
+## Node sorting
+The node sorting policy is set for a partition via the config.
+Each partition can use a different policy.
+
+The following configuration entry sets the node sorting policy to `fair` for the partition `default`:
+```yaml
+partitions:
+ - name: default
+ nodesortpolicy:
+ type: fair
+```
+
+### FairnessPolicy
+Short description: available resource, descending order
+Config value: fair (default)
+Behaviour:
+Sort the list of nodes by the amount of available resources so that the node with the _highest_ amount of available resource is the first in the list.
+All resources defined on a node will be taken into account when calculating the usage.
+Resources of the same type are compared for the nodes.
+
+This results in a node with the lowest utilisation to be considered first for assigning new allocation.
+Resulting in a spread of allocations over all available nodes.
+Leading to an overall lower utilisation of the individual available nodes, unless the whole environment is highly utilised.
+Keeping the load on all nodes at a similar level does help
+In an environment that auto scales by adding new nodes this could trigger unexpected auto scale requests.
+
+### BinPackingPolicy
+Short description: available resource, ascending order
+Config value: binpacking
+Behaviour:
+Sort the list of nodes by the amount of available resources so that the node with the _lowest_ amount of available resource is the first in the list.
+All resources defined on a node will be taken into account when calculating the usage.
+Resources of the same type are compared for the nodes.
+
+This results in a node with the highest utilisation to be considered first for assigning new allocation.
+Resulting in a high(er) utilisation of a small(er) number of nodes, better suited for cloud deployments.
+
+## Resource weighting
+Node sorting policies may use the utilization of a node to determine ordering. Because nodes can have several unique
+resource types, a node's utilization is determined by a weighted average of its individual resource types. Resource
+weighting can be customized by using the `resourceweights` section of `nodesortpolicy`. If `resourceweights` is not
+present or empty, the default configuration sets the weight of both `vcore` and `memory` equally to `1.0`. All other
+resource types are ignored. Only resource types explicitly mentioned will have a weight.
+
+YuniKorn tracks CPU resources internally as the `vcore` resource type. This maps to the Kubernetes resource type `cpu`.
+All other resource types have consistent naming between YuniKorn and Kubernetes.
+
+For example, in the default configuration, if a node has `90%` of its CPU and `50%` of its memory allocated, the node
+will be considered to be `70%` utilized.
+
+The following configuration entry sets the weight of `vcore` to `4.0` and `memory` to `1.0` for the partition `default`.
+This will weight CPU usage four times higher than memory usage:
+```yaml
+partitions:
+ - name: default
+ nodesortpolicy:
+ type: fair
+ resourceweights:
+ vcore: 4.0
+ memory: 1.0
+```
+
+With this configuration, In this example, if a node has `90%` of its CPU and `50%` of its memory allocated, the node
+will be considered to be `82%` utilized.
+
+Note that weights are relative to each other, so specifying weights of `{ 4.0, 1.0 }` is equivalent to
+`{ 1.0, 0.25 }`. Negative weights are not allowed.
+
+## Request sorting
+There is currently one policy for sorting requests within an application.
+This policy is not configurable.
+Sorting requests is only possible based on the priority of the request.
+If there are multiple requests within an application that have the same priority the order of the requests is undetermined.
+This means that the order of requests with the same priority can, and most likely will, change between runs.
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/user_guide/trouble_shooting.md b/versioned_docs/version-0.12.0/user_guide/trouble_shooting.md
new file mode 100644
index 0000000..549d5e0
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/trouble_shooting.md
@@ -0,0 +1,192 @@
+---
+id: trouble_shooting
+title: Trouble Shooting
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## Scheduler logs
+
+### Retrieve scheduler logs
+
+Currently, the scheduler writes its logs to stdout/stderr, docker container handles the redirection of these logs to a
+local location on the underneath node, you can read more document [here](https://docs.docker.com/config/containers/logging/configure/).
+These logs can be retrieved by [kubectl logs](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#logs). Such as:
+
+```shell script
+// get the scheduler pod
+kubectl get pod -l component=yunikorn-scheduler -n yunikorn
+NAME READY STATUS RESTARTS AGE
+yunikorn-scheduler-766d7d6cdd-44b82 2/2 Running 0 33h
+
+// retrieve logs
+kubectl logs yunikorn-scheduler-766d7d6cdd-44b82 yunikorn-scheduler-k8s -n yunikorn
+```
+
+In most cases, this command cannot get all logs because the scheduler is rolling logs very fast. To retrieve more logs in
+the past, you will need to setup the [cluster level logging](https://kubernetes.io/docs/concepts/cluster-administration/logging/#cluster-level-logging-architectures).
+The recommended setup is to leverage [fluentd](https://www.fluentd.org/) to collect and persistent logs on an external storage, e.g s3.
+
+### Set Logging Level
+
+:::note
+Changing the logging level requires a restart of the scheduler pod.
+:::
+
+Stop the scheduler:
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=0
+```
+edit the deployment config in vim:
+
+```shell script
+kubectl edit deployment yunikorn-scheduler -n yunikorn
+```
+
+add `LOG_LEVEL` to the `env` field of the container template. For example setting `LOG_LEVEL` to `0` sets the logging
+level to `INFO`.
+
+```yaml
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ ...
+spec:
+ template:
+ ...
+ spec:
+ containers:
+ - env:
+ - name: LOG_LEVEL
+ value: '0'
+```
+
+Start the scheduler:
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=1
+```
+
+Available logging levels:
+
+| Value | Logging Level |
+|:-----: |:-------------: |
+| -1 | DEBUG |
+| 0 | INFO |
+| 1 | WARN |
+| 2 | ERROR |
+| 3 | DPanic |
+| 4 | Panic |
+| 5 | Fatal |
+
+## Pods are stuck at Pending state
+
+If some pods are stuck at Pending state, that means the scheduler could not find a node to allocate the pod. There are
+several possibilities to cause this:
+
+### 1. Non of the nodes satisfy pod placement requirement
+
+A pod can be configured with some placement constraints, such as [node-selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector),
+[affinity/anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity),
+do not have certain toleration for node [taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/), etc.
+To debug such issues, you can describe the pod by:
+
+```shell script
+kubectl describe pod <pod-name> -n <namespace>
+```
+
+the pod events will contain the predicate failures and that explains why nodes are not qualified for allocation.
+
+### 2. The queue is running out of capacity
+
+If the queue is running out of capacity, pods will be pending for available queue resources. To check if a queue is still
+having enough capacity for the pending pods, there are several approaches:
+
+1) check the queue usage from yunikorn UI
+
+If you do not know how to access the UI, you can refer the document [here](../get_started/get_started.md#access-the-web-ui). Go
+to the `Queues` page, navigate to the queue where this job is submitted to. You will be able to see the available capacity
+left for the queue.
+
+2) check the pod events
+
+Run the `kubectl describe pod` to get the pod events. If you see some event like:
+`Application <appID> does not fit into <queuePath> queue`. That means the pod could not get allocated because the queue
+is running out of capacity.
+
+The pod will be allocated if some other pods in this queue is completed or removed. If the pod remains pending even
+the queue has capacity, that may because it is waiting for the cluster to scale up.
+
+## Restart the scheduler
+
+YuniKorn can recover its state upon a restart. YuniKorn scheduler pod is deployed as a deployment, restart the scheduler
+can be done by scale down and up the replica:
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=0
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=1
+```
+
+## Gang Scheduling
+
+### 1. No placeholders created, app's pods are pending
+
+*Reason*: This is usually because the app is rejected by the scheduler, therefore non of the pods are scheduled.
+The common reasons caused the rejection are: 1) The taskGroups definition is invalid. The scheduler does the
+sanity check upon app submission, to ensure all the taskGroups are defined correctly, if these info are malformed,
+the scheduler rejects the app; 2) The total min resources defined in the taskGroups is bigger than the queues' max
+capacity, scheduler rejects the app because it won't fit into the queue's capacity. Check the pod event for relevant messages,
+and you will also be able to find more detail error messages from the schedulers' log.
+
+*Solution*: Correct the taskGroups definition and retry submitting the app.
+
+### 2. Not all placeholders can be allocated
+
+*Reason*: The placeholders also consume resources, if not all of them can be allocated, that usually means either the queue
+or the cluster has no sufficient resources for them. In this case, the placeholders will be cleaned up after a certain
+amount of time, defined by the `placeholderTimeoutInSeconds` scheduling policy parameter.
+
+*Solution*: Note, if the placeholder timeout reaches, currently the app will transit to failed state and can not be scheduled
+anymore. You can increase the placeholder timeout value if you are willing to wait for a longer time. In the future, a fallback policy
+might be added to provide some retry other than failing the app.
+
+### 3. Not all placeholders are swapped
+
+*Reason*: This usually means the actual app's pods are less than the minMembers defined in the taskGroups.
+
+*Solution*: Check the `minMember` in the taskGroup field and ensure it is correctly set. The `minMember` can be less than
+the actual pods, setting it to bigger than the actual number of pods is invalid.
+
+### 4.Placeholders are not cleaned up when the app terminated
+
+*Reason*: All the placeholders are set an [ownerReference](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#owners-and-dependents)
+to the first real pod of the app, or the controller reference. If the placeholder could not be cleaned up, that means
+the garbage collection is not working properly.
+
+*Solution*: check the placeholder `ownerReference` and the garbage collector in Kubernetes.
+
+
+## Still got questions?
+
+No problem! The Apache YuniKorn community will be happy to help. You can reach out to the community with the following options:
+
+1. Post your questions to dev@yunikorn.apache.org
+2. Join the [YuniKorn slack channel](https://join.slack.com/t/yunikornworkspace/shared_invite/enQtNzAzMjY0OTI4MjYzLTBmMDdkYTAwNDMwNTE3NWVjZWE1OTczMWE4NDI2Yzg3MmEyZjUyYTZlMDE5M2U4ZjZhNmYyNGFmYjY4ZGYyMGE) and post your questions to the `#yunikorn-user` channel.
+3. Join the [community sync up meetings](http://yunikorn.apache.org/community/getInvolved#community-meetings) and directly talk to the community members.
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/user_guide/usergroup_resolution.md b/versioned_docs/version-0.12.0/user_guide/usergroup_resolution.md
new file mode 100644
index 0000000..0816a20
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/usergroup_resolution.md
@@ -0,0 +1,68 @@
+---
+id: usergroup_resolution
+title: User & Group Resolution
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## User resolution
+
+User information is an important aspect of the scheduling cycle. It is one of the key identifier that can be used to determine the queue to which a job should be submitted. The Yunikorn Scheduler relies on the K8s Shim to provide user information. In the world of Kubernetes, there is no object defined that identfies the actual user. This is by design and more information can be found [here](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#users-in-kubernetes)
+
+Since, Kubernetes has no pre-defined field or resource for user information and individual cluster deployments with unique user identification tools can vary, we have defined a standard way of identifying the user. Yunikorn requires a Kubernetes [Label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) added. Using the [recommendation](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/) provided here, the default label is defined as below:
+
+| Label | Value |
+|----------------------------------------------- |--------------------- |
+| yunikorn.apache.org/username | User name. It can have duplicate entries but only the first value will be used. The default user is `nobody` |
+
+Example:
+```yaml
+metadata:
+ labels:
+ yunikorn.apache.org/username: "john"
+```
+:::tip
+In order to make this field uniquiely identifiable to the authorized user, the suggestion is to add this label as an immutable field by the user identification tool used by the cluster administrators. The cluster administrators or users are free to use any method or tool to add this field and value. This includes adding it manually at the time of submission.
+:::
+
+:::note Assumption
+Assumption:
+ Yunikorn assumes that all pods belonging to an application are owned by the same user. We recommend that the user label is added to every pod of an app. This is to ensure that there is no discrepency.
+:::
+
+The `yunikorn.apache.org/username` key can be customized by overriding the default value using the `USER_LABEL_KEY`env variable to the [K8s](https://github.com/apache/incubator-yunikorn-release/blob/master/helm-charts/yunikorn/templates/deployment.yaml). This is particularly useful in scenarios where the user label is already being added or if the label has to be modified for some secuirty reasons.
+
+```yaml
+ env:
+ - name: USER_LABEL_KEY
+ value: "custom_user_label"
+```
+
+## Group resolution
+
+Group membership resolution is pluggables and is defined here. Groups do not have to be part of provided user and group object. When the object is added to the cache the groups are automatically resolved based on the resolution that is configured.
+The resolver which is linked to the cache can be set per partition.
+
+The default group resolver is "no resolver".
+This resolver just echos the user name and a primary group with the same name as the user.
+
+Other resolvers are:
+* OS resolver
+* test resolver
diff --git a/versioned_docs/version-0.12.0/user_guide/workloads/run_flink.md b/versioned_docs/version-0.12.0/user_guide/workloads/run_flink.md
new file mode 100644
index 0000000..d20e815
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/workloads/run_flink.md
@@ -0,0 +1,66 @@
+---
+id: run_flink
+title: Run Flink Jobs
+description: How to run Flink jobs with YuniKorn
+image: https://svn.apache.org/repos/asf/flink/site/img/logo/png/100/flink_squirrel_100_color.png
+keywords:
+ - spark
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+It's very easy to run [Apache Flink](https://flink.apache.org/) on Kubernetes with YuniKorn. Depending on which mode is
+used to run Flink on Kubernetes, the configuration is slight different.
+
+## Standalone mode
+
+Please follow [Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html) to get details and examples of standalone deploy mode.
+In this mode, we can directly add required labels (applicationId and queue) in Deployment/Job spec to run flink application with YuniKorn scheduler, as well as [Run workloads with YuniKorn Scheduler](#run-workloads-with-yunikorn-scheduler).
+
+## Native mode
+
+Please follow [Native Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/native_kubernetes.html) to get details and examples of native deploy mode.
+Running flink application with YuniKorn scheduler in native mode is only supported for flink 1.11 or above, we can leverage two flink configurations `kubernetes.jobmanager.labels` and `kubernetes.taskmanager.labels` to set the required labels.
+Examples:
+
+* Start a flink session
+```
+./bin/kubernetes-session.sh \
+ -Dkubernetes.cluster-id=<ClusterId> \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dresourcemanager.taskmanager-timeout=3600000 \
+ -Dkubernetes.jobmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ -Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox
+```
+
+* Start a flink application
+```
+./bin/flink run-application -p 8 -t kubernetes-application \
+ -Dkubernetes.cluster-id=<ClusterId> \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dkubernetes.container.image=<CustomImageName> \
+ -Dkubernetes.jobmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ -Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ local:///opt/flink/usrlib/my-flink-job.jar
+```
\ No newline at end of file
diff --git a/versioned_docs/version-0.12.0/user_guide/workloads/run_spark.md b/versioned_docs/version-0.12.0/user_guide/workloads/run_spark.md
new file mode 100644
index 0000000..ca75526
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/workloads/run_spark.md
@@ -0,0 +1,149 @@
+---
+id: run_spark
+title: Run Spark Jobs
+description: How to run Spark jobs with YuniKorn
+keywords:
+ - spark
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+:::note
+This document assumes you have YuniKorn and its admission-controller both installed. Please refer to
+[get started](../../get_started/get_started.md) to see how that is done.
+:::
+
+## Prepare the docker image for Spark
+
+To run Spark on Kubernetes, you'll need the Spark docker images. You can 1) use the docker images provided by the YuniKorn
+team, or 2) build one from scratch. If you want to build your own Spark docker image, you can
+* Download a Spark version that has Kubernetes support, URL: https://github.com/apache/spark
+* Build spark with Kubernetes support:
+```shell script
+mvn -Pyarn -Phadoop-2.7 -Dhadoop.version=2.7.4 -Phive -Pkubernetes -Phive-thriftserver -DskipTests package
+```
+
+## Create a namespace for Spark jobs
+
+Create a namespace:
+
+```shell script
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: spark-test
+EOF
+```
+
+Create service account and cluster role bindings under `spark-test` namespace:
+
+```shell script
+cat <<EOF | kubectl apply -n spark-test -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: spark
+ namespace: spark-test
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: spark-cluster-role
+ namespace: spark-test
+rules:
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get", "watch", "list", "create", "delete"]
+- apiGroups: [""]
+ resources: ["configmaps"]
+ verbs: ["get", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: spark-cluster-role-binding
+ namespace: spark-test
+subjects:
+- kind: ServiceAccount
+ name: spark
+ namespace: spark-test
+roleRef:
+ kind: ClusterRole
+ name: spark-cluster-role
+ apiGroup: rbac.authorization.k8s.io
+EOF
+```
+
+:::note
+Do NOT use `ClusterRole` and `ClusterRoleBinding` to run Spark jobs in production, please configure a more fine-grained
+security context for running Spark jobs. See more about how to configure proper RBAC rules [here](https://kubernetes.io/docs/reference/access-authn-authz/rbac/).
+:::
+
+## Submit a Spark job
+
+If this is running from local machine, you will need to start the proxy in order to talk to the api-server.
+```shell script
+kubectl proxy
+```
+
+Run a simple SparkPi job (this assumes that the Spark binaries are installed to `/usr/local` directory).
+```shell script
+export SPARK_HOME=/usr/local/spark-2.4.4-bin-hadoop2.7/
+${SPARK_HOME}/bin/spark-submit --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --class org.apache.spark.examples.SparkPi \
+ --conf spark.executor.instances=1 \
+ --conf spark.kubernetes.namespace=spark-test \
+ --conf spark.kubernetes.executor.request.cores=1 \
+ --conf spark.kubernetes.container.image=apache/yunikorn:spark-2.4.4 \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-test:spark \
+ local:///opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar
+```
+
+You'll see Spark driver and executors been created on Kubernetes:
+
+
+
+You can also view the job info from YuniKorn UI. If you do not know how to access the YuniKorn UI, please read the document
+[here](../../get_started/get_started.md#access-the-web-ui).
+
+
+
+## What happens behind the scenes?
+
+When the Spark job is submitted to the cluster, the job is submitted to `spark-test` namespace. The Spark driver pod will
+be firstly created under this namespace. Since this cluster has YuniKorn admission-controller enabled, when the driver pod
+get created, the admission-controller mutates the pod's spec and injects `schedulerName=yunikorn`, by doing this, the
+default K8s scheduler will skip this pod and it will be scheduled by YuniKorn instead. See how this is done by [configuring
+another scheduler in Kubernetes](https://kubernetes.io/docs/tasks/extend-kubernetes/configure-multiple-schedulers/).
+
+The default configuration has placement rule enabled, which automatically maps the `spark-test` namespace to a YuniKorn
+queue `root.spark-test`. All Spark jobs submitted to this namespace will be automatically submitted to the queue first.
+To see more about how placement rule works, please see doc [placement-rules](user_guide/placement_rules.md). By far,
+the namespace defines the security context of the pods, and the queue determines how the job and pods will be scheduled
+with considering of job ordering, queue resource fairness, etc. Note, this is the simplest setup, which doesn't enforce
+the queue capacities. The queue is considered as having unlimited capacity.
+
+YuniKorn reuses the Spark application ID set in label `spark-app-selector`, and this job is submitted
+to YuniKorn and being considered as a job. The job is scheduled and running as there is sufficient resources in the cluster.
+YuniKorn allocates the driver pod to a node, binds the pod and starts all the containers. Once the driver pod gets started,
+it requests for a bunch of executor pods to run its tasks. Those pods will be created in the same namespace as well and
+scheduled by YuniKorn as well.
diff --git a/versioned_docs/version-0.12.0/user_guide/workloads/run_tensorflow.md b/versioned_docs/version-0.12.0/user_guide/workloads/run_tensorflow.md
new file mode 100644
index 0000000..5448990
--- /dev/null
+++ b/versioned_docs/version-0.12.0/user_guide/workloads/run_tensorflow.md
@@ -0,0 +1,93 @@
+---
+id: run_tf
+title: Run TensorFlow Jobs
+description: How to run TensorFlow jobs with YuniKorn
+keywords:
+ - tensorflow
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This guide gives an overview of how to set up [training-operator](https://github.com/kubeflow/training-operator)
+and how to run a Tensorflow job with YuniKorn scheduler. The training-operator is a unified training operator maintained by
+Kubeflow. It not only supports TensorFlow but also PyTorch, XGboots, etc.
+
+## Install training-operator
+You can use the following command to install training operator in kubeflow namespace by default. If you have problems with installation,
+please refer to [this doc](https://github.com/kubeflow/training-operator#installation) for details.
+```
+kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone?ref=v1.3.0"
+```
+
+## Prepare the docker image
+Before you start running a TensorFlow job on Kubernetes, you'll need to build the docker image.
+1. Download files from [deployment/examples/tfjob](https://github.com/apache/incubator-yunikorn-k8shim/tree/master/deployments/examples/tfjob)
+2. To build this docker image with the following command
+
+```
+docker build -f Dockerfile -t kubeflow/tf-dist-mnist-test:1.0 .
+```
+
+## Run a TensorFlow job
+Here is a TFJob yaml for MNIST [example](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/deployments/examples/tfjob/tf-job-mnist.yaml).
+
+```yaml
+apiVersion: kubeflow.org/v1
+kind: TFJob
+metadata:
+ name: dist-mnist-for-e2e-test
+ namespace: kubeflow
+spec:
+ tfReplicaSpecs:
+ PS:
+ replicas: 2
+ restartPolicy: Never
+ template:
+ metadata:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ queue: root.sandbox
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - name: tensorflow
+ image: kubeflow/tf-dist-mnist-test:1.0
+ Worker:
+ replicas: 4
+ restartPolicy: Never
+ template:
+ metadata:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ queue: root.sandbox
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - name: tensorflow
+ image: kubeflow/tf-dist-mnist-test:1.0
+```
+Create the TFJob
+```
+kubectl create -f deployments/examples/tfjob/tf-job-mnist.yaml
+```
+You can view the job info from YuniKorn UI. If you do not know how to access the YuniKorn UI,
+please read the document [here](../../get_started/get_started.md#access-the-web-ui).
+
+
diff --git a/versioned_sidebars/version-0.12.0-sidebars.json b/versioned_sidebars/version-0.12.0-sidebars.json
new file mode 100644
index 0000000..3898d5e
--- /dev/null
+++ b/versioned_sidebars/version-0.12.0-sidebars.json
@@ -0,0 +1,208 @@
+{
+ "version-0.12.0/docs": [
+ {
+ "type": "category",
+ "collapsed": true,
+ "collapsible": true,
+ "label": "Get Started",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.12.0/get_started/user_guide"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/get_started/core_features"
+ }
+ ]
+ },
+ {
+ "type": "category",
+ "collapsed": true,
+ "collapsible": true,
+ "label": "User Guide",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/queue_config"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/placement_rules"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/usergroup_resolution"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/sorting_policies"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/acls"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/resource_quota_management"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/gang_scheduling"
+ },
+ {
+ "type": "category",
+ "label": "Workloads",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/workloads/run_spark"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/workloads/run_flink"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/workloads/run_tf"
+ }
+ ],
+ "collapsible": true,
+ "collapsed": true
+ },
+ {
+ "type": "category",
+ "label": "REST APIs",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.12.0/api/cluster"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/api/scheduler"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/api/system"
+ }
+ ],
+ "collapsible": true,
+ "collapsed": true
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/user_guide/trouble_shooting"
+ }
+ ]
+ },
+ {
+ "type": "category",
+ "collapsed": true,
+ "collapsible": true,
+ "label": "Developer Guide",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.12.0/developer_guide/env_setup"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/developer_guide/build"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/developer_guide/deployment"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/developer_guide/openshift_development"
+ },
+ {
+ "type": "category",
+ "label": "Designs",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/architecture"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/scheduler_core_design"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/cache_removal"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/k8shim"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/cross_queue_preemption"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/namespace_resource_quota"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/pluggable_app_management"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/resilience"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/predicates"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/scheduler_configuration"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/state_aware_scheduling"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/scheduler_object_states"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/design/gang_scheduling"
+ }
+ ],
+ "collapsible": true,
+ "collapsed": true
+ }
+ ]
+ },
+ {
+ "type": "category",
+ "collapsed": true,
+ "collapsible": true,
+ "label": "Performance",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.12.0/performance/evaluate_perf_function_with_kubemark"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/performance/performance_tutorial"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/performance/metrics"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.12.0/performance/profiling"
+ }
+ ]
+ }
+ ]
+}
diff --git a/versions.json b/versions.json
index 90e93ec..5de7aaf 100644
--- a/versions.json
+++ b/versions.json
@@ -1,4 +1,5 @@
[
+ "0.12.0",
"0.11.0",
"0.10.0",
"0.9.0",