You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@yunikorn.apache.org by wi...@apache.org on 2021/04/13 05:29:05 UTC
[incubator-yunikorn-site] branch master updated: [YUNIKORN-613]
release v0.10.0 (#45)n
This is an automated email from the ASF dual-hosted git repository.
wilfreds pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-yunikorn-site.git
The following commit(s) were added to refs/heads/master by this push:
new 1823932 [YUNIKORN-613] release v0.10.0 (#45)n
1823932 is described below
commit 182393282d2ce9b47846e7ca888f2ac825f22c44
Author: Tao Yang <ta...@apache.org>
AuthorDate: Tue Apr 13 15:27:08 2021 +1000
[YUNIKORN-613] release v0.10.0 (#45)n
Update the download page to announce the new release.
Add documentation for the new release.
Fixes: #45
---
docusaurus.config.js | 2 +-
src/pages/community/download.md | 3 +-
src/pages/release-announce/0.10.0.md | 52 +++
versioned_docs/version-0.10.0/api/cluster.md | 62 +++
versioned_docs/version-0.10.0/api/scheduler.md | 517 +++++++++++++++++++++
versioned_docs/version-0.10.0/api/system.md | 225 +++++++++
.../version-0.10.0/assets/application-state.png | Bin 0 -> 89514 bytes
.../version-0.10.0/assets/architecture.png | Bin 0 -> 188534 bytes
.../version-0.10.0/assets/cpu_profile.jpg | Bin 0 -> 259919 bytes
.../version-0.10.0/assets/dashboard_secret.png | Bin 0 -> 31815 bytes
.../assets/dashboard_token_select.png | Bin 0 -> 47533 bytes
.../assets/docker-dektop-minikube.png | Bin 0 -> 249095 bytes
.../version-0.10.0/assets/docker-desktop.png | Bin 0 -> 247891 bytes
.../version-0.10.0/assets/fifo-state-example.png | Bin 0 -> 239302 bytes
.../assets/gang_scheduling_iintro.png | Bin 0 -> 43907 bytes
.../version-0.10.0/assets/goland_debug.jpg | Bin 0 -> 198845 bytes
.../assets/goland_ide_pref_imports.png | Bin 0 -> 71562 bytes
.../assets/goland_ide_pref_inspections.png | Bin 0 -> 115226 bytes
.../assets/goland_ide_pref_other.png | Bin 0 -> 51200 bytes
.../version-0.10.0/assets/goland_ide_pref_tabs.png | Bin 0 -> 52260 bytes
.../version-0.10.0/assets/namespace-mapping.png | Bin 0 -> 327547 bytes
.../version-0.10.0/assets/node-bin-packing.png | Bin 0 -> 232909 bytes
versioned_docs/version-0.10.0/assets/node-fair.png | Bin 0 -> 462310 bytes
.../version-0.10.0/assets/object-state.png | Bin 0 -> 39732 bytes
.../version-0.10.0/assets/pluggable-app-mgmt.jpg | Bin 0 -> 79170 bytes
.../version-0.10.0/assets/queue-fairness.png | Bin 0 -> 173299 bytes
.../assets/queue-resource-quotas.png | Bin 0 -> 283689 bytes
.../assets/resilience-node-recovery.jpg | Bin 0 -> 319477 bytes
.../version-0.10.0/assets/resilience-workflow.jpg | Bin 0 -> 441551 bytes
.../version-0.10.0/assets/spark-jobs-on-ui.png | Bin 0 -> 528736 bytes
.../version-0.10.0/assets/spark-pods.png | Bin 0 -> 303407 bytes
.../version-0.10.0/assets/throughput.png | Bin 0 -> 252615 bytes
.../version-0.10.0/assets/yk-ui-screenshots.gif | Bin 0 -> 813848 bytes
.../version-0.10.0/design/architecture.md | 62 +++
.../version-0.10.0/design/cache_removal.md | 446 ++++++++++++++++++
.../design/cross_queue_preemption.md | 126 +++++
versioned_docs/version-0.10.0/design/k8shim.md | 83 ++++
.../design/namespace_resource_quota.md | 183 ++++++++
.../design/pluggable_app_management.md | 75 +++
versioned_docs/version-0.10.0/design/predicates.md | 80 ++++
versioned_docs/version-0.10.0/design/resilience.md | 144 ++++++
.../design/scheduler_configuration.md | 246 ++++++++++
.../version-0.10.0/design/scheduler_core_design.md | 401 ++++++++++++++++
.../design/scheduler_object_states.md | 127 +++++
.../design/state_aware_scheduling.md | 112 +++++
.../version-0.10.0/developer_guide/build.md | 178 +++++++
.../version-0.10.0/developer_guide/deployment.md | 132 ++++++
.../version-0.10.0/developer_guide/env_setup.md | 156 +++++++
.../developer_guide/openshift_development.md | 182 ++++++++
.../version-0.10.0/get_started/core_features.md | 34 ++
.../version-0.10.0/get_started/get_started.md | 73 +++
.../evaluate_perf_function_with_kubemark.md | 103 ++++
.../version-0.10.0/performance/metrics.md | 72 +++
.../version-0.10.0/performance/profiling.md | 122 +++++
versioned_docs/version-0.10.0/user_guide/acls.md | 143 ++++++
.../version-0.10.0/user_guide/gang_scheduling.md | 221 +++++++++
.../version-0.10.0/user_guide/placement_rules.md | 354 ++++++++++++++
.../version-0.10.0/user_guide/queue_config.md | 307 ++++++++++++
.../user_guide/resource_quota_mgmt.md | 324 +++++++++++++
.../version-0.10.0/user_guide/sorting_policies.md | 154 ++++++
.../version-0.10.0/user_guide/trouble_shooting.md | 192 ++++++++
.../user_guide/workloads/run_flink.md | 66 +++
.../user_guide/workloads/run_spark.md | 145 ++++++
.../user_guide/workloads/run_tensorflow.md | 40 ++
versioned_sidebars/version-0.10.0-sidebars.json | 194 ++++++++
versions.json | 1 +
66 files changed, 6137 insertions(+), 2 deletions(-)
diff --git a/docusaurus.config.js b/docusaurus.config.js
index 7d402cc..ba3545a 100644
--- a/docusaurus.config.js
+++ b/docusaurus.config.js
@@ -46,7 +46,7 @@ module.exports = {
announcementBar: {
id: 'new_release',
content:
- '0.9.0 has been released, check the DOWNLOADS',
+ '0.10.0 has been released, check the DOWNLOADS',
backgroundColor: '#fafbfc',
textColor: '#091E42',
},
diff --git a/src/pages/community/download.md b/src/pages/community/download.md
index acc5df2..a6777a9 100644
--- a/src/pages/community/download.md
+++ b/src/pages/community/download.md
@@ -26,10 +26,11 @@ under the License.
Apache YuniKorn (Incubating) is released as source code tarballs. The downloads are distributed via mirror sites and should be checked for tampering using GPG or SHA-512.
-The latest release of Apache YuniKorn is v0.9.0.
+The latest release of Apache YuniKorn is v0.10.0.
| Version | Release date | Source download | Docker images | Release notes |
| ---- | ---- | ---- | ---- | ---- |
+| v0.10.0 | 2021-04-09 | [Download](https://www.apache.org/dyn/closer.cgi/incubator/yunikorn/0.10.0/apache-yunikorn-0.10.0-incubating-src.tar.gz)<br />[Checksum](https://downloads.apache.org/incubator/yunikorn/0.10.0/apache-yunikorn-0.10.0-incubating-src.tar.gz.sha512) & [Signature](https://downloads.apache.org/incubator/yunikorn/0.10.0/apache-yunikorn-0.10.0-incubating-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-0.10.0/images/sha256-e8f44044876 [...]
| v0.9.0 | 2020-08-28 | [Download](https://www.apache.org/dyn/closer.cgi/incubator/yunikorn/0.9.0/apache-yunikorn-0.9.0-incubating-src.tar.gz)<br />[Checksum](https://downloads.apache.org/incubator/yunikorn/0.9.0/apache-yunikorn-0.9.0-incubating-src.tar.gz.sha512) & [Signature](https://downloads.apache.org/incubator/yunikorn/0.9.0/apache-yunikorn-0.9.0-incubating-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-0.9.0/images/sha256-2835a6a0988c44e7802 [...]
| v0.8.0 | 2020-05-04 | [Download](https://www.apache.org/dyn/closer.cgi/incubator/yunikorn/0.8.0-incubating/apache-yunikorn-0.8.0-incubating-src.tar.gz)<br />[Checksum](https://downloads.apache.org/incubator/yunikorn/0.8.0-incubating/apache-yunikorn-0.8.0-incubating-src.tar.gz.sha512) & [Signature](https://downloads.apache.org/incubator/yunikorn/0.8.0-incubating/apache-yunikorn-0.8.0-incubating-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-0.8.0/ [...]
diff --git a/src/pages/release-announce/0.10.0.md b/src/pages/release-announce/0.10.0.md
new file mode 100644
index 0000000..330525c
--- /dev/null
+++ b/src/pages/release-announce/0.10.0.md
@@ -0,0 +1,52 @@
+---
+id: rn-0.10.0
+title: Release Announcement v0.10.0
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+# Release Announcement v0.10.0
+It gives us great pleasure to announce that the Apache YuniKorn (Incubating) community has voted to release 0.10.0. Apache YuniKorn (Incubating) is a standalone resource scheduler, designed for managing, and scheduling Big Data workloads on the container orchestration framework for on-prem and on-cloud use cases.
+
+The notable feature added in this release is the Gang Scheduling, YuniKorn now can provide gang scheduling capabilities while running applications on Kubernetes. Along with that, various bug fixes and improvements are included in this release.
+
+## Overview
+The Apache YuniKorn (Incubating) community has fixed over 187 JIRAs in this release: [YuniKorn jiras resolved in 0.10.0](https://issues.apache.org/jira/projects/YUNIKORN/versions/12348351)
+
+Release date: 2021-04-09
+
+## Highlights
+
+### Supported Kubernetes Versions
+From this release on, supported Kubernetes versions have been updated to 1.16.x, 1.17.x and 1.18.x. (Earlier versions support 1.13.x, 1.14.x and 1.15.x.) YuniKorn support matrix primarily supports 3 major Kubernetes versions.
+
+### Gang Scheduling
+In this release, YuniKorn starts to support the Gang Scheduling. Users can apply Gang Scheduling for the applications requiring gang scheduling semantics, such as Spark, Tensorflow, Pytorch, etc. YuniKorn proactively reserves resources for gang scheduling applications, which can work more efficiently with cluster-autoscaler. The initial support has been well tested with Spark, and it can be used with the native Spark on K8s or the Spark K8s operator. For more information how to enable an [...]
+
+### Simplify the logic in the Scheduler Core
+The community has done a major code refactoring for the scheduler core, tracked by (YUNKORN-317)[https://issues.apache.org/jira/browse/YUNIKORN-317]. This refactoring combines the cache and scheduler module into one that removes a lot of complexity in the code, improves the efficiency of the core scheduling logic. And as a result, it is much simpler to read and mitigates the maintenance effort.
+
+### Application Tracking API and CRD Phase One
+This release introduces an application tracking API and K8s custom resource definition (CRD) to further improve the user experience. The CRD will be used by the app operator/job server to interact with YuniKorn, to provide a better app lifecycle management. The first phase has defined the common protocol messages and CRD object formats.
+
+### Web UI Refurbishment
+The community has made some usability improvements for the web UI, including a redesigned web UI layout, tweaked look and feel to provide better user experience, bug fixes, etc.
+
+## Community
+The Apache YuniKorn community is pleased to welcome new committers Julia Kinga Marton and Tingyao Huang. We would love to see more committers joining the community and help to shape the project. In the past few months, we continue to have bi-weekly community meetings in 2 different time zones, ad-hoc meetings, offline channel discussions. The community will continue to be open.
diff --git a/versioned_docs/version-0.10.0/api/cluster.md b/versioned_docs/version-0.10.0/api/cluster.md
new file mode 100644
index 0000000..bae308c
--- /dev/null
+++ b/versioned_docs/version-0.10.0/api/cluster.md
@@ -0,0 +1,62 @@
+---
+id: cluster
+title: Cluster
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Clusters
+
+Returns general information about the clusters managed by the YuniKorn Scheduler. Information includes number of (total, failed, pending, running, completed) applications and containers.
+
+**URL** : `/ws/v1/clusters`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+As an example, here is a response from a 2-node cluster with 3 applications and 4 running containers.
+
+```json
+[
+ {
+ "clusterName": "kubernetes",
+ "totalApplications": "3",
+ "failedApplications": "1",
+ "pendingApplications": "",
+ "runningApplications": "3",
+ "completedApplications": "",
+ "totalContainers": "4",
+ "failedContainers": "",
+ "pendingContainers": "",
+ "runningContainers": "4",
+ "activeNodes": "2",
+ "totalNodes": "2",
+ "failedNodes": ""
+ }
+]
+```
+
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/api/scheduler.md b/versioned_docs/version-0.10.0/api/scheduler.md
new file mode 100644
index 0000000..d8e9452
--- /dev/null
+++ b/versioned_docs/version-0.10.0/api/scheduler.md
@@ -0,0 +1,517 @@
+---
+id: scheduler
+title: Scheduler
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Queues
+
+Displays general information about the queues like name, status, capacities and properties.
+The queues' hierarchy is kept in the response json.
+
+**URL** : `/ws/v1/queues`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+For the default queue hierarchy (only `root.default` leaf queue exists) a similar response to the following is sent back to the client:
+
+```json
+{
+ "partitionName": "[mycluster]default",
+ "capacity": {
+ "capacity": "map[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:80000 pods:110 vcore:60000]",
+ "usedcapacity": "0"
+ },
+ "nodes": null,
+ "queues": {
+ "queuename": "root",
+ "status": "Active",
+ "capacities": {
+ "capacity": "[]",
+ "maxcapacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:80000 pods:110 vcore:60000]",
+ "usedcapacity": "[memory:8000 vcore:8000]",
+ "absusedcapacity": "[memory:54 vcore:80]"
+ },
+ "queues": [
+ {
+ "queuename": "default",
+ "status": "Active",
+ "capacities": {
+ "capacity": "[]",
+ "maxcapacity": "[]",
+ "usedcapacity": "[memory:8000 vcore:8000]",
+ "absusedcapacity": "[]"
+ },
+ "queues": null,
+ "properties": {}
+ }
+ ],
+ "properties": {}
+ }
+}
+```
+
+## Applications
+
+Displays general information about the applications like used resources, queue name, submission time and allocations.
+
+**URL** : `/ws/v1/apps`
+
+**Method** : `GET`
+
+**Query Params** :
+
+1. queue=<fully qualified queue name\>
+
+The fully qualified queue name used to filter the applications that run within the given queue. For example, "/ws/v1/apps?queue=root.default" returns the applications running in "root.default" queue.
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+In the example below there are three allocations belonging to two applications.
+
+```json
+[
+ {
+ "applicationID": "application-0002",
+ "usedResource": "[memory:4000 vcore:4000]",
+ "partition": "[mycluster]default",
+ "queueName": "root.default",
+ "submissionTime": 1595939756253216000,
+ "allocations": [
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": null,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "resource": "[memory:4000 vcore:4000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default"
+ }
+ ],
+ "applicationState": "Running"
+ },
+ {
+ "applicationID": "application-0001",
+ "usedResource": "[memory:4000 vcore:4000]",
+ "partition": "[mycluster]default",
+ "queueName": "root.default",
+ "submissionTime": 1595939756253460000,
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": null,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default"
+ },
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": null,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0002",
+ "applicationId": "application-0001",
+ "partition": "default"
+ }
+ ],
+ "applicationState": "Running"
+ }
+]
+```
+
+## Nodes
+
+Displays general information about the nodes managed by YuniKorn.
+Node details include host and rack name, capacity, resources and allocations.
+
+**URL** : `/ws/v1/nodes`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+Here you can see an example response from a 2-node cluster having 3 allocations.
+
+```json
+[
+ {
+ "partitionName": "[mycluster]default",
+ "nodesInfo": [
+ {
+ "nodeID": "node-0001",
+ "hostName": "",
+ "rackName": "",
+ "capacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:14577 pods:110 vcore:10000]",
+ "allocated": "[memory:6000 vcore:6000]",
+ "occupied": "[memory:154 vcore:750]",
+ "available": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:6423 pods:110 vcore:1250]",
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": null,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default"
+ },
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": null,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "resource": "[memory:4000 vcore:4000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default"
+ }
+ ],
+ "schedulable": true
+ },
+ {
+ "nodeID": "node-0002",
+ "hostName": "",
+ "rackName": "",
+ "capacity": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:14577 pods:110 vcore:10000]",
+ "allocated": "[memory:2000 vcore:2000]",
+ "occupied": "[memory:154 vcore:750]",
+ "available": "[ephemeral-storage:75850798569 hugepages-1Gi:0 hugepages-2Mi:0 memory:6423 pods:110 vcore:1250]",
+ "allocations": [
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": null,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "resource": "[memory:2000 vcore:2000]",
+ "priority": "<nil>",
+ "queueName": "root.default",
+ "nodeId": "node-0002",
+ "applicationId": "application-0001",
+ "partition": "default"
+ }
+ ],
+ "schedulable": true
+ }
+ ]
+ }
+]
+```
+
+## Goroutines info
+
+Dumps the stack traces of the currently running goroutines.
+
+**URL** : `/ws/v1/stack`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+goroutine 356 [running
+]:
+github.com/apache/incubator-yunikorn-core/pkg/webservice.getStackInfo.func1(0x30a0060,
+0xc003e900e0,
+0x2)
+ /yunikorn/go/pkg/mod/github.com/apache/incubator-yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/handlers.go: 41 +0xab
+github.com/apache/incubator-yunikorn-core/pkg/webservice.getStackInfo(0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /yunikorn/go/pkg/mod/github.com/apache/incubator-yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/handlers.go: 48 +0x71
+net/http.HandlerFunc.ServeHTTP(0x2df0e10,
+0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /usr/local/go/src/net/http/server.go: 1995 +0x52
+github.com/apache/incubator-yunikorn-core/pkg/webservice.Logger.func1(0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /yunikorn/go/pkg/mod/github.com/apache/incubator-yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/webservice.go: 65 +0xd4
+net/http.HandlerFunc.ServeHTTP(0xc00003a570,
+0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /usr/local/go/src/net/http/server.go: 1995 +0x52
+github.com/gorilla/mux.(*Router).ServeHTTP(0xc00029cb40,
+0x30a0060,
+0xc003e900e0,
+0xc0063fee00)
+ /yunikorn/go/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go: 212 +0x140
+net/http.serverHandler.ServeHTTP(0xc0000df520,
+0x30a0060,
+0xc003e900e0,
+0xc0063fee00)
+ /usr/local/go/src/net/http/server.go: 2774 +0xcf
+net/http.(*conn).serve(0xc0000eab40,
+0x30a61a0,
+0xc003b74000)
+ /usr/local/go/src/net/http/server.go: 1878 +0x812
+created by net/http.(*Server).Serve
+ /usr/local/go/src/net/http/server.go: 2884 +0x4c5
+
+goroutine 1 [chan receive,
+ 26 minutes
+]:
+main.main()
+ /yunikorn/pkg/shim/main.go: 52 +0x67a
+
+goroutine 19 [syscall,
+ 26 minutes
+]:
+os/signal.signal_recv(0x1096f91)
+ /usr/local/go/src/runtime/sigqueue.go: 139 +0x9f
+os/signal.loop()
+ /usr/local/go/src/os/signal/signal_unix.go: 23 +0x30
+created by os/signal.init.0
+ /usr/local/go/src/os/signal/signal_unix.go: 29 +0x4f
+
+...
+```
+
+## Metrics
+
+Endpoint to retrieve metrics from the Prometheus server.
+The metrics are dumped with help messages and type information.
+
+**URL** : `/ws/v1/metrics`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"} 2.567e-05
+go_gc_duration_seconds{quantile="0.25"} 3.5727e-05
+go_gc_duration_seconds{quantile="0.5"} 4.5144e-05
+go_gc_duration_seconds{quantile="0.75"} 6.0024e-05
+go_gc_duration_seconds{quantile="1"} 0.00022528
+go_gc_duration_seconds_sum 0.021561648
+go_gc_duration_seconds_count 436
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines 82
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+go_info{version="go1.12.17"} 1
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes 9.6866248e+07
+
+...
+
+# HELP yunikorn_scheduler_vcore_nodes_usage Nodes resource usage, by resource name.
+# TYPE yunikorn_scheduler_vcore_nodes_usage gauge
+yunikorn_scheduler_vcore_nodes_usage{range="(10%, 20%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(20%,30%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(30%,40%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(40%,50%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(50%,60%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(60%,70%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(70%,80%]"} 1
+yunikorn_scheduler_vcore_nodes_usage{range="(80%,90%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(90%,100%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="[0,10%]"} 0
+```
+
+## Configuration validation
+
+**URL** : `/ws/v1/validate-conf`
+
+**Method** : `POST`
+
+**Auth required** : NO
+
+### Success response
+
+Regardless whether the configuration is allowed or not if the server was able to process the request, it will yield a 200 HTTP status code.
+
+**Code** : `200 OK`
+
+#### Allowed configuration
+
+Sending the following simple configuration yields an accept
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+```
+
+Reponse
+
+```json
+{
+ "allowed": true,
+ "reason": ""
+}
+```
+
+#### Disallowed configuration
+
+The following configuration is not allowed due to the "wrong_text" field put into the yaml file.
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+ - wrong_text
+```
+
+Reponse
+
+```json
+{
+ "allowed": false,
+ "reason": "yaml: unmarshal errors:\n line 7: cannot unmarshal !!str `wrong_text` into configs.PartitionConfig"
+}
+```
+
+## Application history
+
+Endpoint to retrieve historical data about the number of total applications by timestamp.
+
+**URL** : `/ws/v1/history/apps`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "timestamp": 1595939966153460000,
+ "totalApplications": "1"
+ },
+ {
+ "timestamp": 1595940026152892000,
+ "totalApplications": "1"
+ },
+ {
+ "timestamp": 1595940086153799000,
+ "totalApplications": "2"
+ },
+ {
+ "timestamp": 1595940146154497000,
+ "totalApplications": "2"
+ },
+ {
+ "timestamp": 1595940206155187000,
+ "totalApplications": "2"
+ }
+]
+```
+
+## Container history
+
+Endpoint to retrieve historical data about the number of total containers by timestamp.
+
+**URL** : `/ws/v1/history/containers`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "timestamp": 1595939966153460000,
+ "totalContainers": "1"
+ },
+ {
+ "timestamp": 1595940026152892000,
+ "totalContainers": "1"
+ },
+ {
+ "timestamp": 1595940086153799000,
+ "totalContainers": "3"
+ },
+ {
+ "timestamp": 1595940146154497000,
+ "totalContainers": "3"
+ },
+ {
+ "timestamp": 1595940206155187000,
+ "totalContainers": "3"
+ }
+]
+```
diff --git a/versioned_docs/version-0.10.0/api/system.md b/versioned_docs/version-0.10.0/api/system.md
new file mode 100644
index 0000000..1d685ff
--- /dev/null
+++ b/versioned_docs/version-0.10.0/api/system.md
@@ -0,0 +1,225 @@
+---
+id: system
+title: System
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+These endpoints are for the [pprof](https://github.com/google/pprof) profiling tool.
+
+## pprof
+
+**URL** : `/debug/pprof/`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+/debug/pprof/
+
+Types of profiles available:
+Count Profile
+273 allocs
+0 block
+0 cmdline
+78 goroutine
+273 heap
+0 mutex
+0 profile
+29 threadcreate
+0 trace
+full goroutine stack dump
+Profile Descriptions:
+
+allocs: A sampling of all past memory allocations
+block: Stack traces that led to blocking on synchronization primitives
+cmdline: The command line invocation of the current program
+goroutine: Stack traces of all current goroutines
+heap: A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
+mutex: Stack traces of holders of contended mutexes
+profile: CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.
+threadcreate: Stack traces that led to the creation of new OS threads
+trace: A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.
+```
+
+## Heap
+
+**URL** : `/debug/pprof/heap`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Thread create
+
+**URL** : `/debug/pprof/threadcreate`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Goroutine
+
+**URL** : `/debug/pprof/goroutine`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Allocations
+
+**URL** : `/debug/pprof/allocs`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Block
+
+**URL** : `/debug/pprof/block`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Mutex
+
+**URL** : `/debug/pprof/mutex`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Cmdline
+
+**URL** : `/debug/pprof/cmdline`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Profile
+
+**URL** : `/debug/pprof/profile`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Symbol
+
+**URL** : `/debug/pprof/symbol`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Trace
+
+**URL** : `/debug/pprof/trace`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
diff --git a/versioned_docs/version-0.10.0/assets/application-state.png b/versioned_docs/version-0.10.0/assets/application-state.png
new file mode 100644
index 0000000..e3a9a5b
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/application-state.png differ
diff --git a/versioned_docs/version-0.10.0/assets/architecture.png b/versioned_docs/version-0.10.0/assets/architecture.png
new file mode 100644
index 0000000..a19dcaa
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/architecture.png differ
diff --git a/versioned_docs/version-0.10.0/assets/cpu_profile.jpg b/versioned_docs/version-0.10.0/assets/cpu_profile.jpg
new file mode 100644
index 0000000..7e99f62
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/cpu_profile.jpg differ
diff --git a/versioned_docs/version-0.10.0/assets/dashboard_secret.png b/versioned_docs/version-0.10.0/assets/dashboard_secret.png
new file mode 100644
index 0000000..60b4f97
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/dashboard_secret.png differ
diff --git a/versioned_docs/version-0.10.0/assets/dashboard_token_select.png b/versioned_docs/version-0.10.0/assets/dashboard_token_select.png
new file mode 100644
index 0000000..59173fd
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/dashboard_token_select.png differ
diff --git a/versioned_docs/version-0.10.0/assets/docker-dektop-minikube.png b/versioned_docs/version-0.10.0/assets/docker-dektop-minikube.png
new file mode 100644
index 0000000..48b3584
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/docker-dektop-minikube.png differ
diff --git a/versioned_docs/version-0.10.0/assets/docker-desktop.png b/versioned_docs/version-0.10.0/assets/docker-desktop.png
new file mode 100644
index 0000000..9224360
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/docker-desktop.png differ
diff --git a/versioned_docs/version-0.10.0/assets/fifo-state-example.png b/versioned_docs/version-0.10.0/assets/fifo-state-example.png
new file mode 100644
index 0000000..ca04c17
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/fifo-state-example.png differ
diff --git a/versioned_docs/version-0.10.0/assets/gang_scheduling_iintro.png b/versioned_docs/version-0.10.0/assets/gang_scheduling_iintro.png
new file mode 100644
index 0000000..b3be207
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/gang_scheduling_iintro.png differ
diff --git a/versioned_docs/version-0.10.0/assets/goland_debug.jpg b/versioned_docs/version-0.10.0/assets/goland_debug.jpg
new file mode 100644
index 0000000..c9ab94c
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/goland_debug.jpg differ
diff --git a/versioned_docs/version-0.10.0/assets/goland_ide_pref_imports.png b/versioned_docs/version-0.10.0/assets/goland_ide_pref_imports.png
new file mode 100644
index 0000000..fbd9b00
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/goland_ide_pref_imports.png differ
diff --git a/versioned_docs/version-0.10.0/assets/goland_ide_pref_inspections.png b/versioned_docs/version-0.10.0/assets/goland_ide_pref_inspections.png
new file mode 100644
index 0000000..395e640
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/goland_ide_pref_inspections.png differ
diff --git a/versioned_docs/version-0.10.0/assets/goland_ide_pref_other.png b/versioned_docs/version-0.10.0/assets/goland_ide_pref_other.png
new file mode 100644
index 0000000..77e9908
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/goland_ide_pref_other.png differ
diff --git a/versioned_docs/version-0.10.0/assets/goland_ide_pref_tabs.png b/versioned_docs/version-0.10.0/assets/goland_ide_pref_tabs.png
new file mode 100644
index 0000000..f6b741a
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/goland_ide_pref_tabs.png differ
diff --git a/versioned_docs/version-0.10.0/assets/namespace-mapping.png b/versioned_docs/version-0.10.0/assets/namespace-mapping.png
new file mode 100644
index 0000000..9ad07da
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/namespace-mapping.png differ
diff --git a/versioned_docs/version-0.10.0/assets/node-bin-packing.png b/versioned_docs/version-0.10.0/assets/node-bin-packing.png
new file mode 100644
index 0000000..9267a03
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/node-bin-packing.png differ
diff --git a/versioned_docs/version-0.10.0/assets/node-fair.png b/versioned_docs/version-0.10.0/assets/node-fair.png
new file mode 100644
index 0000000..2e404ab
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/node-fair.png differ
diff --git a/versioned_docs/version-0.10.0/assets/object-state.png b/versioned_docs/version-0.10.0/assets/object-state.png
new file mode 100644
index 0000000..9baca07
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/object-state.png differ
diff --git a/versioned_docs/version-0.10.0/assets/pluggable-app-mgmt.jpg b/versioned_docs/version-0.10.0/assets/pluggable-app-mgmt.jpg
new file mode 100644
index 0000000..443b8ad
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/pluggable-app-mgmt.jpg differ
diff --git a/versioned_docs/version-0.10.0/assets/queue-fairness.png b/versioned_docs/version-0.10.0/assets/queue-fairness.png
new file mode 100644
index 0000000..7c78ed7
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/queue-fairness.png differ
diff --git a/versioned_docs/version-0.10.0/assets/queue-resource-quotas.png b/versioned_docs/version-0.10.0/assets/queue-resource-quotas.png
new file mode 100644
index 0000000..fb72138
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/queue-resource-quotas.png differ
diff --git a/versioned_docs/version-0.10.0/assets/resilience-node-recovery.jpg b/versioned_docs/version-0.10.0/assets/resilience-node-recovery.jpg
new file mode 100644
index 0000000..3847451
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/resilience-node-recovery.jpg differ
diff --git a/versioned_docs/version-0.10.0/assets/resilience-workflow.jpg b/versioned_docs/version-0.10.0/assets/resilience-workflow.jpg
new file mode 100644
index 0000000..40ab6ba
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/resilience-workflow.jpg differ
diff --git a/versioned_docs/version-0.10.0/assets/spark-jobs-on-ui.png b/versioned_docs/version-0.10.0/assets/spark-jobs-on-ui.png
new file mode 100644
index 0000000..dabeb30
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/spark-jobs-on-ui.png differ
diff --git a/versioned_docs/version-0.10.0/assets/spark-pods.png b/versioned_docs/version-0.10.0/assets/spark-pods.png
new file mode 100644
index 0000000..e1f72e0
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/spark-pods.png differ
diff --git a/versioned_docs/version-0.10.0/assets/throughput.png b/versioned_docs/version-0.10.0/assets/throughput.png
new file mode 100644
index 0000000..8ced22c
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/throughput.png differ
diff --git a/versioned_docs/version-0.10.0/assets/yk-ui-screenshots.gif b/versioned_docs/version-0.10.0/assets/yk-ui-screenshots.gif
new file mode 100644
index 0000000..77dec56
Binary files /dev/null and b/versioned_docs/version-0.10.0/assets/yk-ui-screenshots.gif differ
diff --git a/versioned_docs/version-0.10.0/design/architecture.md b/versioned_docs/version-0.10.0/design/architecture.md
new file mode 100644
index 0000000..e7e2569
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/architecture.md
@@ -0,0 +1,62 @@
+---
+id: architecture
+title: Architecture
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Apache YuniKorn (Incubating) is a light-weight, universal resource scheduler for container orchestrator systems.
+It is created to achieve fine-grained resource sharing for various workloads efficiently on a large scale, multi-tenant,
+and cloud-native environment. YuniKorn brings a unified, cross-platform, scheduling experience for mixed workloads that
+consist of stateless batch workloads and stateful services.
+
+YuniKorn now supports K8s and can be deployed as a custom K8s scheduler. YuniKorn's architecture design also allows
+adding different shim layer and adopt to different ResourceManager implementation including Apache Hadoop YARN,
+or any other systems.
+
+## Architecture
+
+Following chart illustrates the high-level architecture of YuniKorn.
+
+<img src={require('./../assets/architecture.png').default} />
+
+## Components
+
+### Scheduler interface
+
+[Scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface) is an abstract layer
+which resource management platform (like YARN/K8s) will speak with, via API like GRPC/programing language bindings.
+
+### Scheduler core
+
+Scheduler core encapsulates all scheduling algorithms, it collects resources from underneath resource management
+platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
+best spot for each request and then sends response allocations to the resource management platform.
+Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface).
+Please read more about the design of schedule core [here](scheduler_core_design.md).
+
+### Kubernetes shim
+
+The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is responsible for translating the Kubernetes
+cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
+And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
+between the shim and the scheduler core is through the [scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface).
+Please read more about the design of the Kubernetes shim [here](k8shim.md)
+
diff --git a/versioned_docs/version-0.10.0/design/cache_removal.md b/versioned_docs/version-0.10.0/design/cache_removal.md
new file mode 100644
index 0000000..9e6c4d4
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/cache_removal.md
@@ -0,0 +1,446 @@
+---
+id: cache_removal
+title: Scheduler cache removal design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+# Proposal to combine Cache and Scheduler's implementation in the core
+This document describes the current state of the scheduler and cache implementation.
+It describes the changes planned based on the analysis that was done of the current behaviour.
+
+## Goals
+The goal is to provide the same functionality before and after the change.
+- unit tests before and after the merge must all pass.
+- Smoke tests defined in the core should all pass without major changes <sup id="s1">[definition](#f1)</sup>.
+- End-to-end tests that are part of the shim code must all pass without changes.
+
+## Background
+The current Scheduler Core is build up around two major components to store the data: the cache and scheduler objects.
+The cache objects form the base for most data to be tracked.
+The Scheduler objects track specific in flight details and are build on top of a cache object.
+
+The communication between the two layers uses a-synchronous events and in some cases direct updates.
+A synchronous update between the scheduler and the cache does mean that there is a short period the scheduler is "out of sync" with the cache.
+This short period can have an impact on the scheduling decisions.
+One of which is logged as [YUNIKORN-169](https://issues.apache.org/jira/browse/YUNIKORN-169).
+
+A further point is the complexity that the two structure brings to the code.
+A distinct set of messages to communicate between the scheduler and the cache.
+A one on one mapping between the scheduler and cache objects shows that the distinction is probably more artificial than required.
+---
+<b id="f1"></b>definition: Major changes for smoke tests are defined as changes to the tests that alter use case and thus test flows. Some changes will be needed as checks made could rely on cache objects which have been removed. [↩](#s1)
+## Structure analysis
+### Objects
+The existing objects as per the code analysis.
+The overlap between the scheduler and the cache objects is shown by showing them at the same line.
+N/A means that there is no equivalent object in either the scheduler or cache.
+
+| Cache Object | Scheduler Object |
+| ------------------------------ | ------------------------------ |
+| ClusterInfo | ClusterSchedulingContext |
+| PartitionInfo | partitionSchedulingContext |
+| AllocationInfo | schedulingAllocation |
+| N/A | schedulingAllocationAsk |
+| N/A | reservation |
+| ApplicationInfo | SchedulingApplication |
+| applicationState | N/A |
+| NodeInfo | SchedulingNode |
+| QueueInfo | SchedulingQueue |
+| SchedulingObjectState | N/A |
+
+The `initializer` code that is part of the cache does not define a specific object.
+It contains a mixture of code defined at the package level and code that is part of the `ClusterInfo` object.
+
+### Events
+Events defined in the core have multiple origins and destinations.
+Some events are only internal for the core between the cache and scheduler.
+These events will be removed.
+
+| Event | Flow | Proposal |
+| ----------------------------------------- | --------------------- | -------- |
+| AllocationProposalBundleEvent | Scheduler -> Cache | Remove |
+| RejectedNewApplicationEvent | Scheduler -> Cache | Remove |
+| ReleaseAllocationsEvent | Scheduler -> Cache | Remove |
+| RemoveRMPartitionsEvent | Scheduler -> Cache | Remove |
+| RemovedApplicationEvent | Scheduler -> Cache | Remove |
+| SchedulerNodeEvent | Cache -> Scheduler | Remove |
+| SchedulerAllocationUpdatesEvent | Cache -> Scheduler | Remove |
+| SchedulerApplicationsUpdateEvent | Cache -> Scheduler | Remove |
+| SchedulerUpdatePartitionsConfigEvent | Cache -> Scheduler | Remove |
+| SchedulerDeletePartitionsConfigEvent | Cache -> Scheduler | Remove |
+| RMApplicationUpdateEvent (add/remove app) | Cache/Scheduler -> RM | Modify |
+| RMRejectedAllocationAskEvent | Cache/Scheduler -> RM | Modify |
+| RemoveRMPartitionsEvent | RM -> Scheduler | |
+| RMUpdateRequestEvent | RM -> Cache | Modify |
+| RegisterRMEvent | RM -> Cache | Modify |
+| ConfigUpdateRMEvent | RM -> Cache | Modify |
+| RMNewAllocationsEvent | Cache -> RM | Modify |
+| RMReleaseAllocationEvent | Cache -> RM | Modify |
+| RMNodeUpdateEvent | Cache -> RM | Modify |
+| | | |
+
+Events that are handled by the cache will need to be handled by the core code after the removal of the cache.
+Two events are handled by the cache and the scheduler.
+
+## Detailed flow analysis
+### Object existing in both cache and scheduler
+The current design is based on the fact that the cache object is the basis for all data storage.
+Each cache object must have a corresponding scheduler object.
+The contract in the core around the cache and scheduler objects was simple.
+If the object exists in both scheduler and cache the object will be added to cache triggering the creation of the corresponding scheduler object.
+Removing the object is always handled in reverse: first from the scheduler which will trigger the removal from the cache.
+An example would be the creation of an application triggered by the `RMUpdateRequestEvent` would be processed by the cache.
+Creating a `SchedulerApplicationsUpdateEvent` to create the corresponding application in the scheduler.
+
+When the application and object state were added they were added into the cache objects.
+The cache objects were considered the data store and thus also contain the state.
+There were no corresponding state objects in the scheduler.
+Maintaining two states for the same object is not possible.
+
+The other exceptions to that rule are two objects that were considered volatile and scheduler only.
+The `schedulingAllocationAsk` tracks outstanding requests for an application in the scheduler.
+The `reservation` tracks a temporary reservation of a node for an application and ask combination.
+
+### Operations to add/remove app
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request an application to be added or removed.
+
+**application add or delete**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processApplicationUpdateFromRMUpdate
+ 2.1: Add new apps to the partition.
+ 2.2: Send removed apps to scheduler (but not remove anything from cache)
+3. scheduler.processApplicationUpdateEvent
+ 3.1: Add new apps to scheduler
+ (when fails, send RejectedNewApplicationEvent to cache)
+ No matter if failed or not, send RMApplicationUpdateEvent to RM.
+ 3.2: Remove app from scheduler
+ Send RemovedApplicationEvent to cache
+```
+
+### Operations to remove allocations and add or remove asks
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request an allocation to be removed.
+The RM can request an ask to be added or removed
+
+**allocation delete**
+This describes the allocation delete initiated by the RM only
+````
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: (by-pass): Send to scheduler via event SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: Update ReconcilePlugin
+ 3.2: Send confirmation of the releases back to Cache via event ReleaseAllocationsEvent
+4. cluster_info.processAllocationReleases to process the confirmed release
+````
+
+**ask add**
+If the ask already exists this add is automatically converted into an update.
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: Ask sanity check (such as existence of partition/app), rejections are send back to the RM via RMRejectedAllocationAskEvent
+ 2.2: pass checked asks to scheduler via SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: Update scheduling application with the new or updated ask.
+ 3.2: rejections are send back to the RM via RMRejectedAllocationAskEvent
+ 3.3: accepted asks are not confirmed to RM or cache
+```
+
+**ask delete**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: (by-pass): Send to scheduler via event SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationReleaseByAllocationKey
+ 3.1: Update scheduling application and remove the ask.
+```
+
+### Operations to add, update or remove nodes
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request a node to be added, updated or removed.
+
+**node add**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewSchedulableNodes
+ 2.1: node sanity check (such as existence of partition/node)
+ 2.2: Add new nodes to the partition.
+ 2.3: notify scheduler of new node via SchedulerNodeEvent
+3. notify RM of node additions and rejections via RMNodeUpdateEvent
+ 3.1: notify the scheduler of allocations to recover via SchedulerAllocationUpdatesEvent
+4. scheduler.processAllocationUpdateEvent
+ 4.1: scheduler creates a new ask based on the Allocation to recover
+ 4.2: recover the allocation on the new node using a special process
+ 4.3: confirm the allocation in the scheduler, on failure update the cache with a ReleaseAllocationsEvent
+```
+
+**node update and removal**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNodeActions
+ 2.1: node sanity check (such as existence of partition/node)
+ 2.2: Node info update (resource change)
+ 2.2.1: update node in cache
+ 2.2.2: notify scheduler of the node update via SchedulerNodeEvent
+ 2.3: Node status update (not removal), update node status in cache only
+ 2.4: Node removal
+ 2.4.1: update node status and remove node from the cache
+ 2.4.2: remove alloations and inform RM via RMReleaseAllocationEvent
+ 2.4.3: notify scheduler of the node removal via SchedulerNodeEvent
+3. scheduler.processNodeEvent add/remove/update the node
+```
+
+### Operations to add, update or remove partitions
+**Add RM**
+```
+1. RMProxy sends commonevents.RemoveRMPartitionsEvent
+ if RM is already registered
+ 1.1: scheduler.removePartitionsBelongToRM
+ 1.1.1: scheduler cleans up
+ 1.1.2: scheduler sends commonevents.RemoveRMPartitionsEvent
+ 1.2: cluster_info.processRemoveRMPartitionsEvent
+ 1.2.1: cache cleans up
+2. RMProxy sends commonevents.RegisterRMEvent
+3. cluster_info.processRMRegistrationEvent
+ 2.1: cache update internal partitions/queues accordingly.
+ 2.2: cache sends to scheduler SchedulerUpdatePartitionsConfigEvent.
+3. scheduler.processUpdatePartitionConfigsEvent
+ 3.1: Scheduler update partition/queue info accordingly.
+```
+
+**Update and Remove partition**
+Triggered by a configuration file update.
+```
+1. RMProxy sends commonevents.ConfigUpdateRMEvent
+2. cluster_info.processRMConfigUpdateEvent
+ 2.1: cache update internal partitions/queues accordingly.
+ 2.2: cache sends to scheduler SchedulerUpdatePartitionsConfigEvent.
+ 2.3: cache marks partitions for deletion (not removed yet).
+ 2.4: cache sends to scheduler SchedulerDeletePartitionsConfigEvent
+3. scheduler.processUpdatePartitionConfigsEvent
+ 3.1: scheduler updates internal partitions/queues accordingly.
+4. scheduler.processDeletePartitionConfigsEvent
+ 4.1: Scheduler set partitionManager.stop = true.
+ 4.2: PartitionManager removes queues, applications, nodes async.
+ This is the REAL CLEANUP including the cache
+```
+
+### Allocations
+Allocations are initiated by the scheduling process.
+The scheduler creates a SchedulingAllocation on the scheduler side which then gets wrapped in an AllocationProposal.
+The scheduler has checked resources etc already and marked the allocation as inflight.
+This description picks up at the point the allocation will be confirmed and finalised.
+
+**New allocation**
+```
+1. Scheduler wraps an SchedulingAllocation in an AllocationProposalBundleEvent
+2. cluster_info.processAllocationProposalEvent
+ preemption case: release preempted allocations
+ 2.1: release the allocation in the cache
+ 2.2: inform the scheduler the allocation is released via SchedulerNodeEvent
+ 2.3: inform the RM the allocation is released via RMReleaseAllocationEvent
+ all cases: add the new allocation
+ 2.4: add the new allocation to the cache
+ 2.5: rejections are send back to the scheduler via SchedulerAllocationUpdatesEvent
+ 2.6: inform the scheduler the allocation is added via SchedulerAllocationUpdatesEvent
+ 2.7: inform the RM the allocation is added via RMNewAllocationsEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: confirmations are added to the scheduler and change from inflight to confirmed.
+ On failure of processing a ReleaseAllocationsEvent is send to the cache *again* to clean up.
+ This is part of the issue in [YUNIKORN-169]
+ cluster_info.processAllocationReleases
+ 3.2: rejections remove the inflight allocation from the scheduler.
+```
+
+## Current locking
+**Cluster Lock:**
+A cluster contains one or more Partition objects. A partition is a sub object of Cluster.
+Adding or Removing ANY Partition requires a write-lock of the cluster.
+Retrieving any object within the cluster will require iterating over the Partition list and thus a read-lock of the cluster
+
+**Partition Lock:**
+The partition object contains all links to Queue, Application or Node objects.
+Adding or Removing ANY Queue, Application or Node needs a write-lock of the partition.
+Retrieving any object within the partition will require a read-lock of the partition to prevent data races
+
+Examples of operation needing a write-lock
+- Allocation processing after scheduling, will change application, queue and node objects.
+ Partition lock is required due to possible updates to reservations.
+- Update of Node Resource
+ It not only affect node's available resource, it also affects the Partition's total allocatable Resource
+
+Example of operations that need a read-lock:
+- Retrieving any Queue, Application or Node needs a read-lock
+ The object itself is not locked as part of the retrieval
+- Confirming an allocation after processing in the cache
+ The partition is only locked for reading to allow retrieval of the objects that will be changed.
+ The changes are made on the underlying objects.
+
+Example of operations that do not need any lock:
+- Scheduling
+ Locks are taken on the specific objects when needed, no direct updates to the partition until the allocation is confirmed.
+
+**Queue lock:**
+A queue can track either applications (leaf type) or other queues (parent type).
+Resources are tracked for both types in the same way.
+
+Adding or removing an Application (leaf type), or a direct child queue (parent type) requires a write-lock of the queue.
+Updating tracked resources requires a write-lock.
+Changes are made recursively never locking more than 1 queue at a time.
+Updating any configuration property on the queue requires a write-lock.
+Retrieving any configuration value, or tracked resource, application or queue requires a read-lock.
+
+Examples of operation needing a write-lock
+- Adding an application to a leaf queue
+- Updating the reservations
+
+Examples of operation needing a read-lock
+- Retrieving an application from a leaf type queue
+- Retrieving the pending resources
+
+**Application lock:**
+An application tracks resources of different types, the allocations and outstanding requests.
+Updating any tracked resources, allocations or requests requires a write-lock.
+Retrieving any of those values requires a read-lock.
+
+Scheduling also requires a write-lock of the application.
+During scheduling the write-lock is held for the application.
+Locks will be taken on the node or queue that need to be accessed or updated.
+Examples of the locks taken on other objects are:
+- a read lock to access queue tracked resources
+- a write-lock to update the in progress allocations on the node
+
+Examples of operation needing a write-lock
+- Adding a new ask
+- Trying to schedule a pending request
+
+Examples of operation needing a read-lock
+- Retrieving the allocated resources
+- Retrieving the pending requests
+
+**Node lock:**
+An node tracks resources of different types and allocations.
+Updating any tracked resources or allocations requires a write-lock.
+Retrieving any of those values requires a read-lock.
+
+Checks run during the allocation phases take locks as required.
+Read-locks when checking write-locks when updating.
+A node is not locked for the whole allocation cycle.
+
+Examples of operation needing a write-lock
+- Adding a new allocation
+- updating the node resources
+
+Examples of operation needing a read-lock
+- Retrieving the allocated resources
+- Retrieving the reservation status
+
+## How to merge Cache and scheduler objects
+Since there is no longer the requirement to distinguish the objects in the cache and scheduler the `scheduling` and `info` parts of the name will be dropped.
+
+Overview of the main moves and merges:
+1. `application_info` & `scheduling_application`: **merge** to `scheduler.object.application`
+2. `allocation_info` & `scheduling_allocation`: **merge** to `scheduler.object.allocation`
+3. `node_info` & `scheduling_node`: **merge** to `scheduler.object.node`
+4. `queue_info` & `scheduling_queue`: **merge** to `scheduler.object.queue`
+5. `partition_info` & `scheduling_partition`: **merge** to `scheduler.PartitionContext`
+6. `cluster_info` & `scheduling_context`: **merge** to `scheduler.ClusterContext`
+7. `application_state`: **move** to `scheduler.object.applicationState`
+8. `object_state`: **move** to `scheduler.object.objectState`
+9. `initializer`: **merge** into `scheduler.ClusterContext`
+
+This move and merge of code includes a refactor of the objects into their own package.
+That thus affects the two scheduler only objects, reservations and schedulingAsk, that are already defined.
+Both will be moved into the objects package.
+
+The top level scheduler package remains for the contexts and scheduling code.
+
+## Code merges
+The first change is the event processing.
+All RM events will now directly be handled in the scheduler.
+Event handling will undergo a major change, far more than a simple merge.
+Only the RM generated events will be left after the merge.
+As described in the analysis above the scheduler is, in almost all cases, notified of changes from RM events.
+
+Broadly speaking there are only three types of changes triggered by the event removal:
+- configuration changes: new scheduler code required as the cache handling is not transferable to the scheduler
+- node, ask and application changes: merge of the cache code into the scheduler
+- allocation changes: removal of confirmation cycle and simplification of the scheduler code
+
+Part of the event handling is the processing of the configuration changes.
+All configuration changes will now update the scheduler objects directly.
+The way the scheduler works is slightly different from the cache which means the code is not transferable.
+
+Nodes and applications are really split between the cache and scheduler.
+Anything that is tracked in the cache object that does not have an equivalent value in the scheduler object will be moved into the scheduler object.
+All references to scheduler objects will be removed.
+With the code merges existing scheduler code that calls out directly into the cache objects will return the newly tracked value in the scheduler object.
+These calls will thus become locked calls in the scheduler.
+
+The concept of an in flight allocation will be removed.
+Allocation will be made in the same scheduling iteration without events or creation of a proposal.
+Removing the need for tracking of allocating resources on the scheduler objects.
+In flight resource tracking was required to make sure that an allocation while not confirmed by the cache would being taken into account while making scheduling decisions.
+
+The application and object state will be an integrated part of the scheduler object.
+A state change is thus immediate and this should prevent an issue like [YUNIKORN-169](https://issues.apache.org/jira/browse/YUNIKORN-169) from occuring.
+
+## Locking after merge
+
+### Direction of lock
+It is possible to acquire another lock while holding a lock, but we need to make sure that we do not allow:
+- Holding A.lock and acquire B's lock.
+- Holding B.lock and acquire B's lock.
+
+The current code in the scheduler takes a lock as late as possible and only for the time period needed.
+Some actions are not locked on the scheduler side just on the cache side as each object has its own lock.
+This means that a read of a value from the cache would not lock the scheduling object.
+
+With the integration of the cache into the scheduler the number of locks will decrease as the number of objects decreases.
+Each equivalent object, cache and scheduler, which used to have their own lock will now have just one.
+After the merge of the code is performed one lock will be left.
+Locking will occur more frequently as the number of fields in the scheduler objects has increased.
+
+Calls that did not lock the scheduler object before the merge will become locked.
+Lock contention could lead to performance degradation.
+The reduced overhead in objects and event handling can hopefully compensate for this.
+One point to keep track of is the change in locking behaviour.
+New behaviour could lead to new deadlock situations when code is simply merged without looking at the order.
+
+### Mitigations for deadlocks
+The locking inside the scheduler will be left as is.
+This means that the main scheduling logic will be taking and releasing locks as required on the objects.
+There are no long held read-locks or write-locks until the application is locked to schedule it.
+
+A major point of attention will need to be that no iterations of objects should be performed while holding on to a lock.
+For instance during scheduling while iterating over a queue's application we should not lock the queue.
+
+Another example would be that event processing in the partition should not lock the partition unneeded.
+The partition should be locked while retrieving for instance the node that needs updating and release the lock before it tries to lock the node itself.
+
+This approach fits in with the current locking approach and will keep the locking changes to a minimum.
+Testing, specifically end-to-end testing, should catch these deadlocks.
+There are no known tools that could be used to detect or describe lock order.
diff --git a/versioned_docs/version-0.10.0/design/cross_queue_preemption.md b/versioned_docs/version-0.10.0/design/cross_queue_preemption.md
new file mode 100644
index 0000000..51c8033
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/cross_queue_preemption.md
@@ -0,0 +1,126 @@
+---
+id: cross_queue_preemption
+title: Cross Queue Preemption
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Problems:
+
+According to lessons we learned from YARN Scheduler preemption.
+
+**Here're top bad things:**
+
+- Preemption is a shotgun instead of a sniper, when a preemption decision is made, nobody knows if preempted resources will go to demanding queue/app/user or not.
+- Preemption logic and allocation is different, we have to implement (and mimic) what we have done in scheduler allocation logic.
+
+**Here're top good things:**
+
+- Preemption is fast (thanks to the shotgun), reclaiming thousands of containers only takes ~ 1 sec.
+- We have understand how painful it is to handle DRF, multiple preemption policies (inter/intra-queue, shotgun/surgical preemption, etc.) And we have developed some good logic
+to make sure better modularization and plug-ability
+
+## Answer some questions for design/implementation choices
+
+**1\. Do we really want preemption-delay? (Or we just want to control pace)**
+
+In CS, we have preemption-delay, which select victims in preemption candidates, wait for a certain time before killing it.
+
+The purposes of preemption delay are: a. give heads-up time to apps so
+they can prepare bad things happen (unfortunately no app do anything for these heads up, at least from what I knew). b. control preemption pace.
+
+And in practice, I found it causes a lot of issues, for example when a
+cluster state keep changing, it is very hard to ensure accurate preemption.
+
+**Proposal:**
+
+Remove the preemption-delay, keep the logics of controlling preemption pace. (such as ```yarn.resourcemanager.monitor.capacity.preemption
+.total_preemption_per_round```). And we can do allocation together with preemption.
+This don't mean containers will be stopped immediately after preemption issued. Instead, RM can control delays between signal a container and kill a container. Such as grace
+termination of POD in K8s: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
+
+**2\. Do we want to do preemption for every scheduling logic, or we can do periodically?**
+
+In CS, we have preemption logic runs periodically, like every 1 sec or 3 sec.
+
+Since preemption logic involves some heavy logics, like calculating shares of queues/apps. And when doing accurate preemption, we may need to scan nodes for preemption candidate.
+Considering this, I propose to have preemption runs periodically. But it is important to note that, we need to try to use as much code as possible for
+allocation-inside-preemption, otherwise there will be too much duplicated logic and very hard to be maintained in the future.
+
+**3\. Preemption cost and function**
+
+We found it is helpful to add cost for preemption, such as container live time, priority, type of container. It could be a cost function (Which returns a numeric value) or it
+could be a comparator (which compare two allocations for preemption ask).
+
+## Pseudo code
+
+Logic of allocation (invoked every allocation cycle)
+
+```
+input:
+ - nAlloc, allocate N allocations for this allocation cycle.
+
+for partition:
+ askCandidates := findAskCandidates(nAlloc, preemption=false)
+
+ allocated, failed_to_allocated := tryAllocate(askCandidates);
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ nAlloc -= len(allocated)
+```
+
+Logic of preemption (invoked every preemption cycle)
+
+```
+// It has to be done for every preemption-policy because calculation is different.
+for preemption-policy:
+ preempt_results := policy.preempt()
+ for preempt_results:
+ send-preempt-result-to-cache-to-commit;
+ updated-missed-opportunity (allocated)
+```
+
+Inside preemption policy
+
+```
+inter-queue-preempt-policy:
+ calculate-preemption-quotas;
+
+ for partitions:
+ total_preempted := resource(0);
+
+ while total_preempted < partition-limited:
+ // queues will be sorted by allocating - preempting
+ // And ignore any key in preemption_mask
+ askCandidates := findAskCandidates(N, preemption=true)
+
+ preempt_results := tryAllocate(askCandidates, preemption=true);
+
+ total_preempted += sigma(preempt_result.allocResource)
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ update-preemption-mask(askCandidates.allocKeys - preempt_results.allocKeys)
+```
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/design/k8shim.md b/versioned_docs/version-0.10.0/design/k8shim.md
new file mode 100644
index 0000000..39ca4d1
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/k8shim.md
@@ -0,0 +1,83 @@
+---
+id: k8shim
+title: Kubernetes Shim Design
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Github repo: https://github.com/apache/incubator-yunikorn-k8shim
+
+Please read the [architecture](architecture.md) doc before reading this one, you will need to understand
+the 3 layer design of YuniKorn before getting to understand what is the Kubernetes shim.
+
+## The Kubernetes shim
+
+The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is responsible for translating the Kubernetes
+cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
+And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
+between the shim and the scheduler core is through the scheduler-interface.
+
+## The admission controller
+
+The admission controller runs in a separate pod, it runs a
+[mutation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook)
+and a [validation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook), where:
+
+1. The `mutation webhook` mutates pod spec by:
+ - adding `schedulerName: yunikorn`
+ - by explicitly specifying the scheduler name, the pod will be scheduled by YuniKorn scheduler
+ - adding `applicationId` label
+ - when a label `applicationId` exists, reuse the given applicationId
+ - when a label `spark-app-selector` exists, reuse the given spark app ID
+ - otherwise, assign a generated application ID for this pod, using convention: `yunikorn-<namespace>-autogen`. this is unique per namespace.
+ - adding `queue` label
+ - when a label `queue` exists, reuse the given queue name. Note, if placement rule is enabled, values set in the label is ignored
+ - otherwise, adds `queue: root.default`
+2. The `validation webhook` validates the configuration set in the configmap
+ - this is used to prevent writing malformed configuration into the configmap
+ - the validation webhook calls scheduler [validation REST API](api/scheduler.md#configuration-validation) to validate configmap updates
+
+### Admission controller deployment
+
+Currently, the deployment of the admission-controller is done as a `post-start` hook in the scheduler deployment, similarly, the
+uninstall is done as a `pre-stop` hook. See the related code [here](https://github.com/apache/incubator-yunikorn-release/blob/56e580af24ed3433e7d73d9ea556b19ad7b74337/helm-charts/yunikorn/templates/deployment.yaml#L80-L85).
+During the installation, it is expected to always co-locate the admission controller with the scheduler pod, this is done
+by adding the pod-affinity in the admission-controller pod, like:
+
+```yaml
+podAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: component
+ operator: In
+ values:
+ - yunikorn-scheduler
+ topologyKey: "kubernetes.io/hostname"
+```
+
+it also tolerates all the taints in case the scheduler pod has some toleration set.
+
+```yaml
+tolerations:
+- operator: "Exists"
+```
+
+
diff --git a/versioned_docs/version-0.10.0/design/namespace_resource_quota.md b/versioned_docs/version-0.10.0/design/namespace_resource_quota.md
new file mode 100644
index 0000000..90830b6
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/namespace_resource_quota.md
@@ -0,0 +1,183 @@
+---
+id: namespace_resource_quota
+title: Namespace Resource Quota
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+In K8s, user can setup namespace with [resource quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) to limit aggregated resource consumption in this namespace. The validation of namespace resource quotas is handled in api-server directly, therefore YuniKorn simply honors the quotas like the default scheduler.
+
+## Best practice
+
+It is not mandatory to setup YuniKorn queues with respect of namespaces.
+However, in practice, it makes more sense to do so.
+Namespace is often used to set a cap for resource consumptions per user-group/team,
+YuniKorn queue is also meant to divide cluster resource into multiple groups.
+Let's go through an example.
+
+### 1. Setup namespace
+
+Namespace: `advertisement`:
+```
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+ name: advertisement
+spec:
+ hard:
+ requests.cpu: "200m"
+ requests.memory: 2000Mi
+ limits.cpu: "200m"
+ limits.memory: 4000Mi
+```
+Create the namespace
+```
+kubectl create namespace advertisement
+kubectl create -f ./advertisement.yaml --namespace=advertisement
+kubectl get quota --namespace=advertisement
+kubectl describe quota advertisement --namespace=advertisement
+
+// output
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 0 200m
+limits.memory 0 4000Mi
+requests.cpu 0 200m
+requests.memory 0 2000Mi
+```
+
+### 2. Setup YuniKorn queues
+
+Queue: `advertisement`:
+```
+name: advertisement
+resources:
+ guaranteed:
+ vcore: 100
+ memory: 1000
+ max:
+ vcore: 200
+ memory: 2000
+```
+
+ensure `QueueMaxResource <= NamespaceResourceQuotaRequests`
+
+### 3. Mapping applications to queues & namespace
+
+In a pod spec
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00001"
+ queue: "root.advertisement"
+ name: task0
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "800M"
+ limits:
+ cpu: "100m"
+ memory: "1000M"
+```
+
+Check Quota
+
+```
+kubectl describe quota advertisement --namespace=advertisement
+
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 100m 200m
+limits.memory 1G 4000Mi
+requests.cpu 50m 200m
+requests.memory 800M 2000Mi
+```
+
+Now submit another application,
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00002"
+ queue: "root.advertisement"
+ name: task1
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "200m"
+ memory: "800M"
+ limits:
+ cpu: "200m"
+ memory: "1000M"
+```
+
+pod will not be able to submitted to api-server, because the requested cpu `200m` + used cpu `100m` = `300m` which exceeds the resource quota.
+
+```
+kubectl create -f pod_ns_adv_task1.yaml
+Error from server (Forbidden): error when creating "pod_ns_adv_task1.yaml": pods "task1" is forbidden: exceeded quota: advertisement, requested: limits.cpu=200m,requests.cpu=200m, used: limits.cpu=100m,requests.cpu=50m, limited: limits.cpu=200m,requests.cpu=200m
+```
+
+## Future Work
+
+For compatibility, we should respect namespaces and resource quotas.
+Resource quota is overlapped with queue configuration in many ways,
+for example the `requests` quota is just like queue's max resource. However,
+there are still a few features resource quota can do but queue cannot, such as
+
+1. Resource `limits`. The aggregated resource from all pods in a namespace cannot exceed this limit.
+2. Storage Resource Quota, e.g storage size, PVC number, etc.
+3. Object Count Quotas, e.g count of PVCs, services, configmaps, etc.
+4. Resource Quota can map to priority class.
+
+Probably we can build something similar to cover (3) in this list.
+But it would be hard to completely support all these cases.
+
+But currently, setting applications mapping to a queue as well as a corresponding namespace is over complex.
+Some future improvements might be:
+
+1. Automatically detects namespaces in k8s-shim and map them to queues. Behind the scenes, we automatically generates queue configuration based on namespace definition. Generated queues are attached under root queue.
+2. When new namespace added/updated/removed, similarly to (1), we automatically update queues.
+3. User can add more configuration to queues, e.g add queue ACL, add child queues on the generated queues.
+4. Applications submitted to namespaces are transparently submitted to corresponding queues.
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/design/pluggable_app_management.md b/versioned_docs/version-0.10.0/design/pluggable_app_management.md
new file mode 100644
index 0000000..d297ada
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/pluggable_app_management.md
@@ -0,0 +1,75 @@
+---
+id: pluggable_app_management
+title: Pluggable App Management
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## The Problem
+
+Currently, we schedule and group an application is based on a label on the pod.
+This generic way works for any type of workload. It does however give us a limited information on the lifecycle
+and application. On the K8s side, operators have been introduced to provide more detail on the application
+and help scheduling. We cannot use them currently and want to add that functionality.
+
+## K8s Operator Pattern
+
+[K8s operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/)
+is a pattern in K8s to manage applications, it's a handy way to manage application's lifecycle out-of-box on K8s.
+You define several CRDs and some controllers to monitor and mutate the state of the application based on the CRD definition.
+
+For example in [spark-k8s-operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator),
+it defines a CRD called `SparkApplication`, the controller watches the events of add/update/delete of this CRD
+and trigger corresponding actions on event notifications. The `SparkApplication` looks like
+[this example](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/examples/spark-pi.yaml). There
+are a lot more popular operators, such as [flink-k8s-operator](https://github.com/GoogleCloudPlatform/flink-on-k8s-operator),
+ [tf-operator](https://github.com/kubeflow/tf-operator), [pytorch-operator](https://github.com/kubeflow/pytorch-operator), etc.
+
+Use Spark as an example. YuniKorn is able to schedule resources for all pods in K8s, that seamlessly supports Spark. It
+works with [native Spark on K8s](https://spark.apache.org/docs/latest/running-on-kubernetes.html), or
+[spark on K8s with operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/design.md#architecture),
+you'll find the difference from the design architecture chart from the given link. To support native Spark on K8s,
+YuniKorn reads pods' spec and group Spark pods by a label-selector, based on `spark-app-selector`.
+The operator approach gives us more context about the Spark job, such as a better understanding about job state.
+But all these info requires us to look at `SparkApplication` CRD, currently, there is no neat way to
+add such functionality. That's why we need to design a flexible approach to support 3rd party operators
+(retrieving info from their CRDs), so we can easily integrate with other operators with small effort.
+
+## Design
+
+The key issue here is we need a app-management interface, that can be easily extended.
+It needs to be decoupled with existing scheduling logic. For each operator, we create a service to manage this type app's lifecycle,
+and communicate with the scheduling cache independently. The high-level design looks like below:
+
+
+
+Where
+- `AppManagementService` is a composite set of services that can be managed together.
+- `AppManager` is a specific app management service for a particular type of application. In each service, it has
+ access to K8s clients, such as informers, listers, in order to monitor CRD events. And it collects necessary info
+ and talk with scheduler cache through `AMProtocol`.
+- `APIProvider` encapsulate a set of useful APIs that can be shared, such as kube-client, pod/node/storage informers, etc.
+ Each of such informers, it can be shared with multiple app managers, to avoid the overhead.
+- `AMProtocol` defines the basic interaction contract between app manager and the scheduler cache, that helps the cache
+ to performs app lifecycle management without understanding what type of the application it is.
+
+In the upon chart, the AppManagementService has 2 services, the _general_ one is managing normal applications, that
+recognizes applications by pod labels; the _spark-k8s-operator_ one watches `SparkApplication` CRD and manage jobs'
+lifecycle defined by this CRD.
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/design/predicates.md b/versioned_docs/version-0.10.0/design/predicates.md
new file mode 100644
index 0000000..f750cbd
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/predicates.md
@@ -0,0 +1,80 @@
+---
+id: predicates
+title: Support K8s Predicates
+---
+
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+## Design
+
+Predicates are a set of pre-registered functions in K8s, the scheduler invokes these functions to check if a pod
+is eligible to be allocated onto a node. Common predicates are: node-selector, pod affinity/anti-affinity etc. To support
+these predicates in YuniKorn, we don't intend to re-implement everything on our own, but to re-use the core predicates
+code as much as possible.
+
+YuniKorn-core is agnostic about underneath RMs, so the predicates functions are implemented in K8s-shim as a `SchedulerPlugin`.
+SchedulerPlugin is a way to plug/extend scheduler capabilities. Shim can implement such plugin and register itself to
+yunikorn-core, so plugged function can be invoked in the scheduler core. Find all supported plugins in
+[types](https://github.com/apache/incubator-yunikorn-core/blob/master/pkg/plugins/types.go).
+
+## Workflow
+
+First, RM needs to register itself to yunikorn-core, it advertises what scheduler plugin interfaces are supported.
+E.g a RM could implement `PredicatePlugin` interface and register itself to yunikorn-core. Then yunikorn-core will
+call PredicatePlugin API to run predicates before making allocation decisions.
+
+
+Following workflow demonstrates how allocation looks like when predicates are involved.
+
+```
+pending pods: A, B
+shim sends requests to core, including A, B
+core starts to schedule A, B
+ partition -> queue -> app -> request
+ schedule A (1)
+ run predicates (3)
+ generate predicates metadata (4)
+ run predicate functions one by one with the metadata
+ success
+ proposal: A->N
+ schedule B (2)
+ run predicates (calling shim API)
+ generate predicates metadata
+ run predicate functions one by one with the metadata
+ success
+ proposal: B->N
+commit the allocation proposal for A and notify k8s-shim
+commit the allocation proposal for B and notify k8s-shim
+shim binds pod A to N
+shim binds pod B to N
+```
+
+(1) and (2) are running in parallel.
+
+(3) yunikorn-core calls a `schedulerPlugin` API to run predicates, this API is implemented on k8s-shim side.
+
+(4) K8s-shim generates metadata based on current scheduler cache, the metadata includes some intermittent states about nodes and pods.
+
+## Predicates White-list
+
+Intentionally, we only support a white-list of predicates. Majorly due to 2 reasons,
+* Predicate functions are time-consuming, it has negative impact on scheduler performance. To support predicates that are only necessary can minimize the impact. This will be configurable via CLI options;
+* The implementation depends heavily on K8s default scheduler code, though we reused some unit tests, the coverage is still a problem. We'll continue to improve the coverage when adding new predicates.
+
+the white-list currently is defined in [DefaultSchedulerPolicy](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/pkg/plugin/predicates/predictor.go).
diff --git a/versioned_docs/version-0.10.0/design/resilience.md b/versioned_docs/version-0.10.0/design/resilience.md
new file mode 100644
index 0000000..c68a708
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/resilience.md
@@ -0,0 +1,144 @@
+---
+id: resilience
+title: Resilience
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This is not a HA (High-availability) design, HA implies that a service can
+survive from a fatal software/hardware failure. That requires one or more
+standby instances providing same services to take over active instance on failures.
+Resilience here means for YuniKorn, we can restart it without losing its state.
+
+## The problem
+
+YuniKorn is designed as a stateless service, it doesn't persist its state, e.g
+applications/queues/allocations etc, to any persistent storage. All states are
+in memory only. This design ensures YuniKorn to be able to response requests with
+low latency, and deployment mode is simple. However, a restart (or recovery) will
+have the problem to lose state data. We need a decent way to reconstruct all
+previous states on a restart.
+
+## Design
+
+### Workflow
+
+Scheduler core has no notion of "state", which means it does not know if it is under recovering.
+It is too complex to maintain a series of `scheduler states` in both core and shim, because we must
+keep them in-sync. However, if we live under a simple assumption: **scheduler core only responses
+requests, the correction of requests is ensured by shim according its current state**.
+The design becomes much simpler. This way, the shim maintains a state machine like below. When
+it is under `running` state, it sends new requests to the scheduler core as long as a new one is found;
+when under `recovering` state, it collects previous allocations and send recovery messages to
+the scheduler core, and waiting for recovery to be accomplished.
+
+Shim scheduler state machine
+
+```
+ Register Recover Success
+New -----------> Registered -----------> Recovering ----------> Running
+ | Fail
+ --------> Failed
+```
+
+Following chart illustrate how yunikorn-core and shim works together on recovery.
+
+
+
+Restart (with recovery) process
+- yunikorn-shim registers itself with yunikorn-core
+- shim enters "recovering" state. Under "recovering" state, the shim only scans existing nodes and allocations, no new scheduling requests will be sent.
+ - shim scans existing nodes from api-server and added them to cache
+ - shim scans existing pods from api-server, filter out the pods that already assigned (scheduled to a node), and added that to cache (allocation in that node)
+ - shim sends update request to yunikorn-core with the info found in previous steps
+- yunikorn-core handles update requests, the steps should look like a replay of allocation process, including
+ - adding node
+ - adding applications
+ - adding allocations
+ - modifying queue resources
+ - update partition info
+- when all nodes are fully recovered, shim transits the state to "running"
+- shim notifies yunikorn-core that recovery is done, then yunikorn-core transits to "running" state.
+
+### How to determine recovery is complete?
+
+Shim queries K8s api-server to get how many nodes were available in this cluster. It tracks the recovering status of each node.
+Once all nodes are recovered, it can claim the recovery is completed. This approach requires us to add `recovering` and `recovered`
+states to nodes' state machine in the shim.
+
+### Node recovery
+
+In the shim layer, it maintains states for each node and pods running on this node. When start to recover nodes,
+all nodes initially are considered as under `recovering`. Only when all pods running on this node are fully recovered,
+the node can be considered as `recovered`.
+
+
+
+Like demonstrated on upon diagram,
+
+- Node0 is still recovering because pod0 is recovering.
+- Node1 is recovered (become schedulable) because all pods on this node have been recovered.
+- Node2 is lost, shim lost contact with this node. If after sometime this node comes back, shim should still try to recover this node.
+
+### Requests for recovery
+
+During recovery process, shim needs to collect all known information of applications, nodes and allocations from the underneath
+Resource Manager and use them for recovery.
+
+#### Applications
+
+Existing applications must be recovered first before allocations. Shim needs to scan all existing applications
+from nodes, and add applications info as a list of `AddApplicationRequest` in the `UpdateRequest`. This is same
+as the fresh application submission.
+
+```
+message AddApplicationRequest {
+ string applicationID = 1;
+ string queueName = 2;
+ string partitionName = 3;
+}
+```
+
+#### Nodes and allocations
+
+Once a shim is registered to the scheduler-core, subsequent requests are sent via `UpdateRequest#NewNodeInfo`
+(see more from [si.proto](https://github.com/apache/incubator-yunikorn-scheduler-interface/blob/master/si.proto)).
+The structure of the messages looks like,
+
+```
+message NewNodeInfo {
+ // nodeID
+ string nodeID = 1;
+ // optional node attributes
+ map<string, string> attributes = 2;
+ // total node resource
+ Resource schedulableResource = 3;
+ // existing allocations on this node
+ repeated Allocation existingAllocations = 4;
+}
+```
+Shim needs to scan all existing allocations on a node and wrap these info up as a `NewNodeInfo`, add that to a
+`UpdateRequest` and then send to scheduler-core.
+
+**Note**: the recovery of existing allocations depend on the existence of applications, which means applications must
+be recovered first. Since scheduler-core handles `UpdateRequest` one by one, it is required that all existing allocations
+in a `UpdateRequest` must from known applications or new applications embedded within the same `UpdateRequest`, which can be
+specified in `NewApplications` field. Scheduler-core ensures `NewApplications` are always processed first.
+
diff --git a/versioned_docs/version-0.10.0/design/scheduler_configuration.md b/versioned_docs/version-0.10.0/design/scheduler_configuration.md
new file mode 100644
index 0000000..54b616a
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/scheduler_configuration.md
@@ -0,0 +1,246 @@
+---
+id: scheduler_configuration
+title: Scheduler Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The Yunikorn core scheduler configuration has two separate areas that need to be configured. The scheduler service itself, things like web service ports etc, and the queue configuration. The split between the two types of configuration is proposed with two points in mind:
+* Separation of duty
+* Dynamic vs Static
+
+The scheduler configuration is mainly static. There is no need to change a web service port or a scheduling policy while the service is running. The queue configuration is far more dynamic and can change while the service is running.
+
+From a separation of duty we can allow an operator that manages the cluster to make changes to the scheduler queues. You would not want to allow that administrator to change the scheduler configuration itself.
+
+Separated from the core scheduler configuration we have one or more shim configurations. We currently cannot anticipate the deployment model of the scheduler and its shims. A shim, like the k8s-shim, might run in the same container or node but there is no guarantee it will. We also do not know the number of shims that will be used with one core scheduler. There is also still the possibility to have multiple instances of the same shim with one core scheduler.
+
+Shim configuration must be independent of the core scheduler configuration.
+## Scheduler Configuration
+Scheduler configuration covers all the configuration needed to start the scheduler and the dependent services. The configuration consists of a simple key value pair. All configuration to start the service must be part of this configuration.
+The scheduler configuration must exclude the queue related configuration.
+
+Scheduler configuration as currently identified
+* Bind host
+* Service port
+* Web bind host
+* Web service port
+* SSL config
+* Shims Configured
+* SchedulerACL
+
+Configuration to consider:
+* Assign multiple containers in one go: use case is bin packing, don’t spread an application over large number of nodes. Needs to become configurable.
+* Pre-emption related configuration:
+ * threshold: do not pre-empt from a queue if the cluster load is below a certain threshold.
+ * Interval: pause between pre-emption checks
+## Queue Configuration
+### Queue Definition
+On startup the scheduler will load the configuration for the queues from the provided configuration file after initialising the service. If there is no queue configuration provided the scheduler should start up with a simple default configuration which performs a well documented default behaviour.
+Based on the kubernetes definition this configuration could be a configMap <sup id="s1">[1](#f1)</sup> but not a CRD.
+
+The queue configuration is dynamic. Changing the queue configuration must not require a scheduler restart.
+Changes should be allowed by either calling the GO based API, the REST based API or by updating the configuration file. Changes made through the API must be persisted in the configuration file. Making changes through an API is not a high priority requirement and could be postponed to a later release.
+
+The queue configuration defines queues in a hierarchy: a tree. The base of the tree is the _root_ queue. The queue configuration must define a single _root_ queue. All queues that are defined in queue configuration are considered _managed_ queues.
+
+The root queue reflect the whole cluster. Resource settings on the root queue are not allowed. The resources available to the root queue are calculated based on the registered node resources in the cluster. If resources would be specified on the root limit the cluster would either be artificially limited to a specific size or expect resources to be available that are not there.
+
+Queues in the hierarchy in the tree are separated by the “.” dot character (ASCII 0x2E). This indirectly means that a queue name itself cannot contain a dot as it interferes with the hierarchy separator. Any queue name in the configuration that contains a dot will cause the configuration to be considered invalid. However we must allow placement rules to create a queue with a dot based input.
+
+Not all queues can be used to submit an application to. Applications can only be submitted to a queue which does not have a queue below it. These queues are defined as the _leaf_ queues of the tree. Queues that are not a _leaf_ and thus can contain other queues or child queues are considered _parent_ queues.
+
+Each queue must have exactly one _parent_ queue, besides the root queue. The root queue cannot have a _parent_ and will be automatically defined as a _parent_ queue type.
+A fully qualified queue name, case insensitive, must be unique in the hierarchy. A queue in the hierarchy can thus be only uniquely identified by its fully qualified path. This means that a queue with the same name is allowed at a different point in the hierarchy.
+Example:
+```
+root.companyA.development
+root.companyB.development
+root.production.companyA
+```
+In the example the queues _companyA_ and _companyB_ are _parent_ queues. Both _development_ queues are _leaf_ queues.
+The second instance of the _companyA_ queue is a _leaf_ queue which is not related to the first instance as it is defined at a different level in the hierarchy.
+
+The queue as defined in the configuration will be assigned a queue type. This can either be implicit based on how the queue is defined in the hierarchy or explicit by setting the optional _parent_ property as part of the queue definition. By default all queues will be assigned their type based on the configuration. There is only one case in which this should automatic process would need to be overridden and that is to mark a _leaf_ in the configuration as a _parent_. The use case is part [...]
+
+Access control lists provide a split between submission permission and administration permissions. Submission access to a queue allows an application to be submitted to the queue by the users or groups specified. The administration permissions allows submission to the queue plus the administrative actions. Administrative actions are currently limited to killing an application and moving an application to a different queue.
+
+Access control lists are checked recursively up to the root of the tree starting at the lowest point in the tree. In other words when the access control list of a queue does not allow access the parent queue is checked. The checks are repeated all the way up to the root of the queues.
+
+On each queue, except the root queue, the following properties can be set:
+* QueueType:
+ * Parent (boolean)
+* Resource settings:
+ * Guaranteed (resource)
+ * Maximum (resource)
+* Running Application limit:
+ * Maximum (integer)
+* Queue Permissions:
+ * SubmitACL (ACL)
+ * AdminACL (ACL)
+* Pre emption setting:
+ * PreEmptionAllowed (boolean)
+* Application sort algorithm:
+ * ApplicationSortPolicy (enumeration: fair, fifo)
+
+On the root queue only the following properties can be set:
+* Running Application limit:
+ * Maximum (integer)
+* Queue Permissions:
+ * SubmitACL (ACL)
+ * AdminACL (ACL)
+* Application sort algorithm:
+ * ApplicationSortPolicy (enumeration: fair, fifo)
+
+### User definition
+Applications are run by a user could run in one or more queues. The queues can have limits set on the resources that can be used. This does not limit the amount of resources that can be used by the user in the cluster.
+
+From an administrative perspective setting a limit of the resources that can be used by a specific user can be important. In this case a user is broadly defined as the identity that submits the application. This can be a service or a person, from a scheduling perspective there is no difference.
+User limits can prevent a take over of a queue or the cluster by a misbehaving user or application. From a multi tenancy perspective user limits also allows for sharing or subdivision of resources within the tenancy however that is defined.
+
+Adding user based limits will allow the cluster administrators to control the cluster wide resource usage of a user:
+* Running Application limit:
+ * Maximum (integer)
+* Resource setting:
+ * Maximum (resource)
+
+### Placement Rules definition
+Schedulers can place an application in a queue dynamically. This means that an application when submitted does not have to include a queue to run in.
+
+A placement rule will use the application details to place the application in the queue. The outcome of running a placement rule will be a fully qualified queue or a `fail`, which means execute the next rule in the list. Rules will be executed in the order that they are defined.
+
+During the evaluation of the rule the result could be a queue name that contains a dot. This is especially true for user and group names which are POSIX compliant. When a rule generates a partial queue name that contains a dot it must be replaced as it is the separator in the hierarchy. The replacement text will be `_dot_`
+
+The first rule that matches, i.e. returns a fully qualified queue name, will halt the execution of the rules. If the application is not placed at the end of the list of rules the application will be rejected. Rules can return queues that are not defined in the configuration only if the rule allows creation of queues.
+
+These queues created by the placement rules are considered _unmanaged_ queues as they are not managed by the administrator in the configuration. An administrator cannot influence the _unmanaged_ queue creation or deletion. The scheduler creates the queue when it is needed and removes the queue automatically when it is no longer used.
+
+Rules provide a fully qualified queue name as the result. To allow for deeper nesting of queues the parent of the queue can be set as part of the rule evaluation. The rule definition should allow a fixed configured fully qualified parent to be specified or it can call a second rule to generate the parent queue. By default a queue is generated as a child of the root queue.
+
+Example:
+Placing an application submitted by the user _user1_ whom is a member of the groups _user1_ and _companyA_ in a queue based on UserName:
+```
+Rule name: UserName
+ Parent: root.fixedparent
+Result: root.fixedparent.user1
+
+Rule name: UserName
+ Parent: SecondaryGroup
+ Filter:
+ Type: allow
+ Groups: company.*
+Result: root.companyA.user1
+
+Rule name: UserName
+Filter:
+ Users: user2,user3
+Result: denied placement
+```
+The default behaviour for placing an application in a queue, which would do the same as using the queue that is provided during submit, would be a rule that takes the provided queue with the create flag set to false.
+
+Access permissions will be enforced as part of the rule evaluation. For _managed_ queues this means that the ACL for the queue itself is checked. For an _unmanaged_ queue the parent queue ACL is the one that is checked. For the definition of the access control list and checks see the [Access Control Lists](#access-control-lists) chapter.
+
+Defining placement rules in the configuration requires the following information per rule:
+* Name:
+ * Name (string)
+* Parent
+ * Parent (string)
+* Create Flag:
+ * Create (boolean)
+* Filter:
+ * A regular expression or list of users/groups to apply the rule to.
+
+The filter can be used to allow the rule to be used (default behaviour) or deny the rule to be used. User or groups matching the filter will be either allowed or denied.
+The filter is defined as follow:
+* Type:
+ * Type (string) which can have no value (empty) or "allow" or "deny", case insensitive.
+* Users:
+ * A list of zero or more user names. If the list is exactly one long it will be interpreted as a regular expression.
+* Groups:
+ * A list of zero or more group names. If the list is exactly one long it will be interpreted as a regular expression.
+
+Proposed rules for placing applications would be:
+* Provided: returns the queue provided during the submission
+* UserName: returns the user name
+* PrimaryGroupName: returns the primary group of the user
+* SecondaryGroupName: returns the first secondary group of the user that matches
+* Fixed: returns the queue name configured in the rule
+* ApplicationType: returns the application type (if available)
+
+For _unmanaged_ queues in the current revision of the configuration you cannot provide any queue specific properties. However in the future we should consider propagating specific resource related settings from a _managed_ parent to the _unmanaged_ child, specifically:
+* Dynamic Resource settings:
+ * Guaranteed (resource)
+ * Maximum (resource)
+* Dynamic Running Application limit:
+ * Maximum (integer)
+
+### Configuration updates
+Updating the queue definition will allow updating the existing queue properties as well as adding and removing queues. A new queue definition will only become active if the configuration can be parsed. The change of the definition is an atomic change which applies all modification in one action.
+
+Updating the queue properties will not automatically trigger further action. This means that if the maximum number of resources of a queue or its parent is changed we leave the applications in the queue running as they are. The scheduler will adhere to the new property values which should see the convergence over time.
+
+A _managed_ queue will only be removed if it is removed from the configuration. Before we can remove a queue it must not be running applications. This means that when a _managed_ queue is removed from the configuration it must be empty or the system needs to allow the queue to drain. Forcing a _managed_ queue to be empty before we can remove it is not possible which means that _managed_ queues are removed in multiple steps:
+1. The queue is removed from the configuration
+1. The queue is marked as `draining`
+1. All managed queues that are `draining` and empty are removed
+
+Long running applications should be handled gracefully when removing a _managed_ queue. The scheduler should at least track and expose that a queue has been in a _draining_ state for an extended period of time. In the optimal case the application should be notified of the queue change to allow it to release resources. In all cases the queue administrators should be notified to allow them to take action. This action would currently be a manual move of the application to a different queue [...]
+
+_Unmanaged_ queues that are not defined in the queue definition are created by the scheduler automatically based on the placement rules. _Unmanaged_ queues have a lifespan independent of the configuration. Whenever an _unmanaged_ queue is empty it will get removed. The queue will automatically be created again when a new application is requesting it via triggering the placement rule.
+
+Removing an empty _managed_ or _unmanaged_ queue is handled by the same removal code which must run independent of the configuration updates and scheduling actions.
+
+Configurations can change over time. The impact of a fail over or restart must still be investigated.
+Base point to make: a changed configuration should not impact the currently running applications. Queues that no longer exist should be handled somehow.
+
+### Access Control Lists
+The scheduler ACL is independent of the queue ACLs. A scheduler administrator is not by default allowed to submit an application or administer the queues in the system.
+
+All ACL types should use the same definition pattern. We should allow at least POSIX user and group names which uses the portable filename character set <sup id="s2">[2](#f2)</sup>. However we should take into account that we could have domain specifiers based on the environment that the system runs in (@ sign as per HADOOP-12751).
+
+By default access control is enabled and access is denied. The only special case is for the core scheduler which automatically adds the system user, the scheduler process owner, to the scheduler ACL. The scheduler process owner is allowed to make sure that the process owner can use the API to call any administrative actions.
+
+Access control lists give access to the users and groups that have been specified in the list. They do not provide the possibility to explicitly remove or deny access to the users and groups specified in the list.
+
+The access control list is defined as:
+```
+ACL ::= “*” | userlist [ “ “ grouplist ]
+userlist ::= “” | user { “,” user }
+grouplist ::= “” | group { “,” group }
+```
+
+This definition specifies a wildcard of * which results in access for everyone. If the user list is empty and the group list is empty nobody will have access. This deny all ACL has two possible representations:
+* an empty access control list.
+* a single space.
+
+If there is no access control list is configured access is denied by default.
+## Shim Configuration
+The shim configuration is highly dependent on the shim implementation. The k8s shim differs from the YARN shim. Currently the k8s shim is configured via command line options but we should not depend on that.
+
+### K8s shim
+The full configuration of the K8s shim is still under development.
+
+### YARN shim
+The full configuration of the YARN shim is still under development.
+
+---
+<br/><b id="f1"></b>1: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#should-i-use-a-configmap-or-a-custom-resource. [↩](#s1)
+<br/><b id="f2"></b>2: The set of characters from which portable filenames are constructed. [↩](#s2)
+<br/>`A B C D E F G H I J K L M N O P Q R S T U V W X Y Z a b c d e f g h i j k l m n o p q r s t u v w x y z 0 1 2 3 4 5 6 7 8 9 . _ -`
diff --git a/versioned_docs/version-0.10.0/design/scheduler_core_design.md b/versioned_docs/version-0.10.0/design/scheduler_core_design.md
new file mode 100644
index 0000000..3b42a8f
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/scheduler_core_design.md
@@ -0,0 +1,401 @@
+---
+id: scheduler_core_design
+title: Scheduler Core Design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+:::caution
+The scheduler core design has changed. [YUNIKORN-317](https://issues.apache.org/jira/browse/YUNIKORN-317) was committed and has removed the scheduler cache.
+This document will not be maintained and is just for historical reference.
+See [scheduler cache removal design](cache_removal.md)
+:::
+
+Github repo: https://github.com/apache/incubator-yunikorn-core/
+
+Scheduler core encapsulates all scheduling algorithms, it collects resources from underneath resource management
+platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
+best spot for each request and then sends response allocations to the resource management platform.
+Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/incubator-yunikorn-scheduler-interface).
+
+## Components:
+
+```
+
+ +---------------+ +--------------+
+ |K8s Shim | |YARN Shim |
+ +---------------+ +--------------+
+
+ +--------------+ +------------+
+ Scheduler- | GRPC Protocol| |Go API |
+ Interface: +--------------+ +------------+
+
++---------------------------------------------------------------------------+
+ +--------------------+
+ |Scheduler API Server|
+ +-------------+ +---------+----------+
+ |AdminService | |
+ +-------------+ |Write Ops +----------------+
+ +-------------+ V ++Scheduler |
+ |Configurator | +-------------------+ Allocate || And |
+ +-------------+ |Cache Event Handler+<-----------------| |
+ +----------> +-------------------+ Preempt ++Preemptor |
+ Update Cfg Handled by policies +----------------+
+ + (Stateless)
+ +------v--------+
+ |Scheduler Cache|
+ +---------------+
+ +---------------------------------------------+
+ |--------+ +------+ +----------+ +----------+ |
+ ||Node | |Queue | |Allocation| |Requests | |
+ |--------+ +------+ +----------+ +----------+ |
+ +---------------------------------------------+
+```
+
+### Scheduler API Server (RMProxy)
+
+Responsible for communication between RM and Scheduler, which implements scheduler-interface GRPC protocol,
+or just APIs. (For intra-process communication w/o Serde).
+
+### Scheduler Cache
+
+Caches all data related to scheduler state, such as used resources of each queues, nodes, allocations.
+Relationship between allocations and nodes, etc. Should not include in-flight data for resource allocation.
+For example to-be-preempted allocation candidates. Fair share resource of queues, etc.
+
+### Scheduler Cache Event Handler
+
+Handles all events which needs to update scheduler internal state. So all the write operations will be carefully handled.
+
+### Admin Service
+
+Handles request from Admin, which can also load configurations from storage and update scheduler policies.
+
+### Scheduler and Preemptor
+
+Handles Scheduler's internal state. (Which is not belong to scheduelr cache), such as internal reservations, etc.
+Scheduler and preemptor will work together, make scheduling or preemption decisions. All allocate/preempt request
+will be handled by event handler.
+
+## Scheduler's responsibility
+
+- According to resource usages between queues, sort queues, applications, and figure out order of application allocation. (This will be used by preemption as well).
+- It is possible that we cannot satisfy some of the allocation request, we need to skip them and find next request.
+- It is possible that some allocation request cannot be satisfied because of resource fragmentation. We need to reserve room for such requests.
+- Different nodes may belong to different disjoint partitions, we can make independent scheduler runs
+- Be able to config and change ordering policies for apps, queues.
+- Application can choose their own way to manage sort of nodes.
+
+## Preemption
+
+- It is important to know "who wants the resource", so we can do preemption based on allocation orders.
+- When do preemption, it is also efficient to trigger allocation op. Think about how to do it.
+- Preemption needs to take care about queue resource balancing.
+
+## Communication between Shim and Core
+
+YuniKorn-Shim (like https://github.com/apache/incubator-yunikorn-k8shim) communicates with core by
+using scheduler-interface (https://github.com/apache/incubator-yunikorn-scheduler-interface).
+Scheduler interface has Go API or GRPC. Currently, yunikorn-k8shim is using Go API to communicate with yunikorn-core
+to avoid extra overhead introduced by GRPC.
+
+**Shim (like K8shim) first need to register with core:**
+
+```go
+func (m *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerRequest, callback api.ResourceManagerCallback) (*si.RegisterResourceManagerResponse, error)
+```
+
+Which indicate ResourceManager's name, a callback function for updateResponse. The design of core is be able to do scheduling for multiple clusters (such as multiple K8s cluster) just with one core instance.
+
+**Shim interacts with core by invoking RMProxy's Update API frequently, which updates new allocation request, allocation to kill, node updates, etc.**
+
+```go
+func (m *RMProxy) Update(request *si.UpdateRequest) error
+```
+
+Response of update (such as new allocated container) will be received by registered callback.
+
+## Configurations & Semantics
+
+Example of configuration:
+
+- Partition is name space.
+- Same queues can under different partitions, but enforced to have same hierarchy.
+
+ Good:
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b c
+ ```
+
+ Good (c in partition y acl=""):
+
+ ```
+ partition=x partition=y
+ a a
+ / \ /
+ b c b
+ ```
+
+ Bad (c in different hierarchy)
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b d
+ /
+ c
+ ```
+
+ Bad (Duplicated c)
+
+ ```
+ partition=x
+ a
+ / \
+ b c
+ /
+ c
+
+ ```
+
+- Different hierarchies can be added
+
+ ```scheduler-conf.yaml
+ partitions:
+ - name: default
+ queues:
+ root:
+ configs:
+ acls:
+ childrens:
+ - a
+ - b
+ - c
+ - ...
+ a:
+ configs:
+ acls:
+ capacity: (capacity is not allowed to set for root)
+ max-capacity: ...
+ mapping-policies:
+ ...
+ - name: partition_a:
+ queues:
+ root:...
+ ```
+
+## How scheduler do allocation
+
+Scheduler runs a separate goroutine to look at asks and available resources, and do resource allocation. Here's allocation logic in pseudo code:
+
+Entry point of scheduler allocation is `scheduler.go: func (s *Scheduler) schedule()`
+
+```
+# First of all, YuniKorn has partition concept, a logical resource pool can consists
+# of one of multiple physical dis-joint partitions. It is similar to YARN's node
+# partition concept.
+
+for partition : partitions:
+ # YuniKorn can reserve allocations for picky asks (such as large request, etc.)
+ # Before doing regular allocation, YuniKorn look at reservedAllocations first.
+ for reservedAllocation : partition.reservedAllocations:
+ reservedAllocation.tryAllocate(..)
+
+ # After tried all reserved allocation, YuniKorn will go to regular allocation
+ partition.tryAllocate(..)
+
+ # If there's any allocation created, scheduler will create an AllocationProposal
+ # and send to Cache to "commit" the AllocationProposal
+```
+
+**Allocation by hierchical of queues**
+
+Inside `partition.tryAllocate`
+
+It recursively traverse from root queue and down to lower level, for each level, logic is inside `pkg/scheduler/scheduling_queue.go func (sq *SchedulingQueue) tryAllocate`
+
+Remember YuniKorn natively supports hierarchical of queues. For ParentQueue (which has sub queues under the parent queue), it uses queue's own sorting policy to sort subqueues and try to allocate from most preferred queue to least-preferred queue.
+
+For LeafQueue (which has applications inside the queue), it uses queue's own sorting policy to sort applications belong to the queue and allocate based on the sorted order.
+
+(All sorting policies can be configured differently at each level.)
+
+**Allocation by application**
+
+When it goes to Application, see (`scheduler_application.go: func (sa *SchedulingApplication) tryAllocate`), It first sort the pending resource requests belong to the application (based on requests' priority). And based on the selected request, and configured node-sorting policy, it sorts nodes belong to the partition and try to allocate resources on the sorted nodes.
+
+When application trying to allocate resources on nodes, it will invokes PredicatePlugin to make sure Shim can confirm the node is good. (For example K8shim runs predicates check for allocation pre-check).
+
+**Allocation completed by scheduler**
+
+Once allocation is done, scheduler will create an AllocationProposal and send to Cache to do further check, we will cover details in the upcoming section.
+
+## Flow of events
+
+Like mentioned before, all communications between components like RMProxy/Cache/Schedulers are done by using async event handler.
+
+RMProxy/Cache/Scheduler include local event queues and event handlers. RMProxy and Scheduler have only one queue (For example: `pkg/scheduler/scheduler.go: handleSchedulerEvent`), and Cache has two queues (One for events from RMProxy, and one for events from Scheduler, which is designed for better performance).
+
+We will talk about how events flowed between components:
+
+**Events for ResourceManager registration and updates:**
+
+```
+Update from ResourceManager -> RMProxy -> RMUpdateRequestEvent Send to Cache
+New ResourceManager registration -> RMProxy -> RegisterRMEvent Send to Cache
+```
+
+**Cache Handles RM Updates**
+
+There're many fields inside RM Update event (`RMUpdateRequestEvent`), among them, we have following categories:
+
+```
+1) Update for Application-related updates
+2) Update for New allocation ask and release.
+3) Node (Such as kubelet) update (New node, remove node, node resource change, etc.)
+```
+
+More details can be found at:
+
+```
+func (m *ClusterInfo) processRMUpdateEvent(event *cacheevent.RMUpdateRequestEvent)
+
+inside cluster_info.go
+```
+
+**Cache send RM updates to Scheduler**
+
+For most cases, Cache propagate updates from RM to scheduler directly (including Application, Node, Asks, etc.). And it is possible that some updates from RM is not valid (such as adding an application to a non-existed queue), for such cases, cache can send an event back to RMProxy and notify the ResourceManager. (See `RMApplicationUpdateEvent.RejectedApplications`)
+
+**Cache handles scheduler config**
+
+Cache also handles scheduler's config changes, see
+
+```go
+func (m *ClusterInfo) processRMConfigUpdateEvent(event *commonevents.ConfigUpdateRMEvent)
+```
+
+Similar to other RM updates, it propages news to scheduelr.
+
+**Scheduler do allocation**
+
+Once an AllocationProposal created by scheduler, scheduler sends `AllocationProposalBundleEvent` to Cache to commit.
+
+Cache look at AllocationProposal under lock, and commit these proposals. The reason to do proposal/commit is Scheduler can run in multi-threads which could cause conflict for resource allocation. This approach is inspired by Borg/Omega/YARN Global Scheduling.
+
+Cache checks more states such as queue resources, node resources (we cannot allocate more resource than nodes' available), etc. Once check is done, Cache updates internal data strcture and send confirmation to Scheduler to update the same, and scheduler sends allocated Allocation to RMProxy so Shim can do further options. For example, K8shim will `bind` an allocation (POD) to kubelet.
+
+```
+Job Add:
+--------
+RM -> Cache -> Scheduler (Implemented)
+
+Job Remove:
+-----------
+RM -> Scheduler -> Cache (Implemented)
+Released allocations: (Same as normal release) (Implemented)
+Note: Make sure remove from scheduler first to avoid new allocated created.
+
+Scheduling Request Add:
+-----------------------
+RM -> Cache -> Scheduler (Implemented)
+Note: Will check if requested job exists, queue exists, etc.
+When any request invalid:
+ Cache -> RM (Implemented)
+ Scheduler -> RM (Implemented)
+
+Scheduling Request remove:
+-------------------------
+RM -> Scheduler -> Cache (Implemented)
+Note: Make sure removed from scheduler first to avoid new container allocated
+
+Allocation remove (Preemption)
+-----------------
+Scheduler -> Cache -> RM (TODO)
+ (confirmation)
+
+Allocation remove (RM voluntarilly ask)
+---------------------------------------
+RM -> Scheduler -> Cache -> RM. (Implemented)
+ (confirmation)
+
+Node Add:
+---------
+RM -> Cache -> Scheduler (Implemented)
+Note: Inside Cache, update allocated resources.
+Error handling: Reject Node to RM (Implemented)
+
+Node Remove:
+------------
+Implemented in cache side
+RM -> Scheduler -> Cache (TODO)
+
+Allocation Proposal:
+--------------------
+Scheduler -> Cache -> RM
+When rejected/accepted:
+ Cache -> Scheduler
+
+Initial: (TODO)
+--------
+1. Admin configured partitions
+2. Cache initializes
+3. Scheduler copies configurations
+
+Relations between Entities
+-------------------------
+1. RM includes one or multiple:
+ - Partitions
+ - Jobs
+ - Nodes
+ - Queues
+
+2. One queue:
+ - Under one partition
+ - Under one RM.
+
+3. One job:
+ - Under one queue (Job with same name can under different partitions)
+ - Under one partition
+
+RM registration: (TODO)
+----------------
+1. RM send registration
+2. If RM already registered, remove old one, including everything belong to RM.
+
+RM termination (TODO)
+--------------
+Just remove the old one.
+
+Update of queues (TODO)
+------------------------
+Admin Service -> Cache
+
+About partition (TODO)
+-----------------------
+Internal partition need to be normalized, for example, RM specify node with partition = xyz.
+Scheduler internally need to normalize it to <rm-id>_xyz
+This need to be done by RMProxy
+
+```
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/design/scheduler_object_states.md b/versioned_docs/version-0.10.0/design/scheduler_object_states.md
new file mode 100644
index 0000000..e71cd4c
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/scheduler_object_states.md
@@ -0,0 +1,127 @@
+---
+id: scheduler_object_states
+title: Scheduler Object States
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+The YuniKorn project uses state machines to track the states of different objects.
+This ranges from applications in the core to nodes in the k8shim.
+The state machines are independent and not shared between the resource managers and core.
+A resource manager shim, and the core can thus have an independent idea of the state of a similar object.
+
+## Core Scheduler
+State change are triggered by events that get processed.
+One event can cause a change for multiple states or no change at all.
+
+### Application State
+Applications have a complex state model.
+An application when created starts ain the new state.
+
+An application can have the following states:
+* New: A new application that is being submitted or created, from here the application transitions into the accepted state when it is ready for scheduling.
+The first ask to be added will trigger the transition.
+* Accepted: The application is ready and part of the scheduling cycle.
+On allocation of the first ask the application moves into a starting state.
+This state is part of the normal scheduling cycle.
+* Starting: The application has exactly one allocation confirmed this corresponds to one running container/pod.
+The application transitions to running if and when more allocations are added to the application.
+This state times out automatically to prevent applications that consist of just one allocation from getting stuck in this state.
+The current time out is set to 5 minutes, and cannot be changed.
+If after the timeout expires the application will auto transition to running.
+The state change on time out is independent of the number of allocations added.
+This state is part of the normal scheduling cycle.
+* Running: The state in which the application will spend most of its time.
+Containers/pods can be added to and removed from the application.
+This state is part of the normal scheduling cycle.
+* Completing: An application that has no pending requests or running containers/pod will be completing.
+This state shows that the application has not been marked completed yet but currently is not actively being scheduled.
+* Completed: An application is considered completed when it has been in the completing state for a defined time period.
+From this state the application can only move to the Expired state, and it cannot move back into any of scheduling states (Running or Completing)
+The current timeout is set to 30 seconds.
+* Expired: The completed application is tracked for a period of time, after that is expired and deleted from the scheduler.
+This is a final state and after this state the application cannot be tracked anymore.
+* Failing: An application marked for failing, what still has some allocations or asks what needs to be cleaned up before entering into the Failed state.
+ The application can be Failing when the partition it belongs to is removed or during gang scheduling, if the placeholder processing times out, and the application has no real allocations yet.
+* Failed: An application is considered failed when it was marked for failure and all the pending requests and allocations were already removed.
+This is a final state. The application cannot change state after entering.
+* Rejected: The application was rejected when it was added to the scheduler.
+This only happens when a resource manager tries to add a new application, when it gets created in a New state, and the scheduler rejects the creation.
+Applications can be rejected due ACLs denying access to a queue the application has specified, or a placement via placement rules has failed.
+This is a final state. The application cannot change state after entering.
+
+The events that can trigger a state change:
+* Reject: rejecting the application by the scheduler (source: core scheduler)
+* Run: progress an application to the next active state (source: core scheduler)
+* Complete: mark an application as idle or complete (source: core scheduler)
+* Fail: fail an application (source: resource manager or core scheduler)
+* Expire: progress the application to the expired state and remove it from the scheduler (source: core scheduler)
+
+Here is a diagram that shows the states with the event that causes the state to change:
+
+
+### Object State
+<!-- fix the draining to stopped transition -->
+The object state is used by the following objects:
+* queues
+* partitions
+
+The object states are as follows:
+* Active: The object is active and used during the scheduling cycle.
+This is the starting and normal state of an object.
+An active object transitions to draining when it is removed.
+* Stopped: The object is stopped and no longer actively scheduled.
+The object if empty is ready to be removed from the scheduler.
+The object can transition back into active state if it gets re-started.
+* Draining: Before an object can be removed it needs to be cleaned up.
+The cleanup starts with placing the object in the draining state.
+In this state it does not accept additions or changes but is still actively being scheduled.
+This allows for a graceful shutdown, cleanup and removal of the object.
+This is the final state.
+
+The events that can trigger a state change:
+* Start: make the object active (source: core scheduler)
+* Stop: make the object inactive (source: core scheduler)
+* Remove: mark an object for removal (source: core scheduler)
+
+Here is a diagram that shows the states with the event that causes the state to change:
+
+
+### Node
+<!-- should start using object state -->
+Node objects in the core are not using a state machine but do have a state.
+A node can have one of two states: `schedulable` or `not schedulable`.
+There is no complex state model or complex transition logic.
+The scheduler can either use the node or not.
+
+The node status changes based on the status provided by the resource manager (shim) that owns the node.
+
+## K8Shim Resource Manager
+
+### Application
+To be added
+
+### Task
+To be added
+
+### Node
+To be added
+
+### Scheduler
+To be added
diff --git a/versioned_docs/version-0.10.0/design/state_aware_scheduling.md b/versioned_docs/version-0.10.0/design/state_aware_scheduling.md
new file mode 100644
index 0000000..f92f93c
--- /dev/null
+++ b/versioned_docs/version-0.10.0/design/state_aware_scheduling.md
@@ -0,0 +1,112 @@
+---
+id: state_aware_scheduling
+title: Batch Workloads Ordering with StateAware Policy
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## The problem
+A common pattern while processing data is that the application can be divided into multiple stages.
+Another way to look at this is the fact that processing needs to be kicked off and that the first step is to start a driver or manager for the application.
+Later stages might depend on the previous stages.
+When running applications in a size limited environment this could lead to a resource exhaustion when submitting multiple applications at the same time.
+These first stages might consume all available resources leaving no room for the next stage(s) to start.
+Often this issue is caused by having a high number of applications start simultaneous and trying to get resources in parallel.
+### Example issue
+When submitting numerous Spark applications in a short amount of time the drivers will all be started shortly after each other.
+The drivers consume all available resources in a queue or in the whole cluster.
+After starting the drivers they will request resources for the executors.
+Since the queue or cluster has no resources left the executors will not be started.
+The driver cannot progress.
+The only way that progress would be made is if and when one of the drivers finishes or fails and frees up resources for executors to be started.
+
+## Design
+### Design goals
+1. Prevent resource exhaustion by first stage allocations
+1. Improve chance for jobs to get minimal required resources over others
+
+### None goals
+1. This is NOT an implementation of Gang scheduling.
+1. No change to the currently supported FAIR or FIFO scheduling algorithms
+1. Fix resource quota usage outside of the core scheduler for submitted but waiting applications
+
+### Possible solutions
+Other batch schedulers like the YARN schedulers use a limit on the number of simultaneous running applications.
+They use either resource constraints on the driver or management stage or set a hard limit of the number of applications that can run in a queue.
+The draw back of that solution is that it does not work well in a cluster that can scale up or down automatically in a cloud environment.
+To work around that percentage based limits could be set on the consumed resources for the driver or management stage.
+This does not alleviate the fact that driver or management stages can be of any size, large and or small, which complicates the percentage scenario further as it does not give a predictable behaviour.
+
+A different solution would be to assume a specific behaviour of the applications.
+Using that assumption a limit on the applications could be set based on the state it is in.
+The spark driver and executor behaviour is the most usual use case.
+This would provide a way to limit scheduling to existing applications and only drip feed new applications into the list of applications to schedule when there are resources available.
+
+### Algorithm
+The algorithm described here is based on the drip feed of new applications into the applications to schedule based on the states of all applications.
+Scheduling is based on the applications in a queue.
+The algorithm will be applied at a queue level.
+This is not a cluster wide setup.
+
+What we want to achieve is the following behaviour: only schedule one (1) application that is in its early stage(s) (called a starting state) at the same time.
+Only consider another new application if and when the previous application has transitioned out of the starting state.
+Applications will always be allocated resources on a first in first out basis based on submission time.
+That means that an application that is newly added and in its starting phase will only get resources if applications in the later stages do not need any resources.
+
+This algorithm will be implemented as an application sorting policy on a queue.
+This allows specific queues to limit parallel application startup while other queues with different work loads can schedule without or with different limitations.
+
+### Fallback mechanism
+A fallback mechanism has to be built into the algorithm.
+Not all applications will request more than one allocation.
+The other case that has to be accounted for could be a misbehaving or a slow application.
+Having an application stuck in the starting state could cause a scheduler livelock and starvation of other applications.
+
+The fall back mechanism proposed is as simple as a time limit on the starting state.
+This means that any application auto progresses out of the starting state.
+The time limit will be set to five (5) minutes hard coded as a starting point and will not be made configurable.
+
+The other fallback mechanism considered was making the number of allocations for the starting state configurable.
+This option provided a number of issues which made it difficult to implement.
+One of the main stumbling blocks is the fact that it requires the application submitter to specify the value.
+It also does not guarantee that the application will leave the starting state either and does not fix the livelock issue.
+If an application was submitted with five required allocation but due to issues during the run never asked for more than four then the livelock would still occur.
+
+Setting a default of zero (0) would also not work as it would bypass the starting state.
+It would make the sorting policy an opt-in instead of an opt-out.
+Setting a default of one (1) does not give us much enhancement to what we currently propose.
+It makes the sorting policy an opt-out but does not give the cluster administrator any control over the scheduling behaviour.
+Weighing those against each other the proposal is to not make this configurable.
+
+### Example run
+Using Spark applications as an example: a new application can only be scheduled if the previous application has at least one (1) executor allocated.
+
+
+
+Assume we have the following Spark apps: App1 & App2 as in the diagram above. The applications were submitted in that order: App1 first, then App2. They were both submitted to the same queue.
+
+1. Both applications are in the queue waiting for the first allocation: accepted by the scheduler. App1 has requested driver D1 and App2 has requested driver D2.
+1. The scheduler sorts the application and allows 1 accepted application to be scheduled (no starting applications yet): App1 as the oldest applications with an outstanding request is scheduled.
+App1 is allocated its driver (D1) and progresses to starting.
+App2 request for a driver is ignored as the scheduler is starting App1 (only 1 application in starting or accepted state is scheduled).
+1. App1 requests executors E11 and E12. The scheduler assigns E11 and E12. At this point the application state changes to running when it has at least 1 executor allocated.
+1. App2 has been waiting to get the driver allocated. Since there are no applications in a starting state the scheduler looks at App2 which is in an accepted state. App2 moves from the accepted state to starting when the driver is allocated.
+1. App2 requests its executor E21. The application state changes to running when E21 is allocated.
+
+This process would repeat itself for any new application submitted.
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/developer_guide/build.md b/versioned_docs/version-0.10.0/developer_guide/build.md
new file mode 100644
index 0000000..b93c02d
--- /dev/null
+++ b/versioned_docs/version-0.10.0/developer_guide/build.md
@@ -0,0 +1,178 @@
+---
+id: build
+title: Build and Run
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn always works with a container orchestrator system. Currently, a Kubernetes shim [yunikorn-k8shim](https://github.com/apache/incubator-yunikorn-k8shim)
+is provided in our repositories, you can leverage it to develop YuniKorn scheduling features and integrate with Kubernetes.
+This document describes resources how to setup dev environment and how to do the development.
+
+## Development Environment setup
+
+Read the [environment setup guide](developer_guide/env_setup.md) first to setup Docker and Kubernetes development environment.
+
+## Build YuniKorn for Kubernetes
+
+Prerequisite:
+- Go 1.12+
+
+You can build the scheduler for Kubernetes from [yunikorn-k8shim](https://github.com/apache/incubator-yunikorn-k8shim) project.
+The build procedure will build all components into a single executable that can be deployed and running on Kubernetes.
+
+Start the integrated build process by pulling the `yunikorn-k8shim` repository:
+```bash
+mkdir $HOME/yunikorn/
+cd $HOME/yunikorn/
+git clone https://github.com/apache/incubator-yunikorn-k8shim.git
+```
+At this point you have an environment that will allow you to build an integrated image for the YuniKorn scheduler.
+
+### A note on Go modules and git version
+Go use git to fetch module information.
+Certain modules cannot be retrieved if the git version installed on the machine used to build is old.
+A message similar to the one below will be logged when trying to build for the first time.
+```text
+go: finding modernc.org/mathutil@v1.0.0
+go: modernc.org/golex@v1.0.0: git fetch -f origin refs/heads/*:refs/heads/* refs/tags/*:refs/tags/* in <location>: exit status 128:
+ error: RPC failed; result=22, HTTP code = 404
+ fatal: The remote end hung up unexpectedly
+```
+Update git to a recent version to fix this issue.
+Git releases later than 1.22 are known to work.
+
+### Build Docker image
+
+Building a docker image can be triggered by following command.
+
+```
+make image
+```
+
+The image with the build in configuration can be deployed directly on kubernetes.
+Some sample deployments that can be used are found under [deployments](https://github.com/apache/incubator-yunikorn-k8shim/tree/master/deployments/scheduler) directory.
+For the deployment that uses a config map you need to set up the ConfigMap in kubernetes.
+How to deploy the scheduler with a ConfigMap is explained in the [scheduler configuration deployment](developer_guide/deployment.md) document.
+
+The image build command will first build the integrated executable and then create the docker image.
+Currently, there are some published docker images under [this docker hub repo](https://hub.docker.com/r/apache/yunikorn), you are free to fetch and use.
+The default image tags are not suitable for deployments to an accessible repository as it uses a hardcoded user and would push to Docker Hub with proper credentials.
+You *must* update the `TAG` variable in the `Makefile` to push to an accessible repository.
+When you update the image tag be aware that the deployment examples given will also need to be updated to reflect the same change.
+
+### Inspect the docker image
+
+The docker image built from previous step has embedded some important build info in image's metadata. You can retrieve
+these info with docker `inspect` command.
+
+```
+docker inspect apache/yunikorn:scheduler-latest
+```
+
+This info includes git revisions (last commit SHA) for each component, to help you understand which version of the source code
+was shipped by this image. They are listed as docker image `labels`, such as
+
+```
+"Labels": {
+ "BuildTimeStamp": "2019-07-16T23:08:06+0800",
+ "Version": "0.1",
+ "yunikorn-core-revision": "dca66c7e5a9e",
+ "yunikorn-k8shim-revision": "bed60f720b28",
+ "yunikorn-scheduler-interface-revision": "3df392eded1f"
+}
+```
+
+### Dependencies
+
+The dependencies in the projects are managed using [go modules](https://blog.golang.org/using-go-modules).
+Go Modules require at least Go version 1.11 to be installed on the development system.
+
+If you want to modify one of the projects locally and build with your local dependencies you will need to change the module file.
+Changing dependencies uses mod `replace` directives as explained in the [Update dependencies](#Updating dependencies).
+
+The YuniKorn project has four repositories three of those repositories have a dependency at the go level.
+These dependencies are part of the go modules and point to the github repositories.
+During the development cycle it can be required to break the dependency on the committed version from github.
+This requires making changes in the module file to allow loading a local copy or a forked copy from a different repository.
+
+#### Affected repositories
+The following dependencies exist between the repositories:
+
+| repository| depends on |
+| --- | --- |
+| yunikorn-core | yunikorn-scheduler-interface |
+| yunikorn-k8shim | yunikorn-scheduler-interface, yunikorn-core |
+| yunikorn-scheduler-interface | none |
+| yunikorn-web | yunikorn-core |
+
+The `yunikorn-web` repository has no direct go dependency on the other repositories. However any change to the `yunikorn-core` webservices can affect the web interface.
+
+#### Making local changes
+
+To make sure that the local changes will not break other parts of the build you should run:
+- A full build `make` (build target depends on the repository)
+- A full unit test run `make test`
+
+Any test failures should be fixed before proceeding.
+
+#### Updating dependencies
+
+The simplest way is to use the `replace` directive in the module file. The `replace` directive allows you to override the import path with a new (local) path.
+There is no need to change any of the imports in the source code. The change must be made in the `go.mod` file of the repository that has the dependency.
+
+Using `replace` to use of a forked dependency, such as:
+```
+replace github.com/apache/incubator-yunikorn-core => example.com/some/forked-yunikorn
+```
+
+There is no requirement to fork and create a new repository. If you do not have a repository you can use a local checked out copy too.
+Using `replace` to use of a local directory as a dependency:
+```
+replace github.com/apache/incubator-yunikorn-core => /User/example/local/checked-out-yunikorn
+```
+and for the same dependency using a relative path:
+```
+replace github.com/apache/incubator-yunikorn-core => ../checked-out-yunikorn
+```
+Note: if the `replace` directive is using a local filesystem path, then the target must have the `go.mod` file at that location.
+
+Further details on the modules' wiki: [When should I use the 'replace' directive?](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
+
+## Build the web UI
+
+Example deployments reference the [YuniKorn web UI](https://github.com/apache/incubator-yunikorn-web).
+The YuniKorn web UI has its own specific requirements for the build. The project has specific requirements for the build follow the steps in the README to prepare a development environment and build how to build the projects.
+The scheduler is fully functional without the web UI.
+
+## Locally run the integrated scheduler
+
+When you have a local development environment setup you can run the scheduler in your local kubernetes environment.
+This has been tested in a Docker desktop with 'Docker for desktop' and Minikube. See the [environment setup guide](developer_guide/env_setup.md) for further details.
+
+```
+make run
+```
+It will connect with the kubernetes cluster using the users configured configuration located in `$HOME/.kube/config`.
+
+You can also use the same approach to run the scheduler locally but connecting to a remote kubernetes cluster,
+as long as the `$HOME/.kube/config` file is pointing to that remote cluster.
+
+
diff --git a/versioned_docs/version-0.10.0/developer_guide/deployment.md b/versioned_docs/version-0.10.0/developer_guide/deployment.md
new file mode 100644
index 0000000..57420cf
--- /dev/null
+++ b/versioned_docs/version-0.10.0/developer_guide/deployment.md
@@ -0,0 +1,132 @@
+---
+id: deployment
+title: Deploy to Kubernetes
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The easiest way to deploy YuniKorn is to leverage our [helm charts](https://hub.helm.sh/charts/yunikorn/yunikorn),
+you can find the guide [here](get_started/get_started.md). This document describes the manual process to deploy YuniKorn
+scheduler and it is majorly for developers.
+
+## Build docker image
+
+Under project root of the `yunikorn-k8shim`, run the command to build an image using the map for the configuration:
+```
+make image
+```
+
+This command will build an image. The image will be tagged with a default version and image tag.
+
+**Note** the default build uses a hardcoded user and tag. You *must* update the `IMAGE_TAG` variable in the `Makefile` to push to an appropriate repository.
+
+
+## Setup RBAC
+
+The first step is to create the RBAC role for the scheduler, see [yunikorn-rbac.yaml](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/deployments/scheduler/yunikorn-rbac.yaml)
+```
+kubectl create -f scheduler/yunikorn-rbac.yaml
+```
+The role is a requirement on the current versions of kubernetes.
+
+## Create the ConfigMap
+
+This must be done before deploying the scheduler. It requires a correctly setup kubernetes environment.
+This kubernetes environment can be either local or remote.
+
+- download configuration file if not available on the node to add to kubernetes:
+```
+curl -o queues.yaml https://raw.githubusercontent.com/apache/incubator-yunikorn-k8shim/master/conf/queues.yaml
+```
+- create ConfigMap in kubernetes:
+```
+kubectl create configmap yunikorn-configs --from-file=queues.yaml
+```
+- check if the ConfigMap was created correctly:
+```
+kubectl describe configmaps yunikorn-configs
+```
+
+**Note** if name of the ConfigMap is changed the volume in the scheduler yaml file must be updated to reference the new name otherwise the changes to the configuration will not be picked up.
+
+## Attach ConfigMap to the Scheduler Pod
+
+The ConfigMap is attached to the scheduler as a special volume. First step is to specify where to mount it in the pod:
+```yaml
+ volumeMounts:
+ - name: config-volume
+ mountPath: /etc/yunikorn/
+```
+Second step is to link the mount point back to the configuration map created in kubernetes:
+```yaml
+ volumes:
+ - name: config-volume
+ configMap:
+ name: yunikorn-configs
+```
+
+Both steps are part of the scheduler yaml file, an example can be seen at [scheduler.yaml](https://github.com/apache/incubator-yunikorn-k8shim/blob/master/deployments/scheduler/scheduler.yaml)
+for reference.
+
+## Deploy the Scheduler
+
+The scheduler can be deployed with following command.
+```
+kubectl create -f deployments/scheduler/scheduler.yaml
+```
+
+The deployment will run 2 containers from your pre-built docker images in 1 pod,
+
+* yunikorn-scheduler-core (yunikorn scheduler core and shim for K8s)
+* yunikorn-scheduler-web (web UI)
+
+The pod is deployed as a customized scheduler, it will take the responsibility to schedule pods which explicitly specifies `schedulerName: yunikorn` in pod's spec. In addition to the `schedulerName`, you will also have to add a label `applicationId` to the pod.
+```yaml
+ metadata:
+ name: pod_example
+ labels:
+ applicationId: appID
+ spec:
+ schedulerName: yunikorn
+```
+
+Note: Admission controller abstracts the addition of `schedulerName` and `applicationId` from the user and hence, routes all traffic to YuniKorn. If you use helm chart to deploy, it will install admission controller along with the scheduler.
+
+## Access to the web UI
+
+When the scheduler is deployed, the web UI is also deployed in a container.
+Port forwarding for the web interface on the standard ports can be turned on via:
+
+```
+POD=`kubectl get pod -l app=yunikorn -o jsonpath="{.items[0].metadata.name}"` && \
+kubectl port-forward ${POD} 9889 9080
+```
+
+`9889` is the default port for Web UI, `9080` is the default port of scheduler's Restful service where web UI retrieves info from.
+Once this is done, web UI will be available at: http://localhost:9889.
+
+## Configuration Hot Refresh
+
+YuniKorn supports to load configuration changes automatically from attached configmap. Simply update the content in the configmap,
+that can be done either via Kubernetes dashboard UI or commandline. _Note_, changes made to the configmap might have some
+delay to be picked up by the scheduler.
+
+
+
diff --git a/versioned_docs/version-0.10.0/developer_guide/env_setup.md b/versioned_docs/version-0.10.0/developer_guide/env_setup.md
new file mode 100644
index 0000000..c45d77e
--- /dev/null
+++ b/versioned_docs/version-0.10.0/developer_guide/env_setup.md
@@ -0,0 +1,156 @@
+---
+id: env_setup
+title: Dev Environment Setup
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+There are several ways to setup a local development environment for Kubernetes, the two most common ones are `Minikube` ([docs](https://kubernetes.io/docs/setup/minikube/)) and `docker-desktop`.
+`Minikube` provisions a local Kubernetes cluster on several Virtual Machines (via VirtualBox or something similar). `docker-desktop` on the other hand, sets up Kubernetes cluster in docker containers.
+
+## Local Kubernetes cluster using Docker Desktop
+
+In this tutorial, we will base all the installs on Docker Desktop.
+Even in this case we can use a lightweight [minikube](#local-kubernetes-cluster-with-minikube) setup which gives the same functionality with less impact.
+
+### Installation
+
+Download and install [Docker-Desktop](https://www.docker.com/products/docker-desktop) on your laptop. Latest version has an embedded version of Kubernetes so no additional install is needed.
+Just simply follow the instruction [here](https://docs.docker.com/docker-for-mac/#kubernetes) to get Kubernetes up and running within docker-desktop.
+
+Once Kubernetes is started in docker desktop, you should see something similar below:
+
+
+
+This means that:
+1. Kubernetes is running.
+1. the command line tool `kubctl` is installed in the `/usr/local/bin` directory.
+1. the Kubernetes context is set to `docker-desktop`.
+
+### Deploy and access dashboard
+
+After setting up the local Kubernetes you need to deploy the dashboard using the following steps:
+1. follow the instructions in [Kubernetes dashboard doc](https://github.com/kubernetes/dashboard) to deploy the dashboard.
+1. start the Kubernetes proxy in the background from a terminal to get access on the dashboard on the local host:
+ ```shell script
+ kubectl proxy &
+ ```
+1. access the dashboard at the following URL: [clickable link](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login)
+
+### Access local Kubernetes cluster
+
+The dashboard as deployed in the previous step requires a token or config to sign in. Here we use the token to sign in. The token is generated automatically and can be retrieved from the system.
+
+1. retrieve the name of the dashboard token:
+ ```shell script
+ kubectl -n kube-system get secret | grep kubernetes-dashboard-token
+ ```
+2. retrieve the content of the token, note that the token name ends with a random 5 character code and needs to be replaced with the result of step 1. As an example:
+ ```shell script
+ kubectl -n kube-system describe secret kubernetes-dashboard-token-tf6n8
+ ```
+3. copy the token value which is part of the `Data` section with the tag `token`.
+4. select the **Token** option in the dashboard web UI:<br/>
+ 
+5. paste the token value into the input box and sign in:<br/>
+ 
+
+## Local Kubernetes cluster with Minikube
+Minikube can be added to an existing Docker Desktop install. Minikube can either use the pre-installed hypervisor or use a hypervisor of choice. These instructions use [HyperKit](https://github.com/moby/hyperkit) which is embedded in Docker Desktop.
+
+If you want to use a different hypervisor then HyperKit make sure that you follow the generic minikube install instructions. Do not forget to install the correct driver for the chosen hypervisor if required.
+The basic instructions are provided in the [minikube install](https://kubernetes.io/docs/tasks/tools/install-minikube/) instructions.
+
+Check hypervisor Docker Desktop should have already installed HyperKit. In a terminal run: `hyperkit` to confirm. Any response other than `hyperkit: command not found` confirms that HyperKit is installed and on the path. If it is not found you can choose a different hypervisor or fix the Docker Desktop install.
+
+### Installing Minikube
+1. install minikube, you can either use brew or directly via these steps:
+ ```shell script
+ curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-amd64
+ chmod +x minikube
+ sudo mv minikube /usr/local/bin
+ ```
+1. install HyperKit driver (required), you can either use brew or directly via these steps:
+ ```shell script
+ curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-hyperkit
+ sudo install -o root -g wheel -m 4755 docker-machine-driver-hyperkit /usr/local/bin/
+ ```
+1. update the minikube config to default to the HyperKit install `minikube config set vm-driver hyperkit`
+1. change docker desktop to use minikube for Kubernetes:<br/>
+ 
+
+### Deploy and access the cluster
+After the installation is done you can start a new cluster.
+1. start the minikube cluster: `minikube start --kubernetes-version v1.14.2`
+1. start the minikube dashboard: `minikube dashboard &`
+
+### Build impact
+When you create images make sure that the build is run after pointing it to the right environment.
+Without setting the enviromnent minikube might not find the docker images when deploying the scheduler.
+1. make sure minikube is started
+1. in the terminal where you wll run the build execute: `eval $(minikube docker-env)`
+1. run the image build from the yunikorn-k8shim repository root: `make image`
+1. deploy the scheduler as per the normal instructions.
+
+## Debug code locally
+
+Note, this instruction requires you have GoLand IDE for development.
+
+In GoLand, go to yunikorn-k8shim project. Then click "Run" -> "Debug..." -> "Edit Configuration..." to get the pop-up configuration window.
+Note, you need to click "+" to create a new profile if the `Go Build` option is not available at the first time.
+
+
+
+The highlighted fields are the configurations you need to add. These include:
+
+- Run Kind: package
+- Package path: point to the path of `pkg/shim` package
+- Working directory: point to the path of the `conf` directory, this is where the program loads configuration file from
+- Program arguments: specify the arguments to run the program, such as `-kubeConfig=/path/to/.kube/config -interval=1s -clusterId=mycluster -clusterVersion=0.1 -name=yunikorn -policyGroup=queues -logEncoding=console -logLevel=-1`.
+Note, you need to replace `/path/to/.kube/config` with the local path to the kubeconfig file. And if you want to change or add more options, you can run `_output/bin/k8s-yunikorn-scheduler -h` to find out.
+
+Once the changes are done, click "Apply", then "Debug". You will need to set proper breakpoints in order to debug the program.
+
+## Access remote Kubernetes cluster
+
+This setup assumes you have already installed a remote Kubernetes cluster.
+For a generic view on how to access a multiple cluster and integrate it follow the [accessing multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) documentation from Kubernetes.
+
+Or follow these simplified steps:
+1. get the Kubernetes `config` file from remote cluster, copy it to the local machine and give it a unique name i.e. `config-remote`
+1. save the `KUBECONFIG` environment variable (if set)
+ ```shell script
+ export KUBECONFIG_SAVED=$KUBECONFIG
+ ```
+1. add the new file to the environment variable
+ ```shell script
+ export KUBECONFIG=$KUBECONFIG:config-remote
+ ```
+1. run the command `kubectl config view` to check that both configs can be accessed
+1. switch context using `kubectl config use-context my-remote-cluster`
+1. confirm that the current context is now switched to the remote cluster config:
+ ```text
+ kubectl config get-contexts
+ CURRENT NAME CLUSTER AUTHINFO NAMESPACE
+ docker-for-desktop docker-for-desktop-cluster docker-for-desktop
+ * my-remote-cluster kubernetes kubernetes-admin
+ ```
+
+More docs can be found [here](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)
diff --git a/versioned_docs/version-0.10.0/developer_guide/openshift_development.md b/versioned_docs/version-0.10.0/developer_guide/openshift_development.md
new file mode 100644
index 0000000..8d21171
--- /dev/null
+++ b/versioned_docs/version-0.10.0/developer_guide/openshift_development.md
@@ -0,0 +1,182 @@
+---
+id: openshift_development
+title: Development in CodeReady Containers
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn is tested against OpenShift and developers can set up their local environment to test patches against OpenShift.
+Our recommended local environment uses CodeReady containers.
+
+## Set up a running CRC cluster
+
+1. Download CodeReady Container binaries
+
+ Select your OS from the dropdown list then click on "Download" (On a Mac, you'll download crc-macos-amd64.tar.xz; on Linux, crc-linux-amd64.tar.xz).
+ You'll be asked to connect using your Red Hat login. If you don't have one, just click on "Create one now". You do *not* need a Red Hat subscription for this.
+
+ Once logged in, download CodeReady Containers binary and the pull secret.
+
+1. Unzip the tar file.
+
+ ```bash
+ tar -xvzf crc-macos-amd64.tar.xz
+ ```
+
+1. Move the crc binary under your path. Like
+
+ ```bash
+ sudo cp `pwd`/crc-macos-$CRCVERSION-amd64/crc /usr/local/bin
+ ```
+
+1. Configure CRC in accordance with your hardware capabilities.
+
+ ```bash
+ crc config set memory 16000
+ crc config set cpus 12
+ crc setup
+ ```
+1. Start the CRC and open the console.
+
+ ```bash
+ crc start --pull-secret-file pull-secret.txt
+ crc console
+ ```
+
+## Testing a patch
+
+The following steps assume you have a running CRC cluster in your laptop. Note that these steps are not tested against a remote CRC cluster.
+
+1. Access your environment through the `oc` command.
+
+ Type in the `crc oc-env` command to a shell.
+ ```bash
+ $ crc oc-env
+ export PATH="/Users/<user>/.crc/bin/oc:$PATH"
+ # Run this command to configure your shell:
+ # eval $(crc oc-env)
+ ```
+ So you need to type in this to access the `oc` comamnd:
+ ```
+ eval $(crc oc-env)
+ ```
+
+1. Log in to `oc`. After the CRC has started it will display a similar message:
+
+ ```
+ To access the cluster, first set up your environment by following 'crc oc-env' instructions.
+ Then you can access it by running 'oc login -u developer -p developer https://api.crc.testing:6443'.
+ To login as an admin, run 'oc login -u kubeadmin -p duduw-yPT9Z-hsUpq-f3pre https://api.crc.testing:6443'.
+ To access the cluster, first set up your environment by following 'crc oc-env' instructions.
+ ```
+
+ Use the `oc login -u kubeadmin ...` command.
+
+1. Get the URL of the local OpenShift cluster's internal private Docker repository by typing the command below.
+
+ ```bash
+ $ oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}'
+ default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+ By default it should be `default-route-openshift-image-registry.apps-crc.testing`. Change the steps above, if the displayed URL is different.
+
+1. Prepare the Docker images.
+
+ You can read more about this at the bottom, in the *Using custom images* section.
+
+1. Prepare the helm chart.
+
+ If you want to use custom Docker images, replace the images in the chart's `values.yaml` config file.
+
+ Note that if you manually pushed the Docker image to the `default-route-openshift-image-registry.apps-crc.testing` docker registry directly you need to have valid certs to access it.
+ On OpenShift there's service for this: `image-registry.openshift-image-registry.svc`, which is easier to use.
+
+ For example, if you want to override all of the three Docker images you should use the following configs:
+ ```yaml
+ image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn
+ tag: scheduler-latest
+ pullPolicy: Always
+
+ admission_controller_image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn
+ tag: admission-latest
+ pullPolicy: Always
+
+ web_image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn-web
+ tag: latest
+ pullPolicy: Always
+ ```
+
+ You can find it in the yunikorn-release repo's helm chart directory.
+
+1. Install the helm charts.
+
+ ```bash
+ helm install yunikorn . -n yunikorn
+ ```
+
+## Using custom images
+
+### Podman
+
+1. Log in into Podman using the following command.
+
+ ```bash
+ podman login --tls-verify=false -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+1. Build the image in the repository e.g. in shim using the generic `make image` command.
+
+1. Verify that the image is present in the repository.
+
+ ```bash
+ podman images
+ REPOSITORY TAG IMAGE ID CREATED SIZE
+ localhost/apache/yunikorn admission-latest 19eb41241d64 About a minute ago 53.5 MB
+ localhost/apache/yunikorn scheduler-latest e60e09b424d9 About a minute ago 543 MB
+ ```
+
+## Directly pushing OS Image Registry
+
+1. Create the images that you wish to replace.
+
+ You can either build new images locally or use official (maybe mix both).
+ * For the -shim and -web images checkout the repository (optionally make your changes) and type the following command:
+ ```bash
+ make clean image REGISTRY=default-route-openshift-image-registry.apps-crc.testing/<project>/<name>:<tag>
+ ```
+ Note that in OpenShift a project is equivalent a Kubernetes namespace. The `yunikorn` project/namespace is recommended.
+ * Using an official image is possible by, retagging it with by the `docker tag` command.
+ ```bash
+ docker tag apache/yunikorn:scheduler-latest default-route-openshift-image-registry.apps-crc.testing/yunikorn/yunikorn:scheduler-latest
+ ```
+
+1. Login to the Docker repository.
+ ```bash
+ docker login -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+1. Push the Docker images to the internal Docker repository
+ ```
+ docker push default-route-openshift-image-registry.apps-crc.testing/yunikorn/yunikorn:scheduler-latest
+ ```
diff --git a/versioned_docs/version-0.10.0/get_started/core_features.md b/versioned_docs/version-0.10.0/get_started/core_features.md
new file mode 100644
index 0000000..0b80dcd
--- /dev/null
+++ b/versioned_docs/version-0.10.0/get_started/core_features.md
@@ -0,0 +1,34 @@
+---
+id: core_features
+title: Features
+keywords:
+ - feature
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The main features of YuniKorn include:
+
+## Gang Scheduling
+In this release, YuniKorn starts to support the Gang Scheduling. Users can apply Gang Scheduling for the applications requiring gang scheduling semantics, such as Spark, Tensorflow, Pytorch, etc. YuniKorn proactively reserves resources for gang scheduling applications, which can work more efficiently with cluster-autoscaler. The initial support has been well tested with Spark, and it can be used with the native Spark on K8s or the Spark K8s operator. For more information how to enable an [...]
+
+## Application Tracking API and CRD Phase One
+This release introduces an application tracking API and K8s custom resource definition (CRD) to further improve the user experience. The CRD will be used by the app operator/job server to interact with YuniKorn, to provide a better app lifecycle management. The first phase has defined the common protocol messages and CRD object formats.
+
diff --git a/versioned_docs/version-0.10.0/get_started/get_started.md b/versioned_docs/version-0.10.0/get_started/get_started.md
new file mode 100644
index 0000000..aa526d0
--- /dev/null
+++ b/versioned_docs/version-0.10.0/get_started/get_started.md
@@ -0,0 +1,73 @@
+---
+id: user_guide
+title: Get Started
+slug: /
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Before reading this guide, we assume you either have a Kubernetes cluster, or a local Kubernetes dev environment, e.g MiniKube.
+It is also assumed that `kubectl` is on your path and properly configured.
+Follow this [guide](developer_guide/env_setup.md) on how to setup a local Kubernetes cluster using docker-desktop.
+
+## Install
+
+The easiest way to get started is to use our Helm Charts to deploy YuniKorn on an existing Kubernetes cluster.
+It is recommended to use Helm 3 or later versions.
+
+```shell script
+helm repo add yunikorn https://apache.github.io/incubator-yunikorn-release
+helm repo update
+kubectl create namespace yunikorn
+helm install yunikorn yunikorn/yunikorn --namespace yunikorn
+```
+
+By default, the helm chart will install the scheduler, web-server and the admission-controller in the cluster.
+When `admission-controller` is installed, it simply routes all traffic to YuniKorn. That means the resource scheduling
+is delegated to YuniKorn. You can disable it by setting `embedAdmissionController` flag to false during the helm install.
+
+Further configuration options for installing YuniKorn via Helm are available in the [YuniKorn Helm hub page](https://hub.helm.sh/charts/yunikorn/yunikorn).
+
+If you don't want to use helm charts, you can find our step-by-step
+tutorial [here](developer_guide/deployment.md).
+
+## Uninstall
+
+Run the following command to uninstall YuniKorn:
+```shell script
+helm uninstall yunikorn --namespace yunikorn
+```
+
+## Access the Web UI
+
+When the scheduler is deployed, the web UI is also deployed in a container.
+Port forwarding for the web interface on the standard port can be turned on via:
+
+```
+kubectl port-forward svc/yunikorn-service 9889:9889 -n yunikorn
+```
+
+`9889` is the default port for web UI.
+Once this is done, web UI will be available at: `http://localhost:9889`.
+
+
+
+YuniKorn UI provides a centralised view for cluster resource capacity, utilization, and all application info.
+
diff --git a/versioned_docs/version-0.10.0/performance/evaluate_perf_function_with_kubemark.md b/versioned_docs/version-0.10.0/performance/evaluate_perf_function_with_kubemark.md
new file mode 100644
index 0000000..f2c7090
--- /dev/null
+++ b/versioned_docs/version-0.10.0/performance/evaluate_perf_function_with_kubemark.md
@@ -0,0 +1,103 @@
+---
+id: evaluate_perf_function_with_kubemark
+title: Evaluate YuniKorn function & performance with Kubemark
+keywords:
+ - performance
+ - throughput
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+All the following tests are done with [Kubemark](https://github.com/kubernetes/kubernetes/blob/release-1.3/docs/devel/kubemark-guide.md#starting-a-kubemark-cluster),
+a tool helps us to simulate large K8s cluster and run experimental workloads.
+There were 18 bare-metal servers being used to simulate 2000/4000 nodes for these tests.
+
+## Scheduler Throughput
+
+When running Big Data batch workloads, e.g Spark, on K8s, scheduler throughput becomes to be one of the main concerns.
+In YuniKorn, we have done lots of optimizations to improve the performance, such as a fully async event-driven system
+and low-latency sorting policies. The following chart reveals the scheduler throughput (by using Kubemark simulated
+environment, and submitting 50,000 pods), comparing to the K8s default scheduler.
+
+
+
+The charts record the time spent until all pods are running on the cluster
+
+| | THROUGHPUT (pods/sec) | THROUGHPUT (pods/sec) |
+|----------------------- |:---------------------: |:---------------------: |
+| ENVIRONMENT (# nodes) | Default Scheduler | YuniKorn |
+| 2000 | 263 | 617 |
+| 4000 | 141 | 373 |
+
+## Resource Fairness between queues
+
+Each of YuniKorn queues has its guaranteed and maximum capacity. When we have lots of jobs submitted to these queues,
+YuniKorn ensures each of them gets its fair share. When we monitor the resource usage of these queues, we can clearly
+see how fairness was enforced:
+
+
+
+We set up 4 heterogeneous queues on this cluster, and submit different workloads against these queues.
+From the chart, we can see the queue resources are increasing nearly in the same trend, which means the resource
+fairness across queues is honored.
+
+## Node sorting policies
+
+There are 2 node sorting policies available in YuniKorn, with regarding the pod distributing flavors. One is *FAIR*,
+which tries best to evenly distribute pods to nodes; the other one is *BIN-PACKING*, which tries best to bin pack pods
+to less number of nodes. The former one is suitable for the Data Center scenarios, it helps to balance the stress of
+cluster nodes; the latter one is suitable to be used on Cloud, it can minimize the number of instances when working
+with auto-scaler, in order to save cost.
+
+### FAIR Policy
+
+We group nodes into 10 buckets, each bucket represents for the number of nodes that has a similar resource
+utilization (a range). To help you understand the chart, imagine the buckets have the following values at a certain
+point of time:
+
+| BUCKET | RESOURCE UTILIZATION RANGE | VALUE |
+|:--------: |:--------------------------: |:-----: |
+| bucket-0 | 0% - 10% | 100 |
+| bucket-1 | 10% - 20% | 300 |
+| ... | | |
+| bucket-9 | 90% - 100% | 0 |
+
+This means at the given time, this cluster has 100 nodes whose utilization is in the range 0% to 10%;
+it has 300 nodes whose utilization is in the range 10% - 20%, and so on… Now, we run lots of workloads and
+collect metrics, see the below chart:
+
+
+
+We can see all nodes have 0% utilization, and then all of them move to bucket-1, then bucket-2 … and eventually
+all nodes moved to bucket-9, which means all capacity is used. In another word, nodes’ resource has been used in
+a fairness manner.
+
+### BIN-PACKING
+
+This is When the bin-packing policy is enabled, we can see the following pattern:
+
+
+
+On the contrary, all nodes are moving between 2 buckets, bucket-0 and bucket-9. Nodes in bucket-0 (0% - 10%)
+are decreasing in a linear manner, and nodes in bucket-9 (90% - 100%) are increasing with the same curve.
+In other words, node resources are being used up one by one.
+
+
+
diff --git a/versioned_docs/version-0.10.0/performance/metrics.md b/versioned_docs/version-0.10.0/performance/metrics.md
new file mode 100644
index 0000000..d7ebfa4
--- /dev/null
+++ b/versioned_docs/version-0.10.0/performance/metrics.md
@@ -0,0 +1,72 @@
+---
+id: metrics
+title: Scheduler Metrics
+keywords:
+ - metrics
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn leverages [Prometheus](https://prometheus.io/) to record metrics. The metrics system keeps tracking of
+scheduler's critical execution paths, to reveal potential performance bottlenecks. Currently, there are two categories
+for these metrics:
+
+- scheduler: generic metrics of the scheduler, such as allocation latency, num of apps etc.
+- queue: each queue has its own metrics sub-system, tracking queue status.
+
+all metrics are declared in `yunikorn` namespace.
+
+## Access Metrics
+
+YuniKorn metrics are collected through Prometheus client library, and exposed via scheduler restful service.
+Once started, they can be accessed via endpoint http://localhost:9080/ws/v1/metrics.
+
+## Aggregate Metrics to Prometheus
+
+It's simple to setup a Prometheus server to grab YuniKorn metrics periodically. Follow these steps:
+
+- Setup Prometheus (read more from [Prometheus docs](https://prometheus.io/docs/prometheus/latest/installation/))
+
+- Configure Prometheus rules: a sample configuration
+
+```yaml
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['docker.for.mac.host.internal:9080']
+```
+
+- start Prometheus
+
+```shell script
+docker pull prom/prometheus:latest
+docker run -p 9090:9090 -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus
+```
+
+Use `docker.for.mac.host.internal` instead of `localhost` if you are running Prometheus in a local docker container
+on Mac OS. Once started, open Prometheus web UI: http://localhost:9090/graph. You'll see all available metrics from
+YuniKorn scheduler.
+
diff --git a/versioned_docs/version-0.10.0/performance/profiling.md b/versioned_docs/version-0.10.0/performance/profiling.md
new file mode 100644
index 0000000..4050ccd
--- /dev/null
+++ b/versioned_docs/version-0.10.0/performance/profiling.md
@@ -0,0 +1,122 @@
+---
+id: profiling
+title: Profiling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Use [pprof](https://github.com/google/pprof) to do CPU, Memory profiling can help you understand the runtime status of YuniKorn scheduler. Profiling instruments have been
+added to YuniKorn rest service, we can easily retrieve and analyze them from HTTP
+endpoints.
+
+## CPU profiling
+
+At this step, ensure you already have YuniKorn running, it can be either running from
+local via a `make run` command, or deployed as a pod running inside of K8s. Then run
+
+```
+go tool pprof http://localhost:9080/debug/pprof/profile
+```
+
+The profile data will be saved on local file system, once that is done, it enters into
+the interactive mode. Now you can run profiling commands, such as
+
+```
+(pprof) top
+Showing nodes accounting for 14380ms, 44.85% of 32060ms total
+Dropped 145 nodes (cum <= 160.30ms)
+Showing top 10 nodes out of 106
+ flat flat% sum% cum cum%
+ 2130ms 6.64% 6.64% 2130ms 6.64% __tsan_read
+ 1950ms 6.08% 12.73% 1950ms 6.08% __tsan::MetaMap::FreeRange
+ 1920ms 5.99% 18.71% 1920ms 5.99% __tsan::MetaMap::GetAndLock
+ 1900ms 5.93% 24.64% 1900ms 5.93% racecall
+ 1290ms 4.02% 28.67% 1290ms 4.02% __tsan_write
+ 1090ms 3.40% 32.06% 3270ms 10.20% runtime.mallocgc
+ 1080ms 3.37% 35.43% 1080ms 3.37% __tsan_func_enter
+ 1020ms 3.18% 38.62% 1120ms 3.49% runtime.scanobject
+ 1010ms 3.15% 41.77% 1010ms 3.15% runtime.nanotime
+ 990ms 3.09% 44.85% 990ms 3.09% __tsan::DenseSlabAlloc::Refill
+```
+
+you can type command such as `web` or `gif` to get a graph that helps you better
+understand the overall performance on critical code paths. You can get something
+like below:
+
+
+
+Note, in order to use these
+options, you need to install the virtualization tool `graphviz` first, if you are using Mac, simply run `brew install graphviz`, for more info please refer [here](https://graphviz.gitlab.io/).
+
+## Memory Profiling
+
+Similarly, you can run
+
+```
+go tool pprof http://localhost:9080/debug/pprof/heap
+```
+
+this will return a snapshot of current heap which allows us to check memory usage.
+Once it enters the interactive mode, you can run some useful commands. Such as
+top can list top memory consumption objects.
+```
+(pprof) top
+Showing nodes accounting for 83.58MB, 98.82% of 84.58MB total
+Showing top 10 nodes out of 86
+ flat flat% sum% cum cum%
+ 32MB 37.84% 37.84% 32MB 37.84% github.com/apache/incubator-yunikorn-core/pkg/cache.NewClusterInfo
+ 16MB 18.92% 56.75% 16MB 18.92% github.com/apache/incubator-yunikorn-core/pkg/rmproxy.NewRMProxy
+ 16MB 18.92% 75.67% 16MB 18.92% github.com/apache/incubator-yunikorn-core/pkg/scheduler.NewScheduler
+ 16MB 18.92% 94.59% 16MB 18.92% github.com/apache/incubator-yunikorn-k8shim/pkg/dispatcher.init.0.func1
+ 1.04MB 1.23% 95.81% 1.04MB 1.23% k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypeWithName
+ 0.52MB 0.61% 96.43% 0.52MB 0.61% github.com/gogo/protobuf/proto.RegisterType
+ 0.51MB 0.61% 97.04% 0.51MB 0.61% sync.(*Map).Store
+ 0.50MB 0.6% 97.63% 0.50MB 0.6% regexp.onePassCopy
+ 0.50MB 0.59% 98.23% 0.50MB 0.59% github.com/json-iterator/go.(*Iterator).ReadString
+ 0.50MB 0.59% 98.82% 0.50MB 0.59% text/template/parse.(*Tree).newText
+```
+
+you can also run `web`, `pdf` or `gif` command to get the graph for heap.
+
+## Download profiling samples and analyze it locally
+
+We have included essential go/go-tool binaries in scheduler docker image, you should be able to do some basic profiling
+analysis inside of the docker container. However, if you want to dig into some issues, it might be better to do the analysis
+locally. Then you need to copy the samples file to local environment first. The command to copy files is like following:
+
+```
+kubectl cp ${SCHEDULER_POD_NAME}:${SAMPLE_PATH_IN_DOCKER_CONTAINER} ${LOCAL_COPY_PATH}
+```
+
+for example
+
+```
+kubectl cp yunikorn-scheduler-cf8f8dd8-6szh5:/root/pprof/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+once you get the file in your local environment, then you can run the `pprof` command for analysis.
+
+```
+go tool pprof /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+## Resources
+
+* pprof Document https://github.com/google/pprof/tree/master/doc.
diff --git a/versioned_docs/version-0.10.0/user_guide/acls.md b/versioned_docs/version-0.10.0/user_guide/acls.md
new file mode 100644
index 0000000..8f41a80
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/acls.md
@@ -0,0 +1,143 @@
+---
+id: acls
+title: ACLs
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+:::caution
+User information is currently not passed to the core scheduler from the kubernetes shim.
+Therefore, the recommendation is to use the wildcard ACL on the root queue for now as per the default configuration.
+:::
+
+## Usage
+Access Control Lists are generic for YuniKorn.
+They can be used in multiple places in YuniKorn.
+The current use case is limited to queue ACLs.
+
+Access control lists give access to the users and groups that have been specified in the list.
+They do not provide the possibility to explicitly remove or deny access to the users and groups specified in the list.
+
+If there is no access control list is configured access is *denied* by default.
+
+## Syntax
+The access control list is defined as:
+```
+ACL ::= “*” | userlist [ “ “ grouplist ]
+userlist ::= “” | user { “,” user }
+grouplist ::= “” | group { “,” group }
+```
+
+This definition specifies a wildcard of * which results in access for everyone.
+
+If the user list is empty and the group list is empty nobody will have access.
+This deny all ACL has two possible representations:
+* an empty access control list. (implicit)
+* a single space. (explicit)
+
+## Example config
+
+### Simple examples
+An ACL that allows access to just the user `sue`
+```yaml
+ adminacl: sue
+```
+Nobody else will get access, this is just for `sue`.
+`john` and `bob` will be denied access.
+
+An ACL that allows access to the user `sue` and the members of the group `dev`.
+```yaml
+ adminacl: sue dev
+```
+The user `sue` gets access based on her explicit mention in the user part of the ACL.
+Even though she is not a member of the group `dev`. Her group membership is irrelevant.
+
+The user named `john` whom is a member of the group `dev` will be allowed access based on his group membership.
+A third user called `bob` whom is not mentioned explicitly and is not a member of the `dev` group will be denied access.
+
+An ACL that allows access to the members of the groups `dev` and `test`.
+```yaml
+ adminacl: " dev,test"
+```
+The ACL must start with a space to indicate that there is no user list.
+If the ACL is not correctly quoted the space is dropped by the yaml parser.
+Since the user list is empty none of the users will get access unless they are a member of either the `dev` or `test` group.
+
+Looking at the same three users as before:
+The user `sue` is not a member of either group and is denied access.
+The user named `john` whom is a member of the group `dev` will be allowed access based on his group membership.
+`bob` is not a member of the `dev` group but is a member of `test` and will be allowed access.
+
+### Escaping and quotation marks
+ACLs are currently implemented in the queue configuration which uses a yaml file.
+This places some limitations on the how to escape the values.
+Incorrectly quoted values will cause a yaml parse error or could lead to the incorrect interpretation of the value.
+
+The following points need to be taken into account:
+1. The wildcard entry must be quoted in the yaml config.
+1. A simple list of users with or without it being followed by a list of groups does not need quoting but may be quoted.
+1. An ACL without a user list and just one or more groups must be quoted to find the starting space:
+
+Correctly quoted ACL example
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: test
+ submitacl: "*"
+ adminacl: sue dev,test
+ - name: product
+ submitacl: " product"
+```
+
+## Access check
+The access check follows the pattern:
+* check if the ACL is the wildcard
+* check if the user is in the user list
+* check if any of the groups the user is a member of is part of the group list
+
+If a check matches the ACL allows access and checking is stopped.
+If none of the checks match the ACL denies access.
+
+## User and Group information
+ACLs require the user's name and group membership.
+User information must be provided by the shims to the core scheduler.
+The current expectation is that the shims only provide the user information and leave the group information empty.
+
+User information is passed around in the scheduler as a combined user and groups object.
+These objects are cached to allow fast lookup and minimise resolution of the groups.
+
+Based on the fact that the shims do not have to provide group information the core has the possibility to resolve the group memberships.
+Group membership resolution is pluggable, see [resolution](#resolution) below.
+If the resolution of the groups of a user fails the result is still cached with a shortened lifetime.
+Users resolution is cached, negatively and positively, per partition.
+Users resolution like many other configs can differ between partition.
+
+### Resolution
+Groups do not have to be part of provided user and group object.
+When the object is added to the cache the groups are automatically resolved based on the resolution that is configured.
+The resolver which is linked to the cache can be set per partition.
+
+The default group resolver is "no resolver".
+This resolver just echos the user name and a primary group with the same name as the user.
+
+Other resolvers are:
+* OS resolver
+* test resolver
diff --git a/versioned_docs/version-0.10.0/user_guide/gang_scheduling.md b/versioned_docs/version-0.10.0/user_guide/gang_scheduling.md
new file mode 100644
index 0000000..abdfe7b
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/gang_scheduling.md
@@ -0,0 +1,221 @@
+---
+id: gang_scheduling
+title: Gang Scheduling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## What is Gang Scheduling
+
+When Gang Scheduling is enabled, YuniKorn schedules the app only when
+the app’s minimal resource request can be satisfied. Otherwise, apps
+will be waiting in the queue. Apps are queued in hierarchy queues,
+with gang scheduling enabled, each resource queue is assigned with the
+maximum number of applications running concurrently with min resource guaranteed.
+
+
+
+## Enable Gang Scheduling
+
+There is no cluster-wide configuration needed to enable Gang Scheduling.
+The scheduler actively monitors the metadata of each app, if the app has included
+a valid taskGroups definition, it will be considered as gang scheduling desired.
+
+:::info Task Group
+A task group is a “gang” of tasks in an app, these tasks are having the same resource profile
+and the same placement constraints. They are considered as homogeneous requests that can be
+treated as the same kind in the scheduler.
+:::
+
+### Prerequisite
+
+For the queues which runs gang scheduling enabled applications, the queue sorting policy needs to be set either
+`FIFO` or `StateAware`. To configure queue sorting policy, please refer to doc: [app sorting policies](user_guide/sorting_policies.md#Application_sorting).
+
+:::info Why FIFO based sorting policy?
+When Gang Scheduling is enabled, the scheduler proactively reserves resources
+for each application. If the queue sorting policy is not FIFO based (StateAware is FIFO based sorting policy),
+the scheduler might reserve partial resources for each app and causing resource segmentation issues.
+:::
+
+### App Configuration
+
+On Kubernetes, YuniKorn discovers apps by loading metadata from individual pod, the first pod of the app
+is required to enclosed with a full copy of app metadata. If the app doesn’t have any notion about the first or second pod,
+then all pods are required to carry the same taskGroups info. Gang scheduling requires taskGroups definition,
+which can be specified via pod annotations. The required fields are:
+
+| Annotation | Value |
+|----------------------------------------------- |--------------------- |
+| yunikorn.apache.org/task-group-name | Task group name, it must be unique within the application |
+| yunikorn.apache.org/task-groups | A list of task groups, each item contains all the info defined for the certain task group |
+| yunikorn.apache.org/schedulingPolicyParameters | Optional. A arbitrary key value pairs to define scheduling policy parameters. Please read [schedulingPolicyParameters section](#scheduling-policy-parameters) |
+
+#### How many task groups needed?
+
+This depends on how many different types of pods this app requests from K8s. A task group is a “gang” of tasks in an app,
+these tasks are having the same resource profile and the same placement constraints. They are considered as homogeneous
+requests that can be treated as the same kind in the scheduler. Use Spark as an example, each job will need to have 2 task groups,
+one for the driver pod and the other one for the executor pods.
+
+#### How to define task groups?
+
+The task group definition is a copy of the app’s real pod definition, values for fields like resources, node-selector
+and toleration should be the same as the real pods. This is to ensure the scheduler can reserve resources with the
+exact correct pod specification.
+
+#### Scheduling Policy Parameters
+
+Scheduling policy related configurable parameters. Apply the parameters in the following format in pod's annotation:
+
+```yaml
+annotations:
+ yunikorn.apache.org/schedulingPolicyParameters: "PARAM1=VALUE1 PARAM2=VALUE2 ..."
+```
+
+Currently, the following parameters are supported:
+
+`placeholderTimeoutInSeconds`
+
+Default value: *15 minutes*.
+This parameter defines the reservation timeout for how long the scheduler should wait until giving up allocating all the placeholders.
+The timeout timer starts to tick when the scheduler *allocates the first placeholder pod*. This ensures if the scheduler
+could not schedule all the placeholder pods, it will eventually give up after a certain amount of time. So that the resources can be
+freed up and used by other apps. If non of the placeholders can be allocated, this timeout won't kick-in. To avoid the placeholder
+pods stuck forever, please refer to [troubleshooting](trouble_shooting.md#gang-scheduling) for solutions.
+
+More scheduling parameters will added in order to provide more flexibility while scheduling apps.
+
+#### Example
+
+The following example is a yaml file for a job. This job launches 2 pods and each pod sleeps 30 seconds.
+The notable change in the pod spec is *spec.template.metadata.annotations*, where we defined `yunikorn.apache.org/task-group-name`
+and `yunikorn.apache.org/task-groups`.
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: gang-scheduling-job-example
+spec:
+ completions: 2
+ parallelism: 2
+ template:
+ metadata:
+ labels:
+ app: sleep
+ applicationId: "gang-scheduling-job-example"
+ queue: root.sandbox
+ annotations:
+ yunikorn.apache.org/task-group-name: task-group-example
+ yunikorn.apache.org/task-groups: |-
+ [{
+ "name": "task-group-example",
+ "minMember": 2,
+ "minResource": {
+ "cpu": "100m",
+ "memory": "50M"
+ },
+ "nodeSelector": {},
+ "tolerations": []
+ }]
+ spec:
+ schedulerName: yunikorn
+ restartPolicy: Never
+ containers:
+ - name: sleep30
+ image: "alpine:latest"
+ command: ["sleep", "30"]
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50M"
+```
+
+When this job is submitted to Kubernetes, 2 pods will be created using the same template, and they all belong to one taskGroup:
+*“task-group-example”*. YuniKorn will create 2 placeholder pods, each uses the resources specified in the taskGroup definition.
+When all 2 placeholders are allocated, the scheduler will bind the the real 2 sleep pods using the spot reserved by the placeholders.
+
+You can add more than one taskGroups if necessary, each taskGroup is identified by the taskGroup name,
+it is required to map each real pod with a pre-defined taskGroup by setting the taskGroup name. Note,
+the task group name is only required to be unique within an application.
+
+### Enable Gang scheduling for Spark jobs
+
+Each Spark job runs 2 types of pods, driver and executor. Hence, we need to define 2 task groups for each job.
+The annotations for the driver pod looks like:
+
+```yaml
+Annotations:
+ yunikorn.apache.org/schedulingPolicyParameters: “placeholderTimeoutSeconds=30”
+ yunikorn.apache.org/taskGroupName: “spark-driver”
+ yunikorn.apache.org/taskGroup: “
+ TaskGroups: [
+ {
+ Name: “spark-driver”
+ minMember: 1,
+ minResource: {
+ Cpu: 1,
+ Memory: 2Gi
+ },
+ Node-selector: ...
+ Tolerations: ...
+ },
+ {
+ Name: “spark-executor”,
+ minMember: 10,
+ minResource: {
+ Cpu: 1,
+ Memory: 2Gi
+ }
+ }
+ ]
+ ”
+```
+
+:::note
+Spark driver and executor pod has memory overhead, that needs to be considered in the taskGroup resources.
+:::
+
+For all the executor pods,
+
+```yaml
+Annotations:
+ # the taskGroup name should match to the names
+ # defined in the taskGroups field
+ yunikorn.apache.org/taskGroupName: “spark-executor”
+```
+
+Once the job is submitted to the scheduler, the job won’t be scheduled immediately.
+Instead, the scheduler will ensure it gets its minimal resources before actually starting the driver/executors.
+
+## Verify Configuration
+
+To verify if the configuration has been done completely and correctly, check the following things:
+1. When an app is submitted, verify the expected number of placeholders are created by the scheduler.
+If you define 2 task groups, 1 with minMember 1 and the other with minMember 5, that means we are expecting 6 placeholder
+gets created once the job is submitted.
+2. Verify the placeholder spec is correct. Each placeholder needs to have the same info as the real pod in the same taskGroup.
+Check field including: namespace, pod resources, node-selector, and toleration.
+3. Verify the placeholders can be allocated on correct type of nodes, and verify the real pods are started by replacing the placeholder pods.
+
+## Troubleshooting
+
+Please see the troubleshooting doc when gang scheduling is enabled [here](trouble_shooting.md#gang-scheduling).
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/user_guide/placement_rules.md b/versioned_docs/version-0.10.0/user_guide/placement_rules.md
new file mode 100644
index 0000000..5f2c64d
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/placement_rules.md
@@ -0,0 +1,354 @@
+---
+id: placement_rules
+title: App Placement Rules
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The basics for the placement rules are described in the [scheduler configuration design document](design/scheduler_configuration.md#placement-rules-definition).
+Multiple rules can be chained to form a placement policy.
+[Access control lists](user_guide/acls.md) and rule filters are defined per rule and enforced per rule.
+This document explains how to build a policy, including the rule usage, that is part of the scheduler with examples.
+
+## Configuration
+Rules are defined per partition as part of the scheduler queue configuration.
+The order that the rules are defined in is the order in which they are executed.
+If a rule matches the policy will stop executing the remaining rules.
+
+A matching rule generates a fully qualified queue name.
+This means that the name returned starts at the _root_ queue.
+There is no limit on the number of levels in the queue hierarchy that can be generated.
+
+When a rule is executed the result of rules that have been executed is unknown and not taken into account.
+Similar for rule that have not been executed yet: rules cannot influence other rules except when they are configured as the [parent](#parent-parameter) rule.
+
+If the policy does not generate a queue name and no more rules are left the application will be rejected.
+
+Basic structure for the rule placement definition in the configuration:
+```yaml
+placementrules:
+ - name: <name of the 1st rule>
+ - name: <name of the 2nd rule>
+```
+Each rule can take a predefined set of parameters in the configuration.
+The name of the rules that can be used are given in the [rule](#rules) description.
+A rule name is not case sensitive in the configuration.
+Rule name must follow the following naming convention:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _
+
+A rule that is not known, i.e. the name does not map to the rules defined below, will cause a initialisation error of the placement manager.
+Rules can also throw a parse error or an error during the initialisation if the parameters are incorrect.
+A rule set with an error can never become active.
+
+A placement manager is considered initialised if it has an active rule set.
+When the configuration is reloaded a new rule set will be created to replace the active rule set.
+In the case that a new rule set loaded contains an error, i.e. is broken, the placement manager ignores the new rule set.
+This means that the placement manager stays in a the state it was in when a broken rule set is loaded.
+If the placement manager keeps using the existing active rule set in the case that it was already initialised.
+A message will be logged about the broken and ignored configuration.
+
+Dots "." in the rule result are replaced by the string "\_dot_".
+A dot is replaced because it is used as the hierarchy separator in the fully qualified queue name.
+Replacing the dot occurs before the full queue hierarchy is build and the result is qualified.
+This means that we allow user name and or tag values to contain dots without the dots affecting the queue hierarchy.
+For queues in the configuration that as an example must map to username with a dot you must specify them as follows:
+A user rule with the user `user.name` will generate the queue name `root.user_dot_name` as output.
+If that "user queue" must be added to the configuration the `user_dot_name` name should be used.
+
+### Create parameter
+The create parameter is a boolean flag that defines if a queue that is generated by the rule may be created if it does not exist.
+There is no guarantee that the queue will be created because the existing queues might prevent the queue creation.
+If the queue generated by the rule does not exist and the flag is not set to _true_ the result of the rule will be a fail.
+
+Basic yaml entry for a rule with `create` flag:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ create: true
+```
+
+The default value is _false_.
+Allowed values: _true_ or _false_, any other value will cause a parse error.
+
+### Parent parameter
+The parent parameter allows specifying a rule that generates a parent queue for the current rule.
+Parent rules can be nested, a parent rule _may_ contain another parent rule.
+There is no enforced limit of parent rules that can be nested.
+
+A parent rule is treated as if it was a rule specified at the top level of the list and thus has the same parameters and requirements as a any other rule in the placement definition.
+The exception is that using a parent rule on a rule that already generates a fully qualified queue is considered a configuration error.
+This error can only occur on the rule of type [fixed](#fixed-rule), see the rule specification for more details.
+
+NOTE: the rule execution traverses down the list of parent rules and executes the last one in the list first.
+This means that the last parent rule will generate the queue directly below the root.
+See the examples for details.
+
+Basic yaml entry for a rule with a `parent` rule:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ parent:
+ name: <name of the parent rule>
+```
+
+The default value is _no_ parent.
+
+### Filter parameter
+The filter on a rule allows filtering the users that the rule applies to.
+A filter is a complex configuration object.
+
+The _users_ and _groups_ that can be configured can be one of two types:
+* a regular expression
+* a list of users or groups.
+
+If the entry for users or groups contains more than 1 entry in the yaml it is always considered a list of either users or groups.
+Duplicate entries in the lists are ignored and do not cause an error.
+Specifying a regular expression beside other list elements is not allowed.
+
+User and group names follow the standard linux user and group conventions.
+For user names:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _ . @ -
+* the last character may also be a $
+
+For group names:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _ -
+
+If the list is exactly one entry it can be either a single user or group or a regular expression.
+When an entry contains a character that is not allowed in a user or group name the entry is considered a regular expression.
+The regular expression must compile as specified.
+A regular expression that does not compile is ignored.
+
+Specifically for the group regular expression: matching is performed one group at a time not the against the list of groups.
+
+Basic `filter` yaml entry:
+```yaml
+filter:
+ type: deny
+ users:
+ - <user name or regexp>
+ - <user name>
+ groups:
+ - <group name or regexp>
+ - <group name>
+```
+
+The default value is _no_ filter.
+
+### Value parameter
+This is a generic value that can be used to pass to a rule to implement or alter its behaviour.
+The value It is used by the [fixed](#fixed-rule) and the [tag](#tag-rule) rule.
+The value is a single value in string form and is not interpreted or manipulated by the system.
+
+Basic yaml entry for a rule with a `value` set:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ value: "any string"
+```
+
+The default is _no_ value.
+
+## Access Control List
+Access control lists are not defined in the rules but they impact the outcome of the placement policy.
+Two access control lists can be defined on a queue:
+1. Submit ACL: _submitacl_
+1. Administration ACL: _adminacl_
+
+The placement rule will only match if the ACL of the queue allows submit access via either ACL.
+The administrative queue ACL also provides _submit_ access.
+If the queue does not exist or does not have an ACL set, the ACL of the parent queue is checked.
+This recursive check is repeated until an ACL provides access or after the ACL of the root queue is checked.
+
+For more detail on the ACL syntax check the [ACL documentation](user_guide/acls.md).
+
+## Rules
+### Provided Rule
+Name to be used in the configuration: *provided*
+
+Returns the queue provided during the submission of the application.
+The behaviour of the this rule is to fully qualify the queue provided by the application if the queue is not fully qualified.
+If a parent rule is set and the queue provided in the application submission is fully qualified then the parent rule will not be executed.
+
+Supported parameters:
+* create
+* parent
+* filter
+
+Example: create the queue passed in by the user if it does not exist under the users name
+```yaml
+placementrules:
+ - name: provided
+ create: true
+ parent:
+ name: user
+ create: true
+```
+Application submit request by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.developer.my_special_queue` (parent rule set the user name)
+
+Application submit request by the user `developer`, queue in the application on submit: `root.dev_queue`<br/>
+Result: `root.dev_queue` (parent rule ignored)
+
+### User Name Rule
+Name to be used in the configuration: *user*
+
+Returns the queue based on the user name that is part of the submitted application.
+
+Supported parameters:
+* create
+* parent
+* filter
+
+Example: submit to a queue based on the user name, do not create the queue if the queue does not exist:
+```yaml
+placementrules:
+ - name: user
+ create: false
+```
+
+Application submit request by the user `finance.test`, queue does exist:<br/>
+Result: `root.finance_dot_test` (note the replacement of the dot)
+
+Application submit request by the user `developer`, queue does not exist:<br/>
+Result: failed, next rule executed
+
+### Fixed Rule
+Name to be used in the configuration: *fixed*
+
+Returns the name configured in the rule parameter _value_.
+The value configured must be a legal queue name or queue hierarchy.
+The name does not have to be a fully qualified queue name.
+The hierarchy in the name uses a dot as a separator for the queue names at the different levels in the hierarchy.
+The fixed rule can only fail if the queue configured does not exist and the create flag is not set as it will always return the configured queue.
+
+If the configured value is a fully qualified queue name configuring a parent rule will be considered an _error_.
+
+Supported parameters:
+* value (required)
+* create
+* parent
+* filter
+
+Example: a fixed queue returned always, without the `create` flag set and thus requires the queue to be defined as a leaf queue in the configuration.
+```yaml
+placementrules:
+ - name: fixed
+ value: last_resort
+```
+
+Application submit request by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.last_resort`
+
+### Tag Rule
+Name to be used in the configuration: *tag*
+Retrieves the queue name from the applications tags.
+The tag name that is checked for its value is configured in the rule using the `value`.
+Configuring a tag rule without a `value` set is an error, however an application does not have to have the tag set.
+
+If the tag is not set on the application the rule fails.
+If the tag value returned from the application is a fully qualified queue name the parent rule, if configured, will not be executed.
+
+Supported parameters:
+* value (required)
+* create
+* parent
+* filter
+
+Example: place an application based on the kubernetes `namespace` which gets sets automatically in the application when submitted:
+```yaml
+placementrules:
+ - name: tag
+ value: namespace
+ create: true
+```
+
+Application submit request for a kubernetes based application in the namespace `default` by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.default`
+
+Application submit request for a kubernetes based application in the namespace `testing` by the user `developer`<br/>
+Result: `root.testing`
+
+Application submit request for a non kubernetes based application by the user `developer`<br/>
+Result: failed, next rule executed
+
+## Complex examples
+In this complex example we chain three rules:
+1. a `user` rule, with a parent rule `tag` using the kubernetes namespace, to be used only for users that are part of and "dev" group.
+1. a `tag` rule using the kubernetes namespace, with a parent rule `fixed` using the existing queue `root.namespaces`.
+1. a `fixed` rule to place everything that reaches this point in the `root.default` queue.
+
+Example:
+```yaml
+placementrules:
+ - name: user
+ create: true
+ filter:
+ type: allow
+ groups:
+ - dev*
+ parent:
+ - name: tag
+ value: namespace
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ - name: fixed
+ value: root.namespaces
+ filter:
+ type: allow
+ users:
+ - john
+ - name: fixed
+ value: root.default
+```
+
+Application submit request for a kubernetes based application in the namespace `testing` by the user `john`<br/>
+Result: `root.namespaces.testing` (matched in rule 2)
+
+Application submit request for a kubernetes based application in the namespace `newapp` by the user `sarah` with groups membership `sarah, test_app, dev_app`<br/>
+Result: `root.newapp.sarah` (matched in rule 1)
+
+Application submit request for a kubernetes based application in the namespace `testapp` by the user `bob` with groups membership `bob`<br/>
+Result: `root.deault` (matched in rule 3)
+
+In this second example we chain two rules:
+1. a `fixed` rule to place everything in the `root.production` queue
+1. a `user` rule, with the create flag set
+
+In this case however we have ACLs set up on the `root.production` queue to only allow two specific user to use this queue.
+So even if the rule matches unless the user is either `john` or `bob` the application will not be placed in the `production` queue.
+All other users will match the second rule and use their own queue, which is created if it does not exist.
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: production
+ submitacl: john,bob
+ placementrules:
+ - name: fixed
+ value: root.production
+ - name: user
+ create: true
+```
diff --git a/versioned_docs/version-0.10.0/user_guide/queue_config.md b/versioned_docs/version-0.10.0/user_guide/queue_config.md
new file mode 100644
index 0000000..fb3e4b8
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/queue_config.md
@@ -0,0 +1,307 @@
+---
+id: queue_config
+title: Partition and Queue Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The basis for the queue configuration is given in the [configuration design document](design/scheduler_configuration.md).
+
+This document provides the generic queue configuration.
+It references both the [Access control lists](user_guide/acls.md) and [placement rule](user_guide/placement_rules.md) documentation.
+
+This document explains how to create the partition and queue configuration for the scheduler with examples.
+
+The scheduler relies on the shim to reliably provide user information as part of the application submission.
+In the current shim there is no reliable way to identify the user and the groups the user belongs to.
+The user and group information provided by the shim is incomplete in the best case.
+This shim limitation impacts the behaviour of user based limits and access control in the scheduler.
+The current implementation only provides the framework in the scheduler and will not be fully functional until the shim changes are added.
+
+## Configuration
+The configuration file for the scheduler that is described here only provides the configuration for the partitions and queues.
+
+By default we use the file called `queues.yaml` in our deployments.
+The filename can be changed via the command line flag `policyGroup` of the scheduler.
+Changing the filename must be followed by corresponding changes in the deployment details, either the `configmap` or the file included in the docker container.
+
+The example file for the configuration is located in the scheduler core's [queues.yaml](https://github.com/apache/incubator-yunikorn-core/blob/master/config/queues.yaml).
+
+## Partitions
+Partitions are the top level of the scheduler configuration.
+There can be more than one partition defined in the configuration.
+
+Basic structure for the partition definition in the configuration:
+```yaml
+partitions:
+ - name: <name of the 1st partition>
+ - name: <name of the 2nd partition>
+```
+The default name for the partition is `default`.
+The partition definition contains the full configuration for the scheduler for a particular shim.
+Each shim uses its own unique partition.
+
+The partition must have at least the following keys defined:
+* name
+* [queues](#queues)
+
+The queues configuration is explained below.
+
+Optionally the following keys can be defined for a partition:
+* [placementrules](#placement-rules)
+* [limits](#limits)
+* nodesortpolicy
+* preemption
+
+Placement rules and limits are explained in their own chapters
+
+The `nodesortpolicy` defines the way the nodes are sorted for the partition.
+Details on the values that can be used are in the [sorting policy](sorting_policies.md#node-sorting) documentation.
+
+The preemption key can currently have only one sub key: _enabled_.
+This boolean value defines the preemption behaviour for the whole partition.
+
+The default value for _enabled_ is _false_.
+Allowed values: _true_ or _false_, any other value will cause a parse error.
+
+Example `partition` yaml entry with _preemption_ flag set and a `nodesortpolicy` of _fair_:
+```yaml
+partitions:
+ - name: <name of the partition>
+ nodesortpolicy: fair
+ preemption:
+ enabled: true
+```
+NOTE:
+Currently the Kubernetes unique shim does not support any other partition than the `default` partition..
+This has been logged as an [jira](https://issues.apache.org/jira/browse/YUNIKORN-22) for the shim.
+
+### Queues
+
+YuniKorn manages resources by leveraging resource queues. The resource queue has the following characters:
+- queues can have **hierarchical** structure
+- each queue can be preset with **min/max capacity** where min-capacity defines the guaranteed resource and the max-capacity defines the resource limit (aka resource quota)
+- tasks must be running under a certain leaf queue
+- queues can be **static** (loading from configuration file) or **dynamical** (internally managed by YuniKorn)
+- queue level **resource fairness is** enforced by the scheduler
+- a job can only run under a specific queue
+
+:::info
+The difference between YuniKorn queue and [Kubernetes namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/):
+Kubernetes namespace provides the scope for the Kubernetes resources, including the security context (i.e who can access the objects), and resource
+boundary when [resource quota](https://kubernetes.io/docs/concepts/policy/resource-quotas/) is defined (i.e how many resources can be used by the objects).
+On the other hand, YuniKorn queue is only used how many resources can be used by a group of jobs, and in which order. It provides
+a more fine-grained control on resource sharing across multiple tenants with considering of resource fairness, job ordering, etc. In most of the cases,
+YuniKorn queue can be used to replace the namespace resource quota, in order to provide more scheduling features.
+:::
+
+The _queues_ entry is the main configuration element.
+It defines a hierarchical structure for the queues.
+
+It can have a `root` queue defined but it is not a required element.
+If the `root` queue is not defined the configuration parsing will insert the root queue for consistency.
+The insertion of the root queue is triggered by:
+* If the configuration has more than one queue defined at the top level a root queue is inserted.
+* If there is only one queue defined at the top level and it is not called `root` a root queue is inserted.
+
+The defined queue or queues will become a child queue of the inserted `root` queue.
+
+Basic `queues` yaml entry with sub queue:
+```yaml
+queues:
+- name: <name of the queue>
+ queues:
+ - name: <name of the queue>
+```
+
+Supported parameters for the queues:
+* name
+* parent
+* queues
+* properties
+* adminacl
+* submitacl
+* [resources](#resources)
+* [limits](#limits)
+
+Each queue must have a _name_.
+The name of a queue must be unique at the level that the queue is defined.
+Since the queue structure is fully hierarchical queues at different points in the hierarchy may have the same name.
+As an example: the queue structure `root.testqueue` and `root.parent.testqueue` is a valid structure.
+A queue cannot contain a dot "." character as that character is used to separate the queues in the hierarchy.
+If the name is not unique for the queue in the configuration or contains a dot a parsing error is generated and the configuration is rejected.
+
+Queues in the structure will automatically get a type assigned.
+The type of the queue is based on the fact that the queue has children or sub queues.
+The two types of queues are:
+* parent
+* leaf
+
+Applications can only be assigned to a _leaf_ queue.
+A queue that has a child or sub queue in the configuration is automatically a _parent_ queue.
+If a queue does not have a sub the queue in the configuration it is a _leaf_ queue, unless the `parent` parameter is set to _true_.
+Trying to override a _parent_ queue type in the configuration will cause a parsing error of the configuration.
+
+Sub queues for a parent queue are defined under the `queues` entry.
+The `queues` entry is a recursive entry for a queue level and uses the exact same set of parameters.
+
+The `properties` parameter is a simple key value pair list.
+The list provides a simple set of properties for the queue.
+There are no limitations on the key or value values, anything is allowed.
+Currently, the property list is only used in the scheduler to define a [sorting order](sorting_policies.md#application-sorting) for a leaf queue.
+Future expansions, like the option to turn on or off preemption on a queue or other sorting policies, would use this same property construct without the need to change the configuration.
+
+Access to a queue is set via the `adminacl` for administrative actions and for submitting an application via the `submitacl` entry.
+ACLs are documented in the [Access control lists](acls.md) document.
+
+Queue resource limits are set via the `resources` parameter.
+User and group limits are set via the `limits` parameter.
+As both entries are complex configuration entries they are explained in [resources](#resources) and [limits](#limits) below.
+
+An example configuration of a queue `root.namespaces` as a _parent_ queue with limits:
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: namespaces
+ parent: true
+ resources:
+ guaranteed:
+ {memory: 1000, vcore: 10}
+ max:
+ {memory: 10000, vcore: 100}
+```
+
+### Placement rules
+
+The placement rules are defined and documented in the [placement rule](placement_rules.md) document.
+
+Each partition can have only one set of placement rules defined.
+If no rules are defined the placement manager is not started and each application *must* have a queue set on submit.
+
+### Limits
+Limits define a set of limit objects for a partition or queue.
+It can be set on either the partition or on a queue at any level.
+```yaml
+limits:
+ - limit: <description>
+ - limit: <description>
+```
+
+A limit object is a complex configuration object.
+It defines one limit for a set of users and or groups.
+Multiple independent limits can be set as part of one `limits` entry on a queue or partition.
+Users and or groups that are not part of the limit setting will not be limited.
+
+A sample limits entry:
+```yaml
+limits:
+ - limit: <description>
+ users:
+ - <user name or "*"">
+ - <user name>
+ groups:
+ - <group name or "*"">
+ - <group name>
+ maxapplications: <1..maxint>
+ maxresources:
+ <resource name 1>: <0..maxint>
+ <resource name 2>: <0..maxint>
+```
+
+Limits are applied recursively in the case of a queue limit.
+This means that a limit on the `root` queue is an overall limit in the cluster for the user or group.
+A `root` queue limit is thus also equivalent with the `partition` limit.
+
+The limit object parameters:
+* limit
+* users
+* groups
+* maxapplications
+* maxresources
+
+The _limit_ parameter is an optional description of the limit entry.
+It is not used for anything but making the configuration understandable and readable.
+
+The _users_ and _groups_ that can be configured can be one of two types:
+* a star "*"
+* a list of users or groups.
+
+If the entry for users or groups contains more than one (1) entry it is always considered a list of either users or groups.
+The star "*" is the wildcard character and matches all users or groups.
+Duplicate entries in the lists are ignored and do not cause a parsing error.
+Specifying a star beside other list elements is not allowed.
+
+_maxapplications_ is an unsigned integer value, larger than 1, which allows you to limit the number of running applications for the configured user or group.
+Specifying a zero maximum applications limit is not allowed as it would implicitly deny access.
+Denying access must be handled via the ACL entries.
+
+The _maxresources_ parameter can be used to specify a limit for one or more resources.
+The _maxresources_ uses the same syntax as the [resources](#resources) parameter for the queue.
+Resources that are not specified in the list are not limited.
+A resource limit can be set to 0.
+This prevents the user or group from requesting the specified resource even though the queue or partition has that specific resource available.
+Specifying an overall resource limit of zero is not allowed.
+This means that at least one of the resources specified in the limit must be greater than zero.
+
+If a resource is not available on a queue the maximum resources on a queue definition should be used.
+Specifying a limit that is effectively zero, _maxapplications_ is zero and all resource limits are zero, is not allowed and will cause a parsing error.
+
+A limit is per user or group.
+It is *not* a combined limit for all the users or groups together.
+
+As an example:
+```yaml
+limit: "example entry"
+maxapplications: 10
+users:
+- sue
+- bob
+```
+In this case both the users `sue` and `bob` are allowed to run 10 applications.
+
+### Resources
+The resources entry for the queue can set the _guaranteed_ and or _maximum_ resources for a queue.
+Resource limits are checked recursively.
+The usage of a leaf queue is the sum of all assigned resources for that queue.
+The usage of a parent queue is the sum of the usage of all queues, leafs and parents, below the parent queue.
+
+The root queue, when defined, cannot have any resource limit set.
+If the root queue has any limit set a parsing error will occur.
+The maximum resource limit for the root queue is automatically equivalent to the cluster size.
+There is no guaranteed resource setting for the root queue.
+
+Maximum resources when configured place a hard limit on the size of all allocations that can be assigned to a queue at any point in time.
+A maximum resource can be set to 0 which makes the resource not available to the queue.
+Guaranteed resources are used in calculating the share of the queue and during allocation.
+It is used as one of the inputs for deciding which queue to give the allocation to.
+Preemption uses the _guaranteed_ resource of a queue as a base which a queue cannot go below.
+
+Basic `resources` yaml entry:
+```yaml
+resources:
+ guaranteed:
+ <resource name 1>: <0..maxint>
+ <resource name 2>: <0..maxint>
+ max:
+ <resource name 1>: <0..maxint>
+ <resource name 2>: <0..maxint>
+```
+Resources that are not specified in the list are not limited, for max resources, or guaranteed in the case of guaranteed resources.
diff --git a/versioned_docs/version-0.10.0/user_guide/resource_quota_mgmt.md b/versioned_docs/version-0.10.0/user_guide/resource_quota_mgmt.md
new file mode 100644
index 0000000..8890fdb
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/resource_quota_mgmt.md
@@ -0,0 +1,324 @@
+---
+id: resource_quota_management
+title: Resource Quota Management
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Quota configuration and rules
+YuniKorn can offer a finer grained resource quota management setup compared to the simple namespace resource quota provided by Kubernetes.
+
+On Kubernetes a pod must fit into the namespace quota when the pod is submitted.
+If the pod does not fit in the namespace quota the pod is rejected.
+The client must implement a retry-mechanism and re-submit the pod if it needs the pod to be scheduled.
+
+Contrary to quotas in Kubernetes YuniKorn does not enforce quotas on submission but only on actively consumed resources.
+To explain the difference: when using YuniKorn for quota enforcement a new pod submitted to Kubernetes is always accepted.
+Yunikorn will queue the pod without counting the queued pod's resources towards the consumed quota.
+When YuniKorn tries to schedule the pod it checks at scheduling time if the pod fits in the quota configured for the queue the pod is assigned to.
+If at that point the pod does not fit in the quota the pod is skipped and not counted in the resource consumption.
+This means that until a scheduling attempt of a pod is successful a pod it is not consuming resources in the YuniKorn quota system.
+
+Resource quotas in YuniKorn are linked to the queue and its place in the queue hierarchy.
+The base of the queue structure, the `root` queue, does not allow setting a quota as it reflects the current size of the cluster.
+Node additions and removals update the `root` queue quota automatically.
+
+Beside the `root` queue the quotas can be set, and is enforced, at any point in the hierarchy.
+Every queue can have a quota set. The quota is enforced recursively throughout the hierarchy.
+This means that a child queue can never use more resources than the **configured** quota of the parent queue.
+Setting a quota on a child queue larger than its parent queue's quota would thus not have any effect and is handled as a configuration error.
+
+In the hierarchy there are some further rules that need to be considered.
+If a parent queue has multiple children the sum of the **usage** of all children combined can never exceed the quota **configured** on the parent.
+However, from a configuration perspective this does not mean that the sum of the **configured** quotas for all children must be smaller than the parent quota.
+
+
+
+As an example the `root.parent` queue has a quota of 900.
+It contains three child queues, two with a quota set.
+The `root.parent.child1` has no quota set and will thus be limited to the `root.parent` quota.
+The two other queues `root.parent.child2` and `root.parent.child3` each have a quota of 750 set.
+During normal operation the total usage of the 3 child queues together will be 900.
+The applications running in each child queue have a demand of more than 1000 each.
+
+Distribution in that case could be any of:
+* all 900 used by just the `child1` queue
+* spread out evenly over the 3 queues (300 by each)
+* `child2` maxed out using 750, and the left over 150 used by `child3`
+
+The exact distribution between the queues will fluctuate and is dependent on the scheduling policies.
+
+## Converting Kubernetes resources and quotas
+Resource support for pods is limited to the resources specified as part of the _requests_ specification:
+* _cpu_ is mapped to _vcore_ with the value in milli cpu.
+* _memory_ is mapped to _memory_ with the value in MB (1 MB = 10^6 B = 1 000 000 B).
+* all other resources are mapped as provided.
+
+Extended resource as per the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) are supported.
+
+Example pod with a single container:
+```yaml
+apiVersion: v1
+kind: Pod
+spec:
+ containers:
+ - name: container-1
+ resources:
+ requests:
+ cpu: "250m"
+ memory: "1Gi"
+ hugepages-1Gi: "1"
+```
+The above specification will set pod resources request for scheduling in YuniKorn to:
+* _vcore_ -> 250
+* _memory_ -> 1074
+* _hugepages-1Gi_ -> 1
+
+Two remarks:
+Multiple container specifications will be aggregated into one total pod resource request automatically.
+All memory is reported in MB with unit conversions applied where needed.
+
+In the case that static queue definitions are used for a queue there is no limit on the type of resource that can be specified in a quota.
+Quota annotations on namespaces, used as part of the automatic queue creation, are limited to the equivalent _cpu_ and _memory_ resources.
+See the [setup](#Namespace-quota) below for the annotations on the namespace for quotas.
+
+## Kubernetes and YuniKorn quota interaction
+The recommendation is to turn off, not configure, the Kubernetes Namespace quotas.
+Using only YuniKorn queue quotas provides a more flexible setup and allows queueing of workloads.
+
+In a setup that has both YuniKorn and Kubernetes quotas turned on consider the following points:
+* Two separate configurations need to be maintained.
+ This increases the maintenance burden, and the possibility of configuration mistakes.
+* Both quotas will be enforced.
+
+Having both quotas turned on can lead to unexpected behaviour.
+The main issue is the fact that the Kubernetes namespace quota is enforced on submit.
+There are three combinations of quota configuration that are possible.
+The 3 combinations could have two effects when used in combination with the YuniKorn quota.
+
+1. Both quotas are _equal_: workloads will not be queued, the full configured quota can be used.
+ - Maximum usage and queueing will be limited to the set quota
+2. Kubernetes quota is _lower_ than YuniKorn: the YuniKorn quota will never be reached and workloads will not be queued.
+ - Maximum usage will be limited to the Kubernetes quota.
+3. Kubernetes quota is _higher_ than YuniKorn: YuniKorn will limit the usage to the quota set in YuniKorn.
+ The Kubernetes quota will be enforced on submit and thus set the limit for the workload that can be queued on top of the YuniKorn quota.
+ - Maximum usage will be limited to the YuniKorn quota.
+ - Workload queueing will be limited to the Kubernetes quota.
+
+:::note
+The following configuration examples are just to demonstrate the format needed
+to create a queue hierarchy with quotas set.
+:::
+
+## Static queue definition
+
+### Goal
+A preconfigured hierarchy of queues with a maximum and guaranteed capacity.
+The users can only submit applications to the leaf queues.
+This approach manages the resource capacity for each of the queues, which is suitable to the scenarios that queues do not change too often.
+
+### Configuration
+Apply the following configuration to YuniKorn's configmap to:
+* setup 3 queues under `root`
+* each queue has a specific guaranteed and maximum capacity
+* anyone can submit to any queue
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ queues:
+ - name: advertisement
+ resources:
+ guaranteed:
+ memory: 500000
+ vcore: 50000
+ max:
+ memory: 800000
+ vcore: 80000
+ - name: search
+ resources:
+ guaranteed:
+ memory: 400000
+ vcore: 40000
+ max:
+ memory: 600000
+ vcore: 60000
+ - name: sandbox
+ resources:
+ guaranteed:
+ memory: 100000
+ vcore: 10000
+ max:
+ memory: 100000
+ vcore: 10000
+```
+
+### Run a workload
+In order to run applications in specific queues, you will need to set the following labels in all pod specs.
+All pods with the same `applicationID` label are considered ti be one application.
+In the below example the application `my-test-app` will run in the queue `root.sandbox`:
+
+```yaml
+labels:
+ app: my-test-app
+ applicationId: "my-test-app-01"
+ queue: root.sandbox
+```
+
+## Namespace to queue mapping
+
+### Goal
+Automatically map a Kubernetes `namespace` to a queue in YuniKorn.
+The user creates the required namespaces in Kubernetes.
+The YuniKorn k8s shim and core scheduler automatically pass the required information and map the namespace to a queue, creating the queue if it does not exist.
+The resource quota will be managed by YuniKorn instead of using the Kubernetes namespace quota.
+This does require the namespaces to be setup without Kubernetes quota enforcement and tags as per the [setup](#Namespace-quota) below.
+
+### Configuration
+Apply the following configuration to YuniKorn's configmap:
+
+```yaml
+partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ properties:
+ application.sort.policy: stateaware
+```
+
+This configuration places an application based on the `tag` rule.
+The tag selected is the `namespace` tag which is automatically added by the k8s shim to all applications that get created.
+The `create` flag is set to true which will trigger the creation of the queue with the same name as the namespace if it does not exist.
+
+Applications within the automatically created child queues will be sorted based sorting policy set on the parent queue.
+In this case the property `application.sort.policy` is in this configuration set to `stateaware`.
+This is a simple app sorting policy applicable for batch jobs, you can find more document [here](sorting_policies.md#StateAwarePolicy).
+
+You can change the configuration using the helm charts during the installation by overwriting the configuration in the
+[helm chart template](https://github.com/apache/incubator-yunikorn-release/blob/master/helm-charts/yunikorn/values.yaml#L71-L81).
+
+### Namespace quota
+Namespaces in Kubernetes contain the quota information.
+If a quota is set on a namespace Kubernetes will automatically enforce the quota.
+In the case that YuniKorn is used for quota enforcement no quota must be set on the namespace.
+
+To allow specifying a quota on the namespace the following annotations should be set in the namespace object:
+```yaml
+yunikorn.apache.org/namespace.max.cpu: "64"
+yunikorn.apache.org/namespace.max.memory: "100Gi"
+```
+YuniKorn will parse these annotations and set the maximum capacity of the queue mapped to this namespace.
+The values specified follow the standard Kubernetes formatting and unit specification.
+Currently, we only support mapping memory and cpu not other resource types.
+
+The example above will limit the queue mapped to the annotated namespace to 64 CPUs and 100GB memory.
+
+### Run a workload
+
+Applications, and the pods that are part of the application, can be submitted without specific labels.
+YuniKorn will automatically add the required tags.
+The configured placement rule will create the queue, if required, and add the application to the queue.
+
+For example, if an application is submitted to namespace `development`, then the application will run in the `root.development` queue.
+
+## Parent queue mapping for namespaces
+
+### Goal
+Though the tag placement rule using the `namespace` tag is capable of placing an application in a queue this might not be enough in all setups.
+In some cases, multi tenancy for example, namespaces need to be grouped together.
+Administrators could annotate namespaces which allows dynamic placement of applications based on multiple annotations if placement rules were setup.
+YuniKorn cannot and does not just add all annotations from a namespace to an application.
+
+To help support this grouping case a parent queue can be tagged on a namespace.
+
+### Configuration
+The configuration for this functionality consists of two pieces:
+1. the mapping rule
+1. the namespace annotation
+
+First we set the following configuration to YuniKorn's configmap:
+
+```yaml
+partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ - name: tag
+ value: namespace.parentqueue
+ queues:
+ - name: root
+ queues:
+ - name: production
+ - name: development
+```
+
+The configuration used for the namespace to queue mapping is the same as [above](#Namespace-to-queue-mapping).
+As an extension to the placement rule a `parent` rule is added to support the grouping.
+The parent rule is used to generate the parent, or the queue above, in the hierarchy.
+The rule uses the tag `namespace.parentqueue` from the application to generate the parent queue name.
+The `namespace.parentqueue` tag is automatically added by the Kubernetes shim but does require a namespace annotation (see below).
+
+In the example rule configuration given the `create` flag is not set on the parent rule.
+This means that the parent queue must exist in the configuration otherwise the application submit will fail.
+For the example configuration this means supported values for the parent are thus limited to `production` and `development`.
+
+Quotas cannot be set on the parent queue using any of these mappings.
+The quota linked to the namespace is set on the namespace queue not the parent as per the namespace mapping provided earlier.
+
+Parent queue quotas must always be set directly in the configuration.
+This requires the `create` flag to be set to `false` on the parent rule.
+
+### Namespace parent queue
+Contrary to the namespace name itself, and inline with the quota settings, the namespaces need to be annotated to use the parent queue mapping.
+Namespace names must be unique in Kubernetes which is not affected by this annotation.
+The same annotation value may be used for multiple namespaces:
+```yaml
+yunikorn.apache.org/parentqueue: root.production
+```
+
+The example annotation above will map the parent queue to the existing `root.production` queue.
+Note that the rule will fully qualify the name if needed, you can thus omit the `root.` part in the annotation.
+If the annotation starts with `root.` the system assumes it is a fully qualified queue name.
+
+To complete the picture here is an image that shows the mapping from Kubernetes namespaces to queues in YuniKorn.
+It uses the annotations on the namespaces in Kubernetes as described, and the example configuration for the mapping rules.
+The `finance` and `sales` namespaces become queues grouped under the parent queue `production`.
+The namespaces `dev` and `test` are placed under the `development` parent queue.
+
+
+
+### Run a workload
+Applications, and the pods that are part of the application, can be submitted without specific labels or changes.
+YuniKorn will add the tags, the placement rules will do the rest.
+The configured placement rule will create the queues, if required, and add the application to the queue.
+
+Since the namespace `finance` is annotated with the example value, and the rules are in place.
+Applications in the `finance` namespace will run in the `root.production.finance` queue that is created dynamically.
diff --git a/versioned_docs/version-0.10.0/user_guide/sorting_policies.md b/versioned_docs/version-0.10.0/user_guide/sorting_policies.md
new file mode 100644
index 0000000..f969cc0
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/sorting_policies.md
@@ -0,0 +1,154 @@
+---
+id: sorting_policies
+title: Sorting Policies
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+The scheduler uses policies allow changing the scheduling behaviour without code changes.
+Policies can be set for:
+* [Applications](#application-sorting)
+* [Nodes](#node-sorting)
+* [Requests](#request-sorting)
+
+## Application sorting
+The application sorting policy is set for each queue via the config.
+A sorting policy setting is only effective on a `leaf` queue.
+Each `leaf` queue can use a different policy.
+
+A sorting policy only specifies the order in which the applications are sorted within a queue.
+That order is crucial in specifying which application is considered first when assigning resources.
+Sorting policies do _not_ affect the number of applications that are scheduled or active in the queue at the same time.
+All applications that have pending resource requests can and will be scheduled in a queue unless specifically filtered out.
+Even when applications are sorted using a first in first out policy multiple applications will run in a queue in parallel.
+
+A `parent` queue will always use the fair policy to sort the child queues.
+
+The following configuration entry sets the application sorting policy to `fifo` for the queue `root.sandbox`:
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: sandbox
+ properties:
+ application.sort.policy: fifo
+```
+
+The only applications that are considered during scheduling must have outstanding requests.
+A filter is applied _while_ sorting the applications to remove all that do not have outstanding requests.
+
+### FifoSortPolicy
+Short description: first in first out, based on application create time
+Config value: fifo (default)
+Behaviour:
+Before sorting the applications are filtered and must have pending resource requests.
+
+After filtering the applications left are sorted based on the application create time stamp only, no other filtering is applied.
+Since applications can only be added while the system is locked there can never be two applications with the exact same time stamp.
+
+The result is that the oldest application that requests resources gets resources.
+Younger applications will be given resources when all the current requests of older applications have been fulfilled.
+
+### FairSortPolicy
+Short description: fair based on usage
+Config value: fair
+Behaviour:
+Before sorting the applications are filtered and must have pending resource requests.
+
+After filtering the applications left are sorted based on the application usage.
+The usage of the application is defined as all confirmed and unconfirmed allocations for the applications.
+All resources defined on the application will be taken into account when calculating the usage.
+
+The result is that the resources available are spread equally over all applications that request resources.
+
+### StateAwarePolicy
+Short description: limit of one (1) application in Starting or Accepted state
+Config value: stateaware
+Behaviour:
+This sorting policy requires an understanding of the application states.
+Applications states are described in the [application states](design/scheduler_object_states.md#application-state) documentation.
+
+Before sorting applications the following filters are applied to all applications in the queue:
+The first filter is based on the application state.
+The following applications pass through the filter and generate the first intermediate list:
+* all applications in the state _running_
+* _one_ (1) application in the _starting_ state
+* if there are _no_ applications in the _starting_ state _one_ (1) application in the _accepted_ state is added
+
+The second filter takes the result of the first filter as an input.
+The preliminary list is filtered again: all applications _without_ a pending request are removed.
+
+After filtering based on status and pending requests the applications that remain are sorted.
+The final list is thus filtered twice with the remaining applications sorted on create time.
+
+To recap the _staring_ and _accepted_ state interactions:
+The application in the _accepted_ state is only added if there is no application in the _starting_ state.
+The application in the _starting_ state does not have to have pending requests.
+Any application in the _starting_ state will prevent _accepted_ applications from being added to the filtered list.
+
+For further details see the [Example run](design/state_aware_scheduling.md#example-run) in the design document.
+
+The result is that already running applications that request resources will get resources first.
+A drip feed of one new applications is added to the list of running applications to be allocated after all running applications.
+
+## Node sorting
+The node sorting policy is set for a partition via the config.
+Each partition can use a different policy.
+
+The following configuration entry sets the node sorting policy to `fair` for the partition `default`:
+```yaml
+partitions:
+ - name: default
+ nodesortpolicy:
+ type: fair
+```
+
+### FairnessPolicy
+Short description: available resource, descending order
+Config value: fair (default)
+Behaviour:
+Sort the list of nodes by the amount of available resources so that the node with the _highest_ amount of available resource is the first in the list.
+All resources defined on a node will be taken into account when calculating the usage.
+Resources of the same type are compared for the nodes.
+
+This results in a node with the lowest utilisation to be considered first for assigning new allocation.
+Resulting in a spread of allocations over all available nodes.
+Leading to an overall lower utilisation of the individual available nodes, unless the whole environment is highly utilised.
+Keeping the load on all nodes at a similar level does help
+In an environment that auto scales by adding new nodes this could trigger unexpected auto scale requests.
+
+### BinPackingPolicy
+Short description: available resource, ascending order
+Config value: binpacking
+Behaviour:
+Sort the list of nodes by the amount of available resources so that the node with the _lowest_ amount of available resource is the first in the list.
+All resources defined on a node will be taken into account when calculating the usage.
+Resources of the same type are compared for the nodes.
+
+This results in a node with the highest utilisation to be considered first for assigning new allocation.
+Resulting in a high(er) utilisation of a small(er) number of nodes, better suited for cloud deployments.
+
+## Request sorting
+There is currently one policy for sorting requests within an application.
+This policy is not configurable.
+Sorting requests is only possible based on the priority of the request.
+If there are multiple requests within an application that have the same priority the order of the requests is undetermined.
+This means that the order of requests with the same priority can, and most likely will, change between runs.
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/user_guide/trouble_shooting.md b/versioned_docs/version-0.10.0/user_guide/trouble_shooting.md
new file mode 100644
index 0000000..549d5e0
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/trouble_shooting.md
@@ -0,0 +1,192 @@
+---
+id: trouble_shooting
+title: Trouble Shooting
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## Scheduler logs
+
+### Retrieve scheduler logs
+
+Currently, the scheduler writes its logs to stdout/stderr, docker container handles the redirection of these logs to a
+local location on the underneath node, you can read more document [here](https://docs.docker.com/config/containers/logging/configure/).
+These logs can be retrieved by [kubectl logs](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#logs). Such as:
+
+```shell script
+// get the scheduler pod
+kubectl get pod -l component=yunikorn-scheduler -n yunikorn
+NAME READY STATUS RESTARTS AGE
+yunikorn-scheduler-766d7d6cdd-44b82 2/2 Running 0 33h
+
+// retrieve logs
+kubectl logs yunikorn-scheduler-766d7d6cdd-44b82 yunikorn-scheduler-k8s -n yunikorn
+```
+
+In most cases, this command cannot get all logs because the scheduler is rolling logs very fast. To retrieve more logs in
+the past, you will need to setup the [cluster level logging](https://kubernetes.io/docs/concepts/cluster-administration/logging/#cluster-level-logging-architectures).
+The recommended setup is to leverage [fluentd](https://www.fluentd.org/) to collect and persistent logs on an external storage, e.g s3.
+
+### Set Logging Level
+
+:::note
+Changing the logging level requires a restart of the scheduler pod.
+:::
+
+Stop the scheduler:
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=0
+```
+edit the deployment config in vim:
+
+```shell script
+kubectl edit deployment yunikorn-scheduler -n yunikorn
+```
+
+add `LOG_LEVEL` to the `env` field of the container template. For example setting `LOG_LEVEL` to `0` sets the logging
+level to `INFO`.
+
+```yaml
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ ...
+spec:
+ template:
+ ...
+ spec:
+ containers:
+ - env:
+ - name: LOG_LEVEL
+ value: '0'
+```
+
+Start the scheduler:
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=1
+```
+
+Available logging levels:
+
+| Value | Logging Level |
+|:-----: |:-------------: |
+| -1 | DEBUG |
+| 0 | INFO |
+| 1 | WARN |
+| 2 | ERROR |
+| 3 | DPanic |
+| 4 | Panic |
+| 5 | Fatal |
+
+## Pods are stuck at Pending state
+
+If some pods are stuck at Pending state, that means the scheduler could not find a node to allocate the pod. There are
+several possibilities to cause this:
+
+### 1. Non of the nodes satisfy pod placement requirement
+
+A pod can be configured with some placement constraints, such as [node-selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector),
+[affinity/anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity),
+do not have certain toleration for node [taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/), etc.
+To debug such issues, you can describe the pod by:
+
+```shell script
+kubectl describe pod <pod-name> -n <namespace>
+```
+
+the pod events will contain the predicate failures and that explains why nodes are not qualified for allocation.
+
+### 2. The queue is running out of capacity
+
+If the queue is running out of capacity, pods will be pending for available queue resources. To check if a queue is still
+having enough capacity for the pending pods, there are several approaches:
+
+1) check the queue usage from yunikorn UI
+
+If you do not know how to access the UI, you can refer the document [here](../get_started/get_started.md#access-the-web-ui). Go
+to the `Queues` page, navigate to the queue where this job is submitted to. You will be able to see the available capacity
+left for the queue.
+
+2) check the pod events
+
+Run the `kubectl describe pod` to get the pod events. If you see some event like:
+`Application <appID> does not fit into <queuePath> queue`. That means the pod could not get allocated because the queue
+is running out of capacity.
+
+The pod will be allocated if some other pods in this queue is completed or removed. If the pod remains pending even
+the queue has capacity, that may because it is waiting for the cluster to scale up.
+
+## Restart the scheduler
+
+YuniKorn can recover its state upon a restart. YuniKorn scheduler pod is deployed as a deployment, restart the scheduler
+can be done by scale down and up the replica:
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=0
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=1
+```
+
+## Gang Scheduling
+
+### 1. No placeholders created, app's pods are pending
+
+*Reason*: This is usually because the app is rejected by the scheduler, therefore non of the pods are scheduled.
+The common reasons caused the rejection are: 1) The taskGroups definition is invalid. The scheduler does the
+sanity check upon app submission, to ensure all the taskGroups are defined correctly, if these info are malformed,
+the scheduler rejects the app; 2) The total min resources defined in the taskGroups is bigger than the queues' max
+capacity, scheduler rejects the app because it won't fit into the queue's capacity. Check the pod event for relevant messages,
+and you will also be able to find more detail error messages from the schedulers' log.
+
+*Solution*: Correct the taskGroups definition and retry submitting the app.
+
+### 2. Not all placeholders can be allocated
+
+*Reason*: The placeholders also consume resources, if not all of them can be allocated, that usually means either the queue
+or the cluster has no sufficient resources for them. In this case, the placeholders will be cleaned up after a certain
+amount of time, defined by the `placeholderTimeoutInSeconds` scheduling policy parameter.
+
+*Solution*: Note, if the placeholder timeout reaches, currently the app will transit to failed state and can not be scheduled
+anymore. You can increase the placeholder timeout value if you are willing to wait for a longer time. In the future, a fallback policy
+might be added to provide some retry other than failing the app.
+
+### 3. Not all placeholders are swapped
+
+*Reason*: This usually means the actual app's pods are less than the minMembers defined in the taskGroups.
+
+*Solution*: Check the `minMember` in the taskGroup field and ensure it is correctly set. The `minMember` can be less than
+the actual pods, setting it to bigger than the actual number of pods is invalid.
+
+### 4.Placeholders are not cleaned up when the app terminated
+
+*Reason*: All the placeholders are set an [ownerReference](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#owners-and-dependents)
+to the first real pod of the app, or the controller reference. If the placeholder could not be cleaned up, that means
+the garbage collection is not working properly.
+
+*Solution*: check the placeholder `ownerReference` and the garbage collector in Kubernetes.
+
+
+## Still got questions?
+
+No problem! The Apache YuniKorn community will be happy to help. You can reach out to the community with the following options:
+
+1. Post your questions to dev@yunikorn.apache.org
+2. Join the [YuniKorn slack channel](https://join.slack.com/t/yunikornworkspace/shared_invite/enQtNzAzMjY0OTI4MjYzLTBmMDdkYTAwNDMwNTE3NWVjZWE1OTczMWE4NDI2Yzg3MmEyZjUyYTZlMDE5M2U4ZjZhNmYyNGFmYjY4ZGYyMGE) and post your questions to the `#yunikorn-user` channel.
+3. Join the [community sync up meetings](http://yunikorn.apache.org/community/getInvolved#community-meetings) and directly talk to the community members.
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/user_guide/workloads/run_flink.md b/versioned_docs/version-0.10.0/user_guide/workloads/run_flink.md
new file mode 100644
index 0000000..d20e815
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/workloads/run_flink.md
@@ -0,0 +1,66 @@
+---
+id: run_flink
+title: Run Flink Jobs
+description: How to run Flink jobs with YuniKorn
+image: https://svn.apache.org/repos/asf/flink/site/img/logo/png/100/flink_squirrel_100_color.png
+keywords:
+ - spark
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+It's very easy to run [Apache Flink](https://flink.apache.org/) on Kubernetes with YuniKorn. Depending on which mode is
+used to run Flink on Kubernetes, the configuration is slight different.
+
+## Standalone mode
+
+Please follow [Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html) to get details and examples of standalone deploy mode.
+In this mode, we can directly add required labels (applicationId and queue) in Deployment/Job spec to run flink application with YuniKorn scheduler, as well as [Run workloads with YuniKorn Scheduler](#run-workloads-with-yunikorn-scheduler).
+
+## Native mode
+
+Please follow [Native Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/native_kubernetes.html) to get details and examples of native deploy mode.
+Running flink application with YuniKorn scheduler in native mode is only supported for flink 1.11 or above, we can leverage two flink configurations `kubernetes.jobmanager.labels` and `kubernetes.taskmanager.labels` to set the required labels.
+Examples:
+
+* Start a flink session
+```
+./bin/kubernetes-session.sh \
+ -Dkubernetes.cluster-id=<ClusterId> \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dresourcemanager.taskmanager-timeout=3600000 \
+ -Dkubernetes.jobmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ -Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox
+```
+
+* Start a flink application
+```
+./bin/flink run-application -p 8 -t kubernetes-application \
+ -Dkubernetes.cluster-id=<ClusterId> \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dkubernetes.container.image=<CustomImageName> \
+ -Dkubernetes.jobmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ -Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ local:///opt/flink/usrlib/my-flink-job.jar
+```
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/user_guide/workloads/run_spark.md b/versioned_docs/version-0.10.0/user_guide/workloads/run_spark.md
new file mode 100644
index 0000000..cee95df
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/workloads/run_spark.md
@@ -0,0 +1,145 @@
+---
+id: run_spark
+title: Run Spark Jobs
+description: How to run Spark jobs with YuniKorn
+keywords:
+ - spark
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+:::note
+This document assumes you have YuniKorn and its admission-controller both installed. Please refer to
+[get started](../../get_started/get_started.md) to see how that is done.
+:::
+
+## Prepare the docker image for Spark
+
+To run Spark on Kubernetes, you'll need the Spark docker images. You can 1) use the docker images provided by the YuniKorn
+team, or 2) build one from scratch. If you want to build your own Spark docker image, you can
+* Download a Spark version that has Kubernetes support, URL: https://github.com/apache/spark
+* Build spark with Kubernetes support:
+```shell script
+mvn -Pyarn -Phadoop-2.7 -Dhadoop.version=2.7.4 -Phive -Pkubernetes -Phive-thriftserver -DskipTests package
+```
+
+## Create a namespace for Spark jobs
+
+Create a namespace:
+
+```shell script
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: spark-test
+EOF
+```
+
+Create service account and cluster role bindings under `spark-test` namespace:
+
+```shell script
+cat <<EOF | kubectl apply -n spark-test -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: spark
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: spark-cluster-role
+rules:
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get", "watch", "list", "create", "delete"]
+- apiGroups: [""]
+ resources: ["configmaps"]
+ verbs: ["get", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: spark-cluster-role-binding
+subjects:
+- kind: ServiceAccount
+ name: spark
+roleRef:
+ kind: ClusterRole
+ name: spark-cluster-role
+ apiGroup: rbac.authorization.k8s.io
+EOF
+```
+
+:::note
+Do NOT use `ClusterRole` and `ClusterRoleBinding` to run Spark jobs in production, please configure a more fine-grained
+security context for running Spark jobs. See more about how to configure proper RBAC rules [here](https://kubernetes.io/docs/reference/access-authn-authz/rbac/).
+:::
+
+## Submit a Spark job
+
+If this is running from local machine, you will need to start the proxy in order to talk to the api-server.
+```shell script
+kubectl proxy
+```
+
+Run a simple SparkPi job (this assumes that the Spark binaries are installed to `/usr/local` directory).
+```shell script
+export SPARK_HOME=/usr/local/spark-2.4.4-bin-hadoop2.7/
+${SPARK_HOME}/bin/spark-submit --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --class org.apache.spark.examples.SparkPi \
+ --conf spark.executor.instances=1 \
+ --conf spark.kubernetes.namespace=spark-test \
+ --conf spark.kubernetes.executor.request.cores=1 \
+ --conf spark.kubernetes.container.image=apache/yunikorn:spark-2.4.4 \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+ local:///opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar
+```
+
+You'll see Spark driver and executors been created on Kubernetes:
+
+
+
+You can also view the job info from YuniKorn UI. If you do not know how to access the YuniKorn UI, please read the document
+[here](../../get_started/get_started.md#access-the-web-ui).
+
+
+
+## What happens behind the scenes?
+
+When the Spark job is submitted to the cluster, the job is submitted to `spark-test` namespace. The Spark driver pod will
+be firstly created under this namespace. Since this cluster has YuniKorn admission-controller enabled, when the driver pod
+get created, the admission-controller mutates the pod's spec and injects `schedulerName=yunikorn`, by doing this, the
+default K8s scheduler will skip this pod and it will be scheduled by YuniKorn instead. See how this is done by [configuring
+another scheduler in Kubernetes](https://kubernetes.io/docs/tasks/extend-kubernetes/configure-multiple-schedulers/).
+
+The default configuration has placement rule enabled, which automatically maps the `spark-test` namespace to a YuniKorn
+queue `root.spark-test`. All Spark jobs submitted to this namespace will be automatically submitted to the queue first.
+To see more about how placement rule works, please see doc [placement-rules](user_guide/placement_rules.md). By far,
+the namespace defines the security context of the pods, and the queue determines how the job and pods will be scheduled
+with considering of job ordering, queue resource fairness, etc. Note, this is the simplest setup, which doesn't enforce
+the queue capacities. The queue is considered as having unlimited capacity.
+
+YuniKorn reuses the Spark application ID set in label `spark-app-selector`, and this job is submitted
+to YuniKorn and being considered as a job. The job is scheduled and running as there is sufficient resources in the cluster.
+YuniKorn allocates the driver pod to a node, binds the pod and starts all the containers. Once the driver pod gets started,
+it requests for a bunch of executor pods to run its tasks. Those pods will be created in the same namespace as well and
+scheduled by YuniKorn as well.
\ No newline at end of file
diff --git a/versioned_docs/version-0.10.0/user_guide/workloads/run_tensorflow.md b/versioned_docs/version-0.10.0/user_guide/workloads/run_tensorflow.md
new file mode 100644
index 0000000..393e330
--- /dev/null
+++ b/versioned_docs/version-0.10.0/user_guide/workloads/run_tensorflow.md
@@ -0,0 +1,40 @@
+---
+id: run_tf
+title: Run Tensorflow Jobs
+keywords:
+ - tensorflow
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Here is an example for Tensorflow job. You must install tf-operator first.
+You can install tf-operator by applying all yaml from two website down below:
+* CRD: https://github.com/kubeflow/manifests/tree/master/tf-training/tf-job-crds/base
+* Deployment: https://github.com/kubeflow/manifests/tree/master/tf-training/tf-job-operator/base
+Also you can install kubeflow which can auto install tf-operator for you, URL: https://www.kubeflow.org/docs/started/getting-started/
+
+A simple Tensorflow job example:
+
+You need to [build the image](https://github.com/kubeflow/tf-operator/tree/master/examples/v1/dist-mnist) which used in example yaml.
+```
+kubectl create -f examples/tfjob/tf-job-mnist.yaml
+```
+
+The file for this example can be found in the [README Tensorflow job](https://github.com/apache/incubator-yunikorn-k8shim/tree/master/deployments/examples#Tensorflow-job) section.
diff --git a/versioned_sidebars/version-0.10.0-sidebars.json b/versioned_sidebars/version-0.10.0-sidebars.json
new file mode 100644
index 0000000..91b9d51
--- /dev/null
+++ b/versioned_sidebars/version-0.10.0-sidebars.json
@@ -0,0 +1,194 @@
+{
+ "version-0.10.0/docs": [
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "Get Started",
+ "items": [
+ {
+ "type": "link",
+ "label": "Release Notes",
+ "href": "/release-announce/0.10.0/"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/get_started/user_guide"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/get_started/core_features"
+ }
+ ]
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "User Guide",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/queue_config"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/placement_rules"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/sorting_policies"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/acls"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/resource_quota_management"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/gang_scheduling"
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "Workloads",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/workloads/run_spark"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/workloads/run_flink"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/workloads/run_tf"
+ }
+ ]
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "REST APIs",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.10.0/api/cluster"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/api/scheduler"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/api/system"
+ }
+ ]
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/user_guide/trouble_shooting"
+ }
+ ]
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "Developer Guide",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.10.0/developer_guide/env_setup"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/developer_guide/build"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/developer_guide/deployment"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/developer_guide/openshift_development"
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "Designs",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/architecture"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/scheduler_core_design"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/cache_removal"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/k8shim"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/cross_queue_preemption"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/namespace_resource_quota"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/pluggable_app_management"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/resilience"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/predicates"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/scheduler_configuration"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/state_aware_scheduling"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/design/scheduler_object_states"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "collapsed": true,
+ "type": "category",
+ "label": "Performance",
+ "items": [
+ {
+ "type": "doc",
+ "id": "version-0.10.0/performance/evaluate_perf_function_with_kubemark"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/performance/metrics"
+ },
+ {
+ "type": "doc",
+ "id": "version-0.10.0/performance/profiling"
+ }
+ ]
+ }
+ ]
+}
diff --git a/versions.json b/versions.json
index 86c73ac..9383b87 100644
--- a/versions.json
+++ b/versions.json
@@ -1,4 +1,5 @@
[
+ "0.10.0",
"0.9.0",
"0.8.0"
]