You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/05/13 17:36:32 UTC
(spark-kubernetes-operator) branch gh-pages created (now b6f5f05)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a change to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
at b6f5f05 Setup GitHub Pages
This branch includes the following new commits:
new b6f5f05 Setup GitHub Pages
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
(spark-kubernetes-operator) 01/01: Setup GitHub Pages
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
commit b6f5f0570680aaf2eb4a6911ec831713413c05ac
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Tue May 13 10:35:40 2025 -0700
Setup GitHub Pages
---
.asf.yaml | 35 -
.github/.licenserc.yaml | 24 -
.github/PULL_REQUEST_TEMPLATE | 53 --
.github/labeler.yml | 67 --
.github/workflows/build_and_test.yml | 166 -----
.github/workflows/labeler.yml | 40 --
.github/workflows/publish_snapshot_chart.yml | 61 --
.github/workflows/publish_snapshot_dockerhub.yml | 48 --
.gitignore | 14 -
LICENSE | 211 ------
LICENSE-binary | 211 ------
NOTICE | 6 -
NOTICE-binary | 6 -
README.md | 109 +---
build-tools/docker/Dockerfile | 52 --
build-tools/docker/docker-entrypoint.sh | 35 -
build-tools/docs-utils/build.gradle | 42 --
.../k8s/operator/utils/ConfOptionDocGenerator.java | 87 ---
.../apache/spark/k8s/operator/utils/DocTable.java | 69 --
.../helm/spark-kubernetes-operator/.helmignore | 12 -
.../helm/spark-kubernetes-operator/Chart.yaml | 21 -
.../templates/_helpers.tpl | 146 -----
.../templates/operator-rbac.yaml | 166 -----
.../templates/spark-operator.yaml | 205 ------
.../templates/tests/test-rbac.yaml | 64 --
.../templates/workload-rbac.yaml | 180 ------
.../helm/spark-kubernetes-operator/values.yaml | 194 ------
build.gradle | 124 ----
config/checkstyle/checkstyle.xml | 209 ------
config/pmd/ruleset.xml | 81 ---
config/spotbugs/spotbugs_exclude.xml | 25 -
deploy.gradle | 128 ----
dev/.rat-excludes | 18 -
dev/check-license | 86 ---
dev/merge_spark_pr.py | 719 ---------------------
docs/architecture.md | 81 ---
docs/config_properties.md | 40 --
docs/configuration.md | 144 -----
docs/operations.md | 151 -----
docs/resources/application_state_machine.png | Bin 82299 -> 0 bytes
docs/resources/cluster_state_machine.png | Bin 15835 -> 0 bytes
docs/resources/prometheus.png | Bin 184821 -> 0 bytes
docs/spark_custom_resources.md | 322 ---------
examples/cluster-java21.yaml | 32 -
examples/cluster-on-yunikorn.yaml | 33 -
examples/cluster-with-hpa-template.yaml | 65 --
examples/cluster-with-hpa.yaml | 45 --
examples/cluster-with-template.yaml | 100 ---
examples/pi-java21.yaml | 32 -
examples/pi-on-yunikorn.yaml | 38 --
examples/pi-scala.yaml | 32 -
examples/pi-with-one-pod.yaml | 29 -
examples/pi-with-spark-connect-plugin.yaml | 34 -
examples/pi.yaml | 32 -
examples/prod-cluster-with-three-workers.yaml | 32 -
examples/pyspark-pi.yaml | 30 -
examples/qa-cluster-with-one-worker.yaml | 32 -
.../spark-connect-server-with-spark-cluster.yaml | 30 -
examples/sql.yaml | 30 -
examples/submit-pi-to-prod.sh | 40 --
examples/submit-spark-connect-server.sh | 39 --
gradle.properties | 19 -
gradle/libs.versions.toml | 70 --
gradle/wrapper/gradle-wrapper.properties | 25 -
gradlew | 259 --------
index.md | 87 +++
index.yaml | 15 +
settings.gradle | 24 -
spark-operator-api/build.gradle | 59 --
.../apache/spark/k8s/operator/BaseResource.java | 36 --
.../org/apache/spark/k8s/operator/Constants.java | 96 ---
.../spark/k8s/operator/SparkApplication.java | 57 --
.../spark/k8s/operator/SparkApplicationList.java | 26 -
.../apache/spark/k8s/operator/SparkCluster.java | 53 --
.../spark/k8s/operator/SparkClusterList.java | 26 -
.../k8s/operator/decorators/ResourceDecorator.java | 26 -
.../apache/spark/k8s/operator/diff/Diffable.java | 22 -
.../spark/k8s/operator/spec/ApplicationSpec.java | 59 --
.../operator/spec/ApplicationTimeoutConfig.java | 66 --
.../k8s/operator/spec/ApplicationTolerations.java | 45 --
.../operator/spec/BaseApplicationTemplateSpec.java | 38 --
.../apache/spark/k8s/operator/spec/BaseSpec.java | 36 --
.../spark/k8s/operator/spec/ClusterSpec.java | 46 --
.../k8s/operator/spec/ClusterTolerations.java | 37 --
.../spark/k8s/operator/spec/ConfigMapSpec.java | 41 --
.../spark/k8s/operator/spec/DeploymentMode.java | 25 -
.../operator/spec/DriverServiceIngressSpec.java | 41 --
.../k8s/operator/spec/ExecutorInstanceConfig.java | 69 --
.../apache/spark/k8s/operator/spec/MasterSpec.java | 41 --
.../k8s/operator/spec/ResourceRetainPolicy.java | 39 --
.../spark/k8s/operator/spec/RestartConfig.java | 39 --
.../spark/k8s/operator/spec/RestartPolicy.java | 39 --
.../spark/k8s/operator/spec/RuntimeVersions.java | 40 --
.../k8s/operator/spec/WorkerInstanceConfig.java | 37 --
.../apache/spark/k8s/operator/spec/WorkerSpec.java | 43 --
.../operator/status/ApplicationAttemptSummary.java | 53 --
.../k8s/operator/status/ApplicationState.java | 50 --
.../operator/status/ApplicationStateSummary.java | 151 -----
.../k8s/operator/status/ApplicationStatus.java | 169 -----
.../spark/k8s/operator/status/AttemptInfo.java | 44 --
.../k8s/operator/status/BaseAttemptSummary.java | 37 --
.../spark/k8s/operator/status/BaseState.java | 37 --
.../k8s/operator/status/BaseStateSummary.java | 29 -
.../spark/k8s/operator/status/BaseStatus.java | 64 --
.../k8s/operator/status/ClusterAttemptSummary.java | 51 --
.../spark/k8s/operator/status/ClusterState.java | 44 --
.../k8s/operator/status/ClusterStateSummary.java | 64 --
.../spark/k8s/operator/status/ClusterStatus.java | 63 --
.../spark/k8s/operator/utils/ModelUtils.java | 122 ----
.../src/main/resources/printer-columns.sh | 27 -
.../k8s/operator/spec/ApplicationSpecTest.java | 49 --
.../spark/k8s/operator/spec/ClusterSpecTest.java | 44 --
.../spark/k8s/operator/spec/RestartPolicyTest.java | 88 ---
.../k8s/operator/status/ApplicationStatusTest.java | 173 -----
.../k8s/operator/status/ClusterStatusTest.java | 44 --
.../spark/k8s/operator/utils/ModelUtilsTest.java | 127 ----
spark-operator/build.gradle | 108 ----
.../apache/spark/k8s/operator/SparkOperator.java | 239 -------
.../operator/client/KubernetesClientFactory.java | 54 --
.../spark/k8s/operator/config/ConfigOption.java | 105 ---
.../k8s/operator/config/SparkOperatorConf.java | 479 --------------
.../operator/config/SparkOperatorConfManager.java | 102 ---
.../config/SparkOperatorConfigMapReconciler.java | 82 ---
.../spark/k8s/operator/context/BaseContext.java | 30 -
.../k8s/operator/context/SparkAppContext.java | 112 ----
.../k8s/operator/context/SparkClusterContext.java | 88 ---
.../k8s/operator/decorators/ClusterDecorator.java | 50 --
.../k8s/operator/decorators/DriverDecorator.java | 53 --
.../decorators/DriverResourceDecorator.java | 73 ---
.../k8s/operator/listeners/BaseStatusListener.java | 28 -
.../operator/listeners/SparkAppStatusListener.java | 26 -
.../listeners/SparkClusterStatusListener.java | 26 -
.../spark/k8s/operator/metrics/JVMMetricSet.java | 74 ---
.../spark/k8s/operator/metrics/MetricsService.java | 57 --
.../spark/k8s/operator/metrics/MetricsSystem.java | 146 -----
.../k8s/operator/metrics/MetricsSystemFactory.java | 107 ---
.../metrics/PrometheusPullModelHandler.java | 85 ---
.../metrics/healthcheck/SentinelManager.java | 216 -------
.../source/KubernetesMetricsInterceptor.java | 172 -----
.../metrics/source/OperatorJosdkMetrics.java | 312 ---------
.../operator/metrics/source/OperatorJvmSource.java | 39 --
.../spark/k8s/operator/probe/HealthProbe.java | 116 ----
.../spark/k8s/operator/probe/ProbeService.java | 63 --
.../spark/k8s/operator/probe/ReadinessProbe.java | 61 --
.../k8s/operator/reconciler/ReconcileProgress.java | 69 --
.../operator/reconciler/SparkAppReconciler.java | 225 -------
.../reconciler/SparkAppResourceSpecFactory.java | 178 -----
.../reconciler/SparkClusterReconciler.java | 181 ------
.../SparkClusterResourceSpecFactory.java | 51 --
.../observers/AppDriverReadyObserver.java | 47 --
.../observers/AppDriverRunningObserver.java | 41 --
.../observers/AppDriverStartObserver.java | 47 --
.../observers/AppDriverTimeoutObserver.java | 89 ---
.../observers/BaseAppDriverObserver.java | 144 -----
.../observers/BaseSecondaryResourceObserver.java | 43 --
.../reconciler/reconcilesteps/AppCleanUpStep.java | 164 -----
.../reconciler/reconcilesteps/AppInitStep.java | 136 ----
.../reconcilesteps/AppReconcileStep.java | 97 ---
.../reconcilesteps/AppResourceObserveStep.java | 42 --
.../reconciler/reconcilesteps/AppRunningStep.java | 92 ---
.../reconcilesteps/AppTerminatedStep.java | 44 --
.../reconcilesteps/AppUnknownStateStep.java | 45 --
.../reconciler/reconcilesteps/AppValidateStep.java | 55 --
.../reconciler/reconcilesteps/ClusterInitStep.java | 103 ---
.../reconcilesteps/ClusterReconcileStep.java | 59 --
.../reconcilesteps/ClusterTerminatedStep.java | 44 --
.../reconcilesteps/ClusterUnknownStateStep.java | 39 --
.../reconcilesteps/ClusterValidateStep.java | 38 --
.../k8s/operator/utils/ClassLoadingUtils.java | 71 --
.../spark/k8s/operator/utils/LoggingUtils.java | 64 --
.../apache/spark/k8s/operator/utils/PodPhase.java | 58 --
.../apache/spark/k8s/operator/utils/PodUtils.java | 100 ---
.../apache/spark/k8s/operator/utils/ProbeUtil.java | 95 ---
.../spark/k8s/operator/utils/ReconcilerUtils.java | 173 -----
.../k8s/operator/utils/SparkAppStatusRecorder.java | 40 --
.../k8s/operator/utils/SparkAppStatusUtils.java | 75 ---
.../operator/utils/SparkClusterStatusRecorder.java | 40 --
.../k8s/operator/utils/SparkExceptionUtils.java | 43 --
.../spark/k8s/operator/utils/StatusRecorder.java | 149 -----
.../org/apache/spark/k8s/operator/utils/Utils.java | 147 -----
spark-operator/src/main/resources/EcsLayout.json | 49 --
.../spark/k8s/operator/SparkOperatorTest.java | 196 ------
.../k8s/operator/config/ConfigOptionTest.java | 198 ------
.../config/SparkOperatorConfManagerTest.java | 76 ---
.../operator/metrics/MetricsSystemFactoryTest.java | 52 --
.../k8s/operator/metrics/MetricsSystemTest.java | 80 ---
.../metrics/healthcheck/SentinelManagerTest.java | 218 -------
.../spark/k8s/operator/metrics/sink/MockSink.java | 69 --
.../source/KubernetesMetricsInterceptorTest.java | 156 -----
.../metrics/source/OperatorJosdkMetricsTest.java | 200 ------
.../spark/k8s/operator/probe/HealthProbeTest.java | 205 ------
.../spark/k8s/operator/probe/ProbeServiceTest.java | 135 ----
.../k8s/operator/probe/ReadinessProbeTest.java | 70 --
.../reconciler/SparkAppReconcilerTest.java | 118 ----
.../SparkAppResourceSpecFactoryTest.java | 73 ---
.../SparkClusterResourceSpecFactoryTest.java | 77 ---
.../reconcilesteps/AppCleanUpStepTest.java | 44 --
.../apache/spark/k8s/operator/utils/TestUtils.java | 72 ---
spark-operator/src/test/resources/EcsLayout.json | 49 --
.../src/test/resources/log4j2.properties | 52 --
.../src/test/resources/spark-operator.properties | 18 -
spark-submission-worker/build.gradle | 44 --
.../spark/k8s/operator/SparkAppDriverConf.java | 73 ---
.../spark/k8s/operator/SparkAppResourceSpec.java | 152 -----
.../k8s/operator/SparkAppSubmissionWorker.java | 186 ------
.../k8s/operator/SparkClusterResourceSpec.java | 356 ----------
.../k8s/operator/SparkClusterSubmissionWorker.java | 50 --
.../k8s/operator/utils/ConfigMapSpecUtils.java | 51 --
.../operator/utils/DriverServiceIngressUtils.java | 95 ---
.../spark/k8s/operator/SparkAppDriverConfTest.java | 75 ---
.../k8s/operator/SparkAppResourceSpecTest.java | 139 ----
.../k8s/operator/SparkAppSubmissionWorkerTest.java | 221 -------
.../k8s/operator/SparkClusterResourceSpecTest.java | 310 ---------
.../operator/SparkClusterSubmissionWorkerTest.java | 73 ---
.../k8s/operator/utils/ConfigMapSpecUtilsTest.java | 70 --
.../utils/DriverServiceIngressUtilsTest.java | 157 -----
.../spark-application/spark-state-transition.yaml | 32 -
.../spark-cluster-state-transition.yaml | 27 -
tests/e2e/assertions/spark-driver-config-map.yaml | 21 -
tests/e2e/assertions/spark-driver-pod.yaml | 30 -
tests/e2e/assertions/spark-exec-pod.yaml | 24 -
tests/e2e/helm/dynamic-config-values-2.yaml | 39 --
tests/e2e/helm/dynamic-config-values.yaml | 32 -
tests/e2e/python/chainsaw-test.yaml | 69 --
tests/e2e/python/pyspark-example.yaml | 32 -
tests/e2e/spark-versions/chainsaw-test.yaml | 107 ---
tests/e2e/spark-versions/spark-example.yaml | 34 -
tests/e2e/state-transition/chainsaw-test.yaml | 85 ---
.../spark-cluster-example-succeeded.yaml | 33 -
.../state-transition/spark-example-succeeded.yaml | 32 -
tests/e2e/watched-namespaces/chainsaw-test.yaml | 122 ----
tests/e2e/watched-namespaces/spark-example.yaml | 34 -
.../spark-operator-dynamic-config-1.yaml | 34 -
.../spark-operator-dynamic-config-2.yaml | 34 -
234 files changed, 107 insertions(+), 19572 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
deleted file mode 100644
index c1409a7..0000000
--- a/.asf.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
----
-github:
- description: "Apache Spark Kubernetes Operator"
- homepage: https://spark.apache.org/
- labels:
- - java
- - spark
- - kubernetes
- enabled_merge_buttons:
- merge: false
- squash: true
- rebase: true
- autolink_jira: SPARK
-
-notifications:
- pullrequests: reviews@spark.apache.org
- issues: reviews@spark.apache.org
- commits: commits@spark.apache.org
- jira_options: link label
diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml
deleted file mode 100644
index 313fb63..0000000
--- a/.github/.licenserc.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-header:
- license:
- spdx-id: Apache-2.0
- copyright-owner: Apache Software Foundation
-
- paths:
- - '**'
-
- paths-ignore:
- - '**/*.md'
- - '**/.gitignore'
- - '.github/**'
- - 'dev/**'
- - 'LICENSE'
- - 'LICENSE-binary'
- - 'NOTICE'
- - 'NOTICE-binary'
- - '.asf.yaml'
- - gradlew
- - 'build/**'
- - '**/*.json'
- - '**/.helmignore'
-
- comment: on-failure
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
deleted file mode 100644
index 885d307..0000000
--- a/.github/PULL_REQUEST_TEMPLATE
+++ /dev/null
@@ -1,53 +0,0 @@
-<!--
-Thanks for sending a pull request! Here are some tips for you:
- 1. If this is your first time, please read our contributor guidelines: https://spark.apache.org/contributing.html
- 2. Ensure you have added or run the appropriate tests for your PR: https://spark.apache.org/developer-tools.html
- 3. If the PR is unfinished, use `Draft` feature of GitHub Action.
- 4. Be sure to keep the PR description updated to reflect all changes.
- 5. Please write your PR title to summarize what this PR proposes.
- 6. If possible, provide a concise example to reproduce the issue for a faster review.
--->
-
-### What changes were proposed in this pull request?
-<!--
-Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
-If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
- 1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
- 2. If there is design documentation, please add the link.
- 3. If there is a discussion in the mailing list, please add the link.
--->
-
-
-### Why are the changes needed?
-<!--
-Please clarify why the changes are needed. For instance,
- 1. If you propose a new API, clarify the use case for a new API.
- 2. If you fix a bug, you can clarify why it is a bug.
--->
-
-
-### Does this PR introduce _any_ user-facing change?
-<!--
-Note that it means *any* user-facing change including all aspects such as the documentation fix.
-If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
-If possible, please also clarify if this is a user-facing change compared to the released Spark versions or within the unreleased branches such as master.
-If no, write 'No'.
--->
-
-
-### How was this patch tested?
-<!--
-If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
-If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
-If tests were not added, please describe why they were not added and/or why it was difficult to add.
-If benchmark tests were added, please run the benchmarks in GitHub Actions for the consistent environment, and the instructions could accord to: https://spark.apache.org/developer-tools.html#github-workflow-benchmarks.
--->
-
-
-### Was this patch authored or co-authored using generative AI tooling?
-<!--
-If generative AI tooling has been used in the process of authoring this patch, please include the
-phrase: 'Generated-by: ' followed by the name of the tool and its version.
-If no, write 'No'.
-Please refer to the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for details.
--->
diff --git a/.github/labeler.yml b/.github/labeler.yml
deleted file mode 100644
index aaebcac..0000000
--- a/.github/labeler.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-INFRA:
- - changed-files:
- - any-glob-to-any-file: [
- '.asf.yaml',
- '.gitignore',
- '.github/**/*',
- 'dev/merge_spark_pr.py',
- 'dev/check-license'
- ]
-
-BUILD:
- - changed-files:
- - any-glob-to-any-file: [
- '**/*.gradle',
- 'gradle/**/*',
- 'config/**/*',
- 'build-tools/**/*'
- ]
-
-DOCS:
- - changed-files:
- - any-glob-to-any-file: [
- '**/README.md'
- ]
-
-EXAMPLES:
- - changed-files:
- - any-glob-to-any-file: [
- 'examples/**/*'
- ]
-
-API:
- - changed-files:
- - any-glob-to-any-file: [
- 'spark-operator-api/**/*'
- ]
-
-SUBMISSION WORKER:
- - changed-files:
- - any-glob-to-any-file: [
- 'spark-submission-worker/**/*'
- ]
-
-OPERATOR:
- - changed-files:
- - any-glob-to-any-file: [
- 'spark-operator/**/*'
- ]
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
deleted file mode 100644
index d8eefd9..0000000
--- a/.github/workflows/build_and_test.yml
+++ /dev/null
@@ -1,166 +0,0 @@
-name: Build and test
-
-on:
- push:
- branches:
- - main
- pull_request:
- branches:
- - main
-
-# Cancel previous PR build and test
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }}
- cancel-in-progress: true
-
-jobs:
- license-check:
- name: "License Check"
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
- - name: Check license header
- uses: apache/skywalking-eyes@main
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- config: .github/.licenserc.yaml
- build-test:
- name: "Build Test CI"
- runs-on: ${{ matrix.os }}
- strategy:
- fail-fast: false
- matrix:
- os: [ 'ubuntu-latest', 'ubuntu-24.04-arm' ]
- java-version: [ 17, 21, 24 ]
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
- - name: Set up JDK ${{ matrix.java-version }}
- uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: 'adopt'
- cache: 'gradle'
- - name: Build with Gradle
- run: |
- ./gradlew build
- build-image:
- name: "Build Operator Image CI"
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- - name: Set up JDK 17
- uses: actions/setup-java@v4
- with:
- java-version: 17
- distribution: 'adopt'
- cache: 'gradle'
- - name: Build Operator Image
- run: |
- ./gradlew buildDockerImage
- docker run spark-kubernetes-operator:$(./gradlew properties | grep '^version:' | awk '{print $2}')
- k8s-integration-tests:
- name: "K8s Integration Tests"
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- kubernetes-version:
- - "1.30.0"
- - "1.33.0"
- mode:
- - dynamic
- - static
- test-group:
- - spark-versions
- - python
- - state-transition
- - watched-namespaces
- exclude:
- - mode: dynamic
- test-group: spark-versions
- - mode: dynamic
- test-group: python
- - mode: dynamic
- test-group: state-transition
- - mode: static
- test-group: watched-namespaces
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- - name: Set up JDK 21
- uses: actions/setup-java@v4
- with:
- java-version: 21
- distribution: 'adopt'
- cache: 'gradle'
- - name: Set up Minikube
- uses: medyagh/setup-minikube@v0.0.19
- with:
- cache: true
- kubernetes-version: ${{ matrix.kubernetes-version }}
- cpus: 2
- memory: 6144m
- - name: Set Up Go
- uses: actions/setup-go@v5
- - name: Set Up Chainsaw
- run: |
- go install github.com/kyverno/chainsaw@latest
- - name: Print K8S pods and nodes info
- run: |
- kubectl get pods -A
- kubectl describe node
- - name: Run Spark K8s Operator on K8S with Dynamic Configuration Disabled
- if: matrix.mode == 'static'
- run: |
- eval $(minikube docker-env)
- ./gradlew buildDockerImage
- ./gradlew spark-operator-api:relocateGeneratedCRD
- helm install spark-kubernetes-operator --create-namespace -f build-tools/helm/spark-kubernetes-operator/values.yaml build-tools/helm/spark-kubernetes-operator/
- helm test spark-kubernetes-operator
- # Use remote host' s docker image
- minikube docker-env --unset
- - name: Run E2E Test with Dynamic Configuration Disabled
- if: matrix.mode == 'static'
- run: |
- chainsaw test --test-dir ./tests/e2e/${{ matrix.test-group }} --parallel 2
- - name: Run Spark K8s Operator on K8S with Dynamic Configuration Enabled
- if: matrix.mode == 'dynamic'
- run: |
- eval $(minikube docker-env)
- ./gradlew buildDockerImage
- ./gradlew spark-operator-api:relocateGeneratedCRD
- helm install spark-kubernetes-operator --create-namespace -f \
- build-tools/helm/spark-kubernetes-operator/values.yaml -f \
- tests/e2e/helm/dynamic-config-values.yaml \
- build-tools/helm/spark-kubernetes-operator/
- minikube docker-env --unset
- - name: Run E2E Test with Dynamic Configuration Enabled
- if: matrix.mode == 'dynamic'
- run: |
- chainsaw test --test-dir ./tests/e2e/${{ matrix.test-group }} --parallel 2
-
- lint:
- name: "Linter and documentation"
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- - name: Set up JDK 17
- uses: actions/setup-java@v4
- with:
- java-version: 17
- distribution: 'adopt'
- cache: 'gradle'
- - name: Linters
- run: |
- ./gradlew build -x test
- - name: Javadoc Generation
- run: |
- ./gradlew javadoc
- - name: Validate helm chart linting
- run: |
- helm lint --strict build-tools/helm/spark-kubernetes-operator
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
deleted file mode 100644
index ccfdeb1..0000000
--- a/.github/workflows/labeler.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Intentionally has a general name.
-# because the test status check created in GitHub Actions
-# currently randomly picks any associated workflow.
-# So, the name was changed to make sense in that context too.
-# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10
-
-name: "On pull requests"
-on: pull_request_target
-
-jobs:
- label:
- name: Label pull requests
- runs-on: ubuntu-latest
- permissions:
- contents: read
- pull-requests: write
- steps:
- - uses: actions/labeler@v5
- with:
- repo-token: "${{ secrets.GITHUB_TOKEN }}"
- sync-labels: true
diff --git a/.github/workflows/publish_snapshot_chart.yml b/.github/workflows/publish_snapshot_chart.yml
deleted file mode 100644
index 7ba69eb..0000000
--- a/.github/workflows/publish_snapshot_chart.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-name: Publish Snapshot Chart
-
-on:
- schedule:
- - cron: '0 0 * * *'
-
- workflow_dispatch:
- inputs:
- branch:
- description: 'list of branches to publish (JSON)'
- required: true
- # keep in sync with default value of strategy matrix 'branch'
- default: '["main", "branch-0.1"]'
-
-jobs:
- publish-snapshot-chart:
- if: ${{ startsWith(github.repository, 'apache/') }}
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- # keep in sync with default value of workflow_dispatch input 'branch'
- branch: ${{ fromJSON( inputs.branch || '["main", "branch-0.1"]' ) }}
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- ref: ${{ matrix.branch }}
- - name: Set up JDK 17
- uses: actions/setup-java@v4
- with:
- java-version: 17
- distribution: 'adopt'
- cache: 'gradle'
- - name: Build Operator
- run: |
- ./gradlew build spark-operator-api:relocateGeneratedCRD -x check --no-daemon
- - name: Build Chart
- env:
- DIR: 'charts'
- run: |
- if [[ ${{ matrix.branch }} != 'main' ]]; then
- export DIR=charts-${{ matrix.branch }}
- fi
- cd build-tools/helm
- sed -i 's/repository: /repository: apache\//' spark-kubernetes-operator/values.yaml
- sed -i 's/tag: .*$/tag: ${{ matrix.branch }}-snapshot/' spark-kubernetes-operator/values.yaml
- mkdir -p tmp/$DIR
- helm package spark-kubernetes-operator -d tmp/$DIR --app-version ${{ matrix.branch }}-snapshot
- helm repo index tmp/$DIR --url https://nightlies.apache.org/spark/$DIR
- helm show chart tmp/$DIR/spark-kubernetes-operator-*.tgz
- - name: Upload
- uses: burnett01/rsync-deployments@5.2
- with:
- switches: -avzr
- path: build-tools/helm/tmp/*
- remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/spark
- remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
- remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
- remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
- remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
diff --git a/.github/workflows/publish_snapshot_dockerhub.yml b/.github/workflows/publish_snapshot_dockerhub.yml
deleted file mode 100644
index d301f3c..0000000
--- a/.github/workflows/publish_snapshot_dockerhub.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: Publish Snapshot Image
-
-on:
- schedule:
- - cron: '0 0 * * *'
-
- workflow_dispatch:
- inputs:
- branch:
- description: 'list of branches to publish (JSON)'
- required: true
- # keep in sync with default value of strategy matrix 'branch'
- default: '["main", "branch-0.1"]'
-
-jobs:
- publish-snapshot-image:
- if: ${{ startsWith(github.repository, 'apache/') }}
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- # keep in sync with default value of workflow_dispatch input 'branch'
- branch: ${{ fromJSON( inputs.branch || '["main", "branch-0.1"]' ) }}
- steps:
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v3
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
- - name: Login to Docker Hub
- uses: docker/login-action@v3
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- ref: ${{ matrix.branch }}
- - name: Build and push
- uses: docker/build-push-action@v6
- with:
- # build cache on Github Actions, See: https://docs.docker.com/build/cache/backends/gha/#using-dockerbuild-push-action
- cache-from: type=gha
- cache-to: type=gha,mode=max
- context: .
- file: build-tools/docker/Dockerfile
- platforms: linux/amd64,linux/arm64
- push: true
- tags: apache/spark-kubernetes-operator:${{ matrix.branch }}-snapshot
diff --git a/.gitignore b/.gitignore
index b174a96..1e73773 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,19 +1,5 @@
*.swp
*~
-.java-version
.DS_Store
.idea/
.vscode
-/lib/
-target/
-build-tools/helm/spark-kubernetes-operator/crds
-
-# Gradle Files #
-################
-.gradle
-.m2
-.out/
-build
-dependencies.lock
-**/dependencies.lock
-gradle/wrapper/gradle-wrapper.jar
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index bde9e98..0000000
--- a/LICENSE
+++ /dev/null
@@ -1,211 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
---------------------------------------------------------------------------------
-
-This product includes a gradle wrapper.
-
-* gradlew and gradle/wrapper/gradle-wrapper.properties
-
-Copyright: 2015-2021 Gradle Authors.
-Home page: https://github.com/gradle/gradle
-License: https://www.apache.org/licenses/LICENSE-2.0
diff --git a/LICENSE-binary b/LICENSE-binary
deleted file mode 100644
index bde9e98..0000000
--- a/LICENSE-binary
+++ /dev/null
@@ -1,211 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
---------------------------------------------------------------------------------
-
-This product includes a gradle wrapper.
-
-* gradlew and gradle/wrapper/gradle-wrapper.properties
-
-Copyright: 2015-2021 Gradle Authors.
-Home page: https://github.com/gradle/gradle
-License: https://www.apache.org/licenses/LICENSE-2.0
diff --git a/NOTICE b/NOTICE
deleted file mode 100644
index 9246cc5..0000000
--- a/NOTICE
+++ /dev/null
@@ -1,6 +0,0 @@
-Apache Spark
-Copyright 2014 and onwards The Apache Software Foundation.
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
diff --git a/NOTICE-binary b/NOTICE-binary
deleted file mode 100644
index 9246cc5..0000000
--- a/NOTICE-binary
+++ /dev/null
@@ -1,6 +0,0 @@
-Apache Spark
-Copyright 2014 and onwards The Apache Software Foundation.
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
diff --git a/README.md b/README.md
index 2a89b4f..bb39e3b 100644
--- a/README.md
+++ b/README.md
@@ -1,46 +1,23 @@
# Apache Spark K8s Operator
-[](https://github.com/apache/spark-kubernetes-operator/actions/workflows/build_and_test.yml)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://img.shields.io/github/repo-size/apache/spark-kubernetes-operator)
Apache Spark™ K8s Operator is a subproject of [Apache Spark](https://spark.apache.org/) and
aims to extend K8s resource manager to manage Apache Spark applications via
[Operator Pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/).
-## Building Spark K8s Operator
-
-Spark K8s Operator is built using Gradle.
-To build, run:
-
-```bash
-$ ./gradlew build -x test
-```
-
-## Running Tests
-
-```bash
-$ ./gradlew build
-```
-
-## Build Docker Image
-
-```bash
-$ ./gradlew buildDockerImage
-```
-
## Install Helm Chart
```bash
-$ ./gradlew spark-operator-api:relocateGeneratedCRD
-
-$ helm install spark-kubernetes-operator --create-namespace -f build-tools/helm/spark-kubernetes-operator/values.yaml build-tools/helm/spark-kubernetes-operator/
+helm repo add spark-kubernetes-operator https://apache.github.io/spark-kubernetes-operator
+helm repo update
+helm install spark-kubernetes-operator
```
## Run Spark Pi App
```bash
-$ kubectl apply -f examples/pi.yaml
+$ kubectl apply -f https://raw.githubusercontent.com/apache/spark-kubernetes-operator/refs/tags/v0.1.0/examples/pi.yaml
$ kubectl get sparkapp
NAME CURRENT STATE AGE
@@ -52,82 +29,16 @@ $ kubectl delete sparkapp/pi
## Run Spark Cluster
```bash
-$ kubectl apply -f examples/prod-cluster-with-three-workers.yaml
+$ kubectl apply -f https://raw.githubusercontent.com/apache/spark-kubernetes-operator/refs/tags/v0.1.0/examples/prod-cluster-with-three-workers.yaml
$ kubectl get sparkcluster
NAME CURRENT STATE AGE
prod RunningHealthy 10s
-$ kubectl port-forward prod-master-0 6066 &
-
-$ ./examples/submit-pi-to-prod.sh
-{
- "action" : "CreateSubmissionResponse",
- "message" : "Driver successfully submitted as driver-20240821181327-0000",
- "serverSparkVersion" : "4.0.0-preview2",
- "submissionId" : "driver-20240821181327-0000",
- "success" : true
-}
-
-$ curl http://localhost:6066/v1/submissions/status/driver-20240821181327-0000/
-{
- "action" : "SubmissionStatusResponse",
- "driverState" : "FINISHED",
- "serverSparkVersion" : "4.0.0-preview2",
- "submissionId" : "driver-20240821181327-0000",
- "success" : true,
- "workerHostPort" : "10.1.5.188:42099",
- "workerId" : "worker-20240821181236-10.1.5.188-42099"
-}
-
$ kubectl delete sparkcluster prod
sparkcluster.spark.apache.org "prod" deleted
```
-## Run Spark Pi App on Apache YuniKorn scheduler
-
-If you have not yet done so, follow [YuniKorn docs](https://yunikorn.apache.org/docs/#install) to install the latest version:
-
-```bash
-$ helm repo add yunikorn https://apache.github.io/yunikorn-release
-
-$ helm repo update
-
-$ helm install yunikorn yunikorn/yunikorn --namespace yunikorn --version 1.6.3 --create-namespace --set embedAdmissionController=false
-```
-
-Submit a Spark app to YuniKorn enabled cluster:
-
-```bash
-
-$ kubectl apply -f examples/pi-on-yunikorn.yaml
-
-$ kubectl describe pod pi-on-yunikorn-0-driver
-...
-Events:
- Type Reason Age From Message
- ---- ------ ---- ---- -------
- Normal Scheduling 14s yunikorn default/pi-on-yunikorn-0-driver is queued and waiting for allocation
- Normal Scheduled 14s yunikorn Successfully assigned default/pi-on-yunikorn-0-driver to node docker-desktop
- Normal PodBindSuccessful 14s yunikorn Pod default/pi-on-yunikorn-0-driver is successfully bound to node docker-desktop
- Normal TaskCompleted 6s yunikorn Task default/pi-on-yunikorn-0-driver is completed
- Normal Pulled 13s kubelet Container image "apache/spark:4.0.0-preview2" already present on machine
- Normal Created 13s kubelet Created container spark-kubernetes-driver
- Normal Started 13s kubelet Started container spark-kubernetes-driver
-
-$ kubectl delete sparkapp pi-on-yunikorn
-sparkapplication.spark.apache.org "pi-on-yunikorn" deleted
-```
-
-## Try nightly build for testing
-
-As of now, you can try `spark-kubernetes-operator` nightly version in the following way.
-
-```
-$ helm install spark-kubernetes-operator \
-https://nightlies.apache.org/spark/charts/spark-kubernetes-operator-0.2.0-SNAPSHOT.tgz
-```
-
## Clean Up
Check the existing Spark applications and clusters. If exists, delete them.
@@ -149,13 +60,3 @@ $ kubectl delete crd sparkapplications.spark.apache.org
$ kubectl delete crd sparkclusters.spark.apache.org
```
-
-In case of nightly builds, remove the snapshot image.
-```
-$ docker rmi apache/spark-kubernetes-operator:main-snapshot
-```
-
-## Contributing
-
-Please review the [Contribution to Spark guide](https://spark.apache.org/contributing.html)
-for information on how to get started contributing to the project.
diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile
deleted file mode 100644
index b6c8bb0..0000000
--- a/build-tools/docker/Dockerfile
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM gradle:8.14-jdk17-noble AS builder
-
-WORKDIR /app
-
-COPY . .
-
-RUN --mount=type=cache,target=/home/gradle/.gradle/caches gradle --no-daemon clean build -x check
-
-FROM azul/zulu-openjdk:24-jre
-
-ARG APP_VERSION=0.2.0-SNAPSHOT
-ARG SPARK_UID=185
-
-LABEL org.opencontainers.image.authors="Apache Spark project <de...@spark.apache.org>"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Kubernetes Operator"
-LABEL org.opencontainers.image.version="${APP_VERSION}"
-
-ENV SPARK_OPERATOR_HOME=/opt/spark-operator
-ENV SPARK_OPERATOR_WORK_DIR=/opt/spark-operator/operator
-ENV SPARK_OPERATOR_JAR=spark-kubernetes-operator.jar
-
-WORKDIR $SPARK_OPERATOR_WORK_DIR
-
-RUN groupadd --system --gid=$SPARK_UID spark && \
- useradd --system --home-dir $SPARK_OPERATOR_HOME --uid=$SPARK_UID --gid=spark spark && \
- chown -R spark:spark $SPARK_OPERATOR_HOME
-
-COPY --from=builder --chown=spark:spark /app/spark-operator/build/libs/spark-kubernetes-operator-$APP_VERSION-all.jar $SPARK_OPERATOR_JAR
-COPY --from=builder --chown=spark:spark /app/build-tools/docker/docker-entrypoint.sh .
-
-USER spark
-
-ENTRYPOINT ["/opt/spark-operator/operator/docker-entrypoint.sh"]
-
-CMD ["help"]
diff --git a/build-tools/docker/docker-entrypoint.sh b/build-tools/docker/docker-entrypoint.sh
deleted file mode 100755
index 1397996..0000000
--- a/build-tools/docker/docker-entrypoint.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-args=("$@")
-
-if [ "$1" = "help" ]; then
- printf "Usage: $(basename "$0") (operator)\n"
- printf " Or $(basename "$0") help\n\n"
- exit 0
-elif [ "$1" = "operator" ]; then
- echo "Starting Operator..."
-
- exec java -cp "./$SPARK_OPERATOR_JAR" "$LOG_CONFIG" "$OPERATOR_JAVA_OPTS" org.apache.spark.k8s.operator.SparkOperator
-fi
-
-args=("${args[@]}")
-
-# Running command in pass-through mode
-exec "${args[@]}"
diff --git a/build-tools/docs-utils/build.gradle b/build-tools/docs-utils/build.gradle
deleted file mode 100644
index 5ce8e64..0000000
--- a/build-tools/docs-utils/build.gradle
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-ext {
- javaMainClass = "org.apache.spark.k8s.operator.utils.ConfOptionDocGenerator"
- docsPath = System.getProperty("user.dir") + "/docs"
-}
-
-dependencies {
- implementation project(":spark-operator")
- implementation(libs.log4j.core)
- implementation(libs.log4j.slf4j.impl)
- compileOnly(libs.lombok)
- annotationProcessor(libs.lombok)
-}
-
-test {
- useJUnitPlatform()
-}
-
-tasks.register('generateConfPropsDoc', Exec) {
- description = "Generate config properties doc for operator"
- commandLine "java", "-classpath", sourceSets.main.runtimeClasspath.getAsPath(), javaMainClass, docsPath
-}
-
-build.finalizedBy(generateConfPropsDoc)
diff --git a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java
deleted file mode 100644
index c0f2f49..0000000
--- a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/ConfOptionDocGenerator.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.utils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.lang.reflect.Field;
-import java.util.List;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.spark.k8s.operator.config.ConfigOption;
-import org.apache.spark.k8s.operator.config.SparkOperatorConf;
-
-@Slf4j
-public class ConfOptionDocGenerator {
- public static final String CONF_FILE_NAME = "config_properties.md";
- public static final String DEFAULT_DOCS_PATH = "docs";
- public static final String GENERATED_FILE_HEADER =
- "This doc is automatically generated by gradle task, manual updates would be overridden.";
-
- public void generate(String docsPath) throws IOException, IllegalAccessException {
- Field[] fields = SparkOperatorConf.class.getDeclaredFields();
- File docsDir = new File(docsPath);
- if (!docsDir.exists() && docsDir.mkdir()) {
- log.info("Creating docs directory at {}", docsPath);
- }
- File generated = new File(docsPath, CONF_FILE_NAME);
- if (generated.createNewFile()) {
- log.info("Creating props at {}/{}", docsPath, CONF_FILE_NAME);
- }
- PrintWriter printWriter = new PrintWriter(generated, "UTF-8");
- printWriter.println(String.format("[//]: # (%s)", GENERATED_FILE_HEADER));
- printWriter.println("# Spark Operator Config Properties");
- DocTable table =
- DocTable.builder()
- .headers(List.of("Key", "Type", "Default Value", "Allow Hot Reloading", "Description"))
- .columns(5)
- .build();
- for (Field f : fields) {
- if (ConfigOption.class.isAssignableFrom(f.getType())) {
- ConfigOption<?> conf = (ConfigOption<?>) f.get(this);
- table.addRow(
- List.of(
- conf.getKey(),
- conf.getTypeParameterClass().getSimpleName(),
- conf.getDefaultValue().toString(),
- String.valueOf(conf.isEnableDynamicOverride()),
- conf.getDescription()));
- }
- }
- table.flush(printWriter);
- printWriter.close();
- }
-
- public static void main(String[] args) throws IOException, IllegalAccessException {
- ConfOptionDocGenerator generator = new ConfOptionDocGenerator();
- String docsPath = DEFAULT_DOCS_PATH;
- if (args.length > 0) {
- docsPath = args[0];
- }
- try {
- generator.generate(docsPath);
- } catch (IOException | IllegalAccessException e) {
- log.error("Failed to generate docs for config props.", e);
- throw e;
- }
- }
-}
diff --git a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java b/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java
deleted file mode 100644
index 73e0597..0000000
--- a/build-tools/docs-utils/src/main/java/org/apache/spark/k8s/operator/utils/DocTable.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.utils;
-
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import lombok.Builder;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-
-@Data
-@RequiredArgsConstructor
-@Builder
-public class DocTable {
- // Separator among cells
- public static final String ROW_SEPARATOR = " | ";
- // Separator for header line
- public static final String HEADER_SEPARATOR = "---";
- private final List<String> headers;
- private final int columns;
- @Builder.Default private final List<List<String>> rows = new ArrayList<>();
-
- public void addRow(List<String> row) {
- rows.add(row);
- }
-
- public void flush(PrintWriter writer) {
- writer.println(joinRow(headers));
- writer.println(joinRow(Collections.nCopies(columns, HEADER_SEPARATOR)));
- for (List<String> row : rows) {
- writer.println(joinRow(row));
- }
- writer.println();
- writer.flush();
- }
-
- private String joinRow(List<String> elements) {
- StringBuilder stringBuilder = new StringBuilder(ROW_SEPARATOR);
- for (String element : elements) {
- stringBuilder.append(element);
- stringBuilder.append(ROW_SEPARATOR);
- }
- if (elements.size() < columns) {
- // Append empty cells to end if needed
- stringBuilder.append(ROW_SEPARATOR.repeat(columns - elements.size()));
- }
- return stringBuilder.toString();
- }
-}
diff --git a/build-tools/helm/spark-kubernetes-operator/.helmignore b/build-tools/helm/spark-kubernetes-operator/.helmignore
deleted file mode 100644
index 5920392..0000000
--- a/build-tools/helm/spark-kubernetes-operator/.helmignore
+++ /dev/null
@@ -1,12 +0,0 @@
-.DS_Store
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
diff --git a/build-tools/helm/spark-kubernetes-operator/Chart.yaml b/build-tools/helm/spark-kubernetes-operator/Chart.yaml
deleted file mode 100644
index 2f0cebc..0000000
--- a/build-tools/helm/spark-kubernetes-operator/Chart.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: v2
-name: spark-kubernetes-operator
-description: A Helm chart for the Apache Spark Kubernetes Operator
-type: application
-version: 0.2.0-SNAPSHOT
-appVersion: 0.2.0-SNAPSHOT
-icon: https://spark.apache.org/favicon.ico
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl b/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl
deleted file mode 100644
index 587c833..0000000
--- a/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl
+++ /dev/null
@@ -1,146 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "spark-operator.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "spark-operator.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "spark-operator.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "spark-operator.commonLabels" -}}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-helm.sh/chart: {{ include "spark-operator.chart" . }}
-{{- end }}
-
-{{/*
-Dynamic config (hot properties) labels
-*/}}
-{{- define "spark-operator.dynamicConfigLabels" -}}
-app.kubernetes.io/name: {{ include "spark-operator.name" . }}
-app.kubernetes.io/component: "operator-dynamic-config-overrides"
-{{- include "spark-operator.commonLabels" . }}
-{{- end }}
-
-{{/*
-Bootstrap config labels
-*/}}
-{{- define "spark-operator.configLabels" -}}
-app.kubernetes.io/name: {{ include "spark-operator.name" . }}
-app.kubernetes.io/component: "operator-config"
-{{- include "spark-operator.commonLabels" . }}
-{{- end }}
-
-{{/*
-Deployment selector labels
-*/}}
-{{- define "spark-operator.deploymentSelectorLabels" -}}
-app.kubernetes.io/name: {{ include "spark-operator.name" . }}
-app.kubernetes.io/component: "operator-deployment"
-{{- end }}
-
-{{/*
-Create the path of the operator image to use
-*/}}
-{{- define "spark-operator.imagePath" -}}
-{{- if .Values.image.digest }}
-{{- .Values.image.repository }}@{{ .Values.image.digest }}
-{{- else }}
-{{- .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}
-{{- end }}
-{{- end }}
-
-{{/*
-List of Spark workload namespaces. If not provied in values, use the same namespace as operator
-*/}}
-{{- define "spark-operator.workloadNamespacesStr" -}}
-{{- if index (.Values.workloadResources.namespaces) "data" }}
-{{- $ns_list := join "," .Values.workloadResources.namespaces.data }}
-{{- printf "%s" $ns_list }}
-{{- else }}
-{{- printf "%s" .Release.Namespace }}
-{{- end }}
-{{- end }}
-
-{{/*
-Default property overrides
-*/}}
-{{- define "spark-operator.defaultPropertyOverrides" -}}
-# Runtime resolved properties
-spark.kubernetes.operator.namespace={{ .Release.Namespace }}
-spark.kubernetes.operator.name={{- include "spark-operator.name" . }}
-spark.kubernetes.operator.dynamicConfig.enabled={{ .Values.operatorConfiguration.dynamicConfig.enable }}
-{{- if .Values.workloadResources.namespaces.overrideWatchedNamespaces }}
-spark.kubernetes.operator.watchedNamespaces={{ include "spark-operator.workloadNamespacesStr" . | trim }}
-{{- end }}
-{{- end }}
-
-{{/*
-Readiness Probe properties overrides
-*/}}
-{{- define "spark-operator.readinessProbe.failureThreshold" -}}
-{{- default 30 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.failureThreshold }}
-{{- end }}
-{{- define "spark-operator.readinessProbe.periodSeconds" -}}
-{{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.periodSeconds }}
-{{- end }}
-
-{{/*
-Liveness Probe properties override
-*/}}
-{{- define "spark-operator.livenessProbe.initialDelaySeconds" -}}
-{{- default 30 .Values.operatorDeployment.operatorPod.operatorContainer.probes.livenessProbe.initialDelaySeconds }}
-{{- end }}
-{{- define "spark-operator.livenessProbe.periodSeconds" -}}
-{{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.livenessProbe.periodSeconds }}
-{{- end }}
-
-{{/*
-Readiness Probe property overrides
-*/}}
-{{- define "spark-operator.probePort" -}}
-{{- default 19091 .Values.operatorDeployment.operatorPod.operatorContainer.probes.port }}
-{{- end }}
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml
deleted file mode 100644
index eebbf55..0000000
--- a/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml
+++ /dev/null
@@ -1,166 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{/*
-RBAC rules used to create the operator (cluster)role
-*/}}
-{{- define "spark-operator.operatorRbacRules" }}
-rules:
- - apiGroups:
- - ""
- resources:
- - pods
- - services
- - configmaps
- - persistentvolumeclaims
- - persistentvolumes
- - events
- verbs:
- - '*'
- - apiGroups:
- - "apps"
- resources:
- - statefulsets
- verbs:
- - '*'
- - apiGroups:
- - "autoscaling"
- resources:
- - horizontalpodautoscalers
- verbs:
- - '*'
- - apiGroups:
- - "spark.apache.org"
- resources:
- - '*'
- verbs:
- - '*'
-{{- end }}
-
-{{/*
-Labels and annotations to be applied on rbacResources
-*/}}
-{{- define "spark-operator.rbacLabelsAnnotations" }}
- labels:
- {{- with .Values.operatorRbac.labels }}
- {{- toYaml . | nindent 4 }}
- {{- end }}
- {{- include "spark-operator.commonLabels" . | nindent 4 }}
- {{- with .Values.operatorRbac.annotations }}
- annotations:
- {{- toYaml . | nindent 4 }}
- {{- end }}
-{{- end }}
-
----
-# Service account and rolebindings for operator
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: {{ .Values.operatorRbac.serviceAccount.name }}
- namespace: {{ .Release.Namespace }}
-{{- template "spark-operator.rbacLabelsAnnotations" $ }}
----
-{{- if .Values.operatorRbac.clusterRoleBinding.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: {{ .Values.operatorRbac.clusterRoleBinding.name }}
-{{- template "spark-operator.rbacLabelsAnnotations" $ }}
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: {{ .Values.operatorRbac.clusterRole.name }}
-subjects:
- - kind: ServiceAccount
- name: {{ .Values.operatorRbac.serviceAccount.name }}
- namespace: {{ .Release.Namespace }}
-{{- end }}
----
-{{- if .Values.operatorRbac.clusterRole.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: {{ .Values.operatorRbac.clusterRole.name }}
-{{- template "spark-operator.rbacLabelsAnnotations" $ }}
-{{- template "spark-operator.operatorRbacRules" $ }}
----
-{{- end }}
-{{- if .Values.operatorConfiguration.dynamicConfig.enable }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: {{ .Values.operatorRbac.configManagement.roleName }}
- namespace: {{ .Release.Namespace }}
-{{- template "spark-operator.rbacLabelsAnnotations" $ }}
-rules:
- - apiGroups:
- - ""
- resources:
- - configmaps
- verbs:
- - '*'
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: {{ .Values.operatorRbac.configManagement.roleBindingName }}
- namespace: {{ .Release.Namespace }}
-{{- template "spark-operator.rbacLabelsAnnotations" $ }}
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: {{ .Values.operatorRbac.configManagement.roleName }}
-subjects:
- - kind: ServiceAccount
- name: {{ .Values.operatorRbac.serviceAccount.name }}
- namespace: {{ .Release.Namespace }}
----
-{{- end }}
-
-{{- $workloadResources := .Values.workloadResources -}}
-{{- $systemNs := .Release.Namespace -}}
-{{- $operatorRbac := .Values.operatorRbac -}}
-{{- if index (.Values.workloadResources.namespaces) "data" }}
-{{- range $workloadNs := .Values.workloadResources.namespaces.data }}
-{{- if $operatorRbac.role.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: {{ $operatorRbac.role.name }}
- namespace: {{ $workloadNs }}
-{{- template "spark-operator.rbacLabelsAnnotations" $ }}
-{{- template "spark-operator.operatorRbacRules" $ }}
----
-{{- end }}
-{{- if $operatorRbac.roleBinding.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: {{ $operatorRbac.roleBinding.name }}
- namespace: {{ $workloadNs }}
-{{- template "spark-operator.rbacLabelsAnnotations" $ }}
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: {{ $operatorRbac.roleBinding.roleRef.kind }}
- name: {{ $operatorRbac.roleBinding.roleRef.name }}
-subjects:
- - kind: ServiceAccount
- name: {{ $operatorRbac.serviceAccount.name }}
- namespace: {{ $systemNs }}
----
-{{- end }}
-{{- end }}
-{{- end }}
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml b/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml
deleted file mode 100644
index 5edd228..0000000
--- a/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml
+++ /dev/null
@@ -1,205 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: {{ include "spark-operator.name" . }}
- namespace: {{ .Release.Namespace }}
- labels:
- {{- include "spark-operator.deploymentSelectorLabels" . | nindent 4 }}
- {{- include "spark-operator.commonLabels" . | nindent 4 }}
-spec:
- replicas: {{ .Values.operatorDeployment.replicas }}
- strategy:
- {{- toYaml .Values.operatorDeployment.strategy | nindent 4 }}
- selector:
- matchLabels:
- {{- include "spark-operator.deploymentSelectorLabels" . | nindent 6 }}
- template:
- metadata:
- labels:
- {{- include "spark-operator.deploymentSelectorLabels" . | nindent 8 }}
- {{- if index (.Values.operatorDeployment.operatorPod) "labels" }}
- {{- with .Values.operatorDeployment.operatorPod.labels }}
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- end }}
- annotations:
- kubectl.kubernetes.io/default-container: {{ .Chart.Name }}
- {{- if index (.Values.operatorDeployment.operatorPod) "annotations" }}
- {{- with .Values.operatorDeployment.operatorPod.annotations }}
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- end }}
- spec:
- {{- with .Values.operatorDeployment.operatorPod.priorityClassName }}
- priorityClassName: {{ . }}
- {{- end }}
- {{- with .Values.operatorDeployment.operatorPod.securityContext }}
- securityContext:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- if .Values.operatorDeployment.operatorPod.nodeSelector }}
- nodeSelector: {{ toYaml .Values.operatorDeployment.operatorPod.nodeSelector | nindent 8 }}
- {{- end }}
- {{- with .Values.operatorDeployment.operatorPod.tolerations }}
- tolerations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.operatorDeployment.operatorPod.affinity }}
- affinity:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.imagePullSecrets }}
- imagePullSecrets:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- serviceAccountName: {{ .Values.operatorRbac.serviceAccount.name }}
- {{- if .Values.operatorDeployment.operatorPod.topologySpreadConstraints }}
- topologySpreadConstraints: {{ toYaml .Values.operatorDeployment.operatorPod.topologySpreadConstraints | nindent 8 }}
- {{- end }}
- containers:
- - name: {{ .Chart.Name }}
- image: {{ include "spark-operator.imagePath" . }}
- imagePullPolicy: {{ .Values.image.pullPolicy }}
- command: [ "./docker-entrypoint.sh", "operator" ]
- ports:
- - containerPort: {{ include "spark-operator.probePort" . }}
- name: probe-port
- env:
- - name: OPERATOR_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: HOST_IP
- valueFrom:
- fieldRef:
- fieldPath: status.hostIP
- - name: POD_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- - name: POD_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: OPERATOR_NAME
- value: {{ include "spark-operator.name" . }}
- - name: LOG_CONFIG
- value: -Dlog4j.configurationFile=/opt/spark-operator/conf/log4j2.properties
- - name: OPERATOR_JAVA_OPTS
- value: {{ .Values.operatorDeployment.operatorPod.operatorContainer.jvmArgs }}
- {{- with .Values.operatorDeployment.operatorPod.operatorContainer.env }}
- {{- toYaml . | nindent 12 }}
- {{- end }}
- {{- with .Values.operatorDeployment.operatorPod.operatorContainer.envFrom }}
- envFrom:
- {{- toYaml . | nindent 12 }}
- {{- end }}
- {{- with .Values.operatorDeployment.operatorPod.operatorContainer.resources }}
- resources:
- {{- toYaml . | nindent 12 }}
- {{- end }}
- readinessProbe:
- httpGet:
- port: probe-port
- path: /readyz
- failureThreshold: {{ include "spark-operator.readinessProbe.failureThreshold" . }}
- periodSeconds: {{ include "spark-operator.readinessProbe.periodSeconds" . }}
- livenessProbe:
- httpGet:
- port: probe-port
- path: /healthz
- initialDelaySeconds: {{ include "spark-operator.livenessProbe.initialDelaySeconds" . }}
- periodSeconds: {{ include "spark-operator.livenessProbe.periodSeconds" . }}
- {{- with .Values.operatorDeployment.operatorPod.operatorContainer.securityContext }}
- securityContext:
- {{- toYaml . | nindent 12 }}
- {{- end }}
- volumeMounts:
- - name: spark-operator-config-volume
- mountPath: /opt/spark-operator/conf
- - name: logs-volume
- mountPath: /opt/spark-operator/logs
- {{- with .Values.operatorDeployment.operatorPod.operatorContainer.volumeMounts }}
- {{- toYaml . | nindent 12 }}
- {{- end }}
- {{- with .Values.operatorDeployment.operatorPod.additionalContainers }}
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- if index (.Values.operatorDeployment.operatorPod) "dnsPolicy" }}
- dnsPolicy: {{ .Values.operatorDeployment.operatorPod.dnsPolicy | quote }}
- {{- end }}
- {{- if index (.Values.operatorDeployment.operatorPod) "dnsConfig" }}
- dnsConfig:
- {{- with .Values.operatorDeployment.operatorPod.dnsConfig }}
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- end }}
- restartPolicy: Always
- volumes:
- - name: spark-operator-config-volume
- configMap:
- name: spark-kubernetes-operator-configuration
- - name: logs-volume
- emptyDir: { }
- {{- with .Values.operatorDeployment.operatorPod.volumes }}
- {{- toYaml . | nindent 8 }}
- {{- end }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: spark-kubernetes-operator-configuration
- namespace: {{ .Release.Namespace }}
- labels:
- {{- include "spark-operator.configLabels" . | nindent 4 }}
-data:
- log4j2.properties: |+
-{{- if .Values.operatorConfiguration.append }}
- {{- $.Files.Get "conf/log4j2.properties" | nindent 4 -}}
-{{- end }}
-{{- if index (.Values.operatorConfiguration) "log4j2.properties" }}
- {{- index (.Values.operatorConfiguration) "log4j2.properties" | nindent 4 -}}
-{{- end }}
- spark-operator.properties: |+
- {{- include "spark-operator.defaultPropertyOverrides" . | nindent 4 }}
-{{- if .Values.operatorConfiguration.append }}
- {{- $.Files.Get "conf/spark-operator.properties" | nindent 4 -}}
-{{- end }}
-{{- if index (.Values.operatorConfiguration) "spark-operator.properties" }}
- {{- index (.Values.operatorConfiguration) "spark-operator.properties" | nindent 4 -}}
-{{- end }}
- metrics.properties: |+
-{{- if index (.Values.operatorConfiguration) "metrics.properties" }}
- {{- index (.Values.operatorConfiguration) "metrics.properties" | nindent 4 -}}
-{{- end }}
----
-{{- if .Values.operatorConfiguration.dynamicConfig.create }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: spark-kubernetes-operator-dynamic-configuration
- namespace: {{ .Release.Namespace }}
- labels:
- {{- include "spark-operator.dynamicConfigLabels" . | nindent 4 }}
- annotations:
- {{- toYaml .Values.operatorConfiguration.dynamicConfig.annotations | nindent 4 }}
-{{- with .Values.operatorConfiguration.dynamicConfig.data }}
-data:
- {{- toYaml . | nindent 2 }}
-{{- end }}
-{{- end }}
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml
deleted file mode 100644
index 090a520..0000000
--- a/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: v1
-kind: Pod
-metadata:
- name: "{{ include "spark-operator.name" . }}-test-operator-rbac"
- namespace: {{ .Release.Namespace }}
- labels:
- {{- include "spark-operator.commonLabels" . | nindent 4 }}
- annotations:
- "helm.sh/hook": test
- "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
-spec:
- containers:
- - name: kubectl
- image: bitnami/kubectl:latest
- command: ['bash', '-c' ]
- args: [
- 'kubectl auth can-i list sparkapplications --all-namespaces',
- 'kubectl auth can-i list sparkclusters --all-namespaces',
- 'kubectl auth can-i create pods --all-namespaces',
- 'kubectl auth can-i create services --all-namespaces',
- 'kubectl auth can-i create configmaps --all-namespaces',
- 'kubectl auth can-i create persistentvolumeclaims',
- 'kubectl auth can-i create events --all-namespaces'
- ]
- serviceAccountName: {{ .Values.operatorRbac.serviceAccount.name }}
- restartPolicy: Never
----
-apiVersion: v1
-kind: Pod
-metadata:
- name: "{{ include "spark-operator.name" . }}-test-workload-rbac"
- namespace: {{ .Release.Namespace }}
- labels:
- {{- include "spark-operator.commonLabels" . | nindent 4 }}
- annotations:
- "helm.sh/hook": test
- "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
-spec:
- containers:
- - name: kubectl
- image: bitnami/kubectl:latest
- command: ['bash', '-c' ]
- args: [
- 'kubectl auth can-i create pods -n {{ .Release.Namespace }}',
- 'kubectl auth can-i create configmaps -n {{ .Release.Namespace }}',
- 'kubectl auth can-i create services -n {{ .Release.Namespace }}',
- 'kubectl auth can-i create persistentvolumeclaims -n {{ .Release.Namespace }}'
- ]
- serviceAccountName: {{ .Values.workloadResources.serviceAccount.name }}
- restartPolicy: Never
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml
deleted file mode 100644
index 06ed788..0000000
--- a/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml
+++ /dev/null
@@ -1,180 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{/*
-RBAC rules used to create the workload (cluster)role based on the scope
-*/}}
-{{- define "spark-operator.workloadRbacRules" }}
-rules:
- - apiGroups:
- - ""
- resources:
- - pods
- - services
- - configmaps
- - persistentvolumeclaims
- verbs:
- - '*'
-{{- end }}
-
-{{/*
-RoleRef for workload service account rolebindings
-*/}}
-{{- define "spark-operator.workloadRoleRef" }}
-roleRef:
- apiGroup: rbac.authorization.k8s.io
-{{- if .Values.workloadResources.clusterRole.create }}
- kind: ClusterRole
- name: {{ .Values.workloadResources.clusterRole.name }}
-{{- else }}
- kind: Role
- name: {{ .Values.workloadResources.role.name }}
-{{- end }}
-{{- end }}
-
-{{/*
-Labels and annotations to be applied
-*/}}
-{{- define "spark-operator.workloadLabels" -}}
- {{- with .Values.workloadResources.labels }}
- {{- toYaml . | nindent 4 }}
- {{- end }}
- {{- include "spark-operator.commonLabels" . | nindent 4 }}
-{{- end }}
-
-{{- define "spark-operator.workloadAnnotations" -}}
- {{- with .Values.workloadResources.annotations }}
- {{- toYaml . | nindent 4 }}
- {{- end }}
-{{- end }}
-
-{{- define "spark-operator.workloadLabelsAnnotations" }}
- labels:
- {{ template "spark-operator.workloadLabels" $ }}
- annotations:
- {{ template "spark-operator.workloadAnnotations" $ }}
-{{- end }}
----
-{{- $workloadResources := .Values.workloadResources -}}
-{{- $systemNs := .Release.Namespace -}}
-{{- $operatorRbac := .Values.operatorRbac -}}
-{{- if index (.Values.workloadResources.namespaces) "data" }}
-{{- range $workloadNs := .Values.workloadResources.namespaces.data }}
-{{- if $workloadResources.namespaces.create }}
-apiVersion: v1
-kind: Namespace
-metadata:
- name: {{ $workloadNs }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
----
-{{- end }}
-{{- if $workloadResources.serviceAccount.create }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: {{ $workloadResources.serviceAccount.name }}
- namespace: {{ $workloadNs }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
----
-{{- end }}
-{{- if $workloadResources.role.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: {{ $workloadResources.role.name }}
- namespace: {{ $workloadNs }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
-{{- template "spark-operator.workloadRbacRules" $ }}
----
-{{- end }}
-{{- if $workloadResources.roleBinding.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: {{ $workloadResources.roleBinding.name }}
- namespace: {{ $workloadNs }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
-{{- template "spark-operator.workloadRoleRef" $ }}
-subjects:
- - kind: ServiceAccount
- name: {{ $workloadResources.serviceAccount.name }}
- namespace: {{ $workloadNs }}
----
-{{- end }}
-{{- end }}
-{{- else }}
-{{- if $workloadResources.serviceAccount.create }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: {{ $workloadResources.serviceAccount.name }}
- namespace: {{ $systemNs }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
----
-{{- end }}
-{{- if $workloadResources.role.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: {{ $workloadResources.role.name }}
- namespace: {{ $systemNs }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
-{{- template "spark-operator.workloadRbacRules" $ }}
----
-{{- end }}
-{{- if $workloadResources.roleBinding.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: {{ $workloadResources.serviceAccount.name }}
- namespace: {{ $systemNs }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
-{{- template "spark-operator.workloadRoleRef" $ }}
-subjects:
- - kind: ServiceAccount
- name: {{ $workloadResources.serviceAccount.name }}
- namespace: {{ $systemNs }}
----
-{{- end }}
-{{- end }}
-
-{{- if $workloadResources.clusterRole.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: {{ $workloadResources.clusterRole.name }}
-{{- template "spark-operator.workloadLabelsAnnotations" $ }}
-{{- template "spark-operator.workloadRbacRules" $ }}
----
-{{- end }}
-{{- if $workloadResources.sparkApplicationSentinel.create }}
-{{- range $sentinelNs := .Values.workloadResources.sparkApplicationSentinel.sentinelNamespaces.data }}
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: {{ $workloadResources.sparkApplicationSentinel.name }}
- namespace: {{ $sentinelNs }}
- labels:
- "spark.operator/sentinel": "true"
- {{- template "spark-operator.workloadLabels" $ }}
- annotations:
- {{- template "spark-operator.workloadAnnotations" $ }}
-spec:
- runtimeVersions:
- sparkVersion: 4.0.0-preview2
- scalaVersion: "2.13"
-{{- end }}
----
-{{- end }}
diff --git a/build-tools/helm/spark-kubernetes-operator/values.yaml b/build-tools/helm/spark-kubernetes-operator/values.yaml
deleted file mode 100644
index a8b4ad3..0000000
--- a/build-tools/helm/spark-kubernetes-operator/values.yaml
+++ /dev/null
@@ -1,194 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-image:
- repository: spark-kubernetes-operator
- pullPolicy: IfNotPresent
- tag: 0.2.0-SNAPSHOT
- # If image digest is set then it takes precedence and the image tag will be ignored
- # digest: ""
-
-imagePullSecrets: [ ]
-
-operatorDeployment:
- # Replicas must be 1 unless leader election is enabled
- replicas: 1
- # Strategy type must be 'Recreate' unless leader election is enabled
- strategy:
- type: Recreate
- operatorPod:
- priorityClassName: null
- annotations: { }
- labels: { }
- affinity: { }
- nodeSelector: { }
- # Node tolerations for operator pod assignment
- # https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
- tolerations: [ ]
- # Topology spread constrains
- # https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
- topologySpreadConstraints: [ ]
- operatorContainer:
- jvmArgs: "-Dfile.encoding=UTF8"
- env:
- - name: "SPARK_USER"
- value: "spark"
- envFrom:
- volumeMounts: { }
- resources:
- limits:
- cpu: "1"
- ephemeral-storage: 2Gi
- memory: 2Gi
- requests:
- cpu: "1"
- ephemeral-storage: 2Gi
- memory: 2Gi
- probes:
- port: 19091
- livenessProbe:
- periodSeconds: 10
- initialDelaySeconds: 30
- startupProbe:
- failureThreshold: 30
- periodSeconds: 10
- # By default, operator container is configured to comply restricted standard
- # https://kubernetes.io/docs/concepts/security/pod-security-standards/
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- runAsNonRoot: true
- runAsUser: 185
- seccompProfile:
- type: RuntimeDefault
- additionalContainers: { }
- volumes: { }
- securityContext: { }
- dnsPolicy:
- dnsConfig:
-
-operatorRbac:
- serviceAccount:
- create: true
- name: "spark-operator"
- clusterRole:
- create: true
- name: "spark-operator-clusterrole"
- clusterRoleBinding:
- create: true
- name: "spark-operator-clusterrolebinding"
- role:
- # If enabled, a Role would be created inside each workload namespace, as configured
- # in {workloadResources.namespaces.data} for operator. Please enable *at least one* of
- # {operatorRbac.clusterRole.create} or {operatorRbac.role.create} for operator
- # unless permission is provided separately from the chart
- create: false
- name: "spark-operator-role"
- roleBinding:
- # If enabled, a RoleBinding would be created inside each workload namespace, as configured
- # in {workloadResources.namespaces.data} for operator. Please enable *at least one* of
- # {operatorRbac.clusterRoleBinding.create} or {operatorRbac.roleBinding.create} for
- # operator unless permission is provided separately from the chart
- create: false
- name: "spark-operator-rolebinding"
- # It's possible to make the rolebinding refers to role / clusterrole as applicable
- roleRef:
- kind: Role
- name: "spark-operator-role"
- configManagement:
- # Role to enable operator loading hot properties from config map
- create: true
- roleName: "spark-operator-config-monitor"
- roleBindingName: "spark-operator-config-monitor-role-binding"
- labels:
- "app.kubernetes.io/component": "operator-rbac"
-
-workloadResources:
- # Create namespace(s), service account(s), clusterrole, role(s) and rolebinding(s) for Spark
- # workloads, including SparkApps and SparkClusters
- namespaces:
- create: true
- # When enabled, value for conf property `spark.operator.watched.namespaces` would be set to
- # the values provided in {workloadResources.namespaces.data} field as well
- overrideWatchedNamespaces: true
- data:
- # - "spark-1"
- # - "spark-2"
- serviceAccount:
- create: true
- name: "spark"
- role:
- # When enabled, a role would be created in each workload namespace, as configured in
- # {workloadResources.namespaces.data} for Spark apps. Please enable *at least one* of
- # {workloadResources.roles.create} or {workloadResources.clusterRole.create} for
- # Spark workload unless permission is provided separately from the chart
- create: false
- name: "spark-workload-role"
- clusterRole:
- # When enabled, a clusterrole would be created for Spark workloads. Please enable *at least one* of
- # {workloadResources.roles.create} or {workloadResources.clusterRole.create} for Spark workload unless
- # permission is provided separately from the chart
- create: true
- name: "spark-workload-clusterrole"
- roleBinding:
- create: true
- name: "spark-workload-rolebinding"
- sparkApplicationSentinel:
- create: false
- name: "spark-app-sentinel"
- sentinelNamespaces:
- data:
- # - "spark-1"
- # - "spark-2"
- # When enabled, a sentinel resource of type SparkApp will be deployed to namespace(s) provided
- # Note that sentinelNamespaces list should be a subset of {workloadResources.namespaces.data}
- # - "default"
- # workload resources are by default annotated to avoid workload abort due to operator upgrade
- annotations:
- "helm.sh/resource-policy": keep
- # labels to be added on workload resources
- labels:
- "app.kubernetes.io/component": "spark-workload"
-
-operatorConfiguration:
- # If set to true, below properties would be appended to default conf files under conf/
- # Otherwise, below would override default conf files
- append: true
- log4j2.properties: |+
- # Logging Overrides
- rootLogger.level=INFO
- rootLogger.appenderRef.stdout.ref = console
- appender.console.type = Console
- appender.console.name = console
- appender.console.target = SYSTEM_ERR
- appender.console.layout.type = PatternLayout
- appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex
- spark-operator.properties: |+
- # Property Overrides. e.g.
- # spark.kubernetes.operator.reconciler.intervalSeconds=60
- metrics.properties: |+
- # Metrics Properties Overrides
- dynamicConfig:
- # Enable this for hot properties loading.
- enable: false
- # Enable this to create a config map for hot property loading
- create: false
- annotations:
- "helm.sh/resource-policy": keep
- data:
- # Spark Operator Config Runtime Properties Overrides. e.g.
- spark.kubernetes.operator.reconciler.intervalSeconds: "60"
diff --git a/build.gradle b/build.gradle
deleted file mode 100644
index 8e13488..0000000
--- a/build.gradle
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-buildscript {
- repositories {
- maven {
- url = uri("https://plugins.gradle.org/m2/")
- }
- }
- dependencies {
- classpath "${libs.spotbugs.gradle.plugin.get()}"
- classpath "${libs.spotless.plugin.gradle.get()}"
- classpath "${libs.shadow.get()}"
- }
-}
-
-assert JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17): "Java 17 or newer is " +
- "required"
-
-allprojects {
- group = "org.apache.spark.k8s.operator"
- version = "0.2.0-SNAPSHOT"
-}
-
-tasks.register('buildDockerImage', Exec) {
- String cmd = "docker build --build-arg APP_VERSION=${version} -t spark-kubernetes-operator:${version} -f build-tools/docker/Dockerfile ."
- println "Build Docker Image: $cmd"
- commandLine "sh", "-c", "$cmd"
-}
-
-subprojects {
- apply plugin: 'idea'
- apply plugin: 'eclipse'
- apply plugin: 'java'
-
- java {
- sourceCompatibility = 17
- targetCompatibility = 17
- }
-
- repositories {
- mavenCentral()
- maven {
- url = "https://repository.apache.org/content/repositories/snapshots/"
- }
- }
-
- apply plugin: 'checkstyle'
- checkstyle {
- toolVersion = libs.versions.checkstyle.get()
- configFile = file("$rootDir/config/checkstyle/checkstyle.xml")
- ignoreFailures = false
- showViolations = true
- }
-
- if (JavaVersion.current() < JavaVersion.VERSION_24) {
- apply plugin: 'pmd'
- pmd {
- ruleSetFiles = files("$rootDir/config/pmd/ruleset.xml")
- toolVersion = libs.versions.pmd.get()
- consoleOutput = true
- ignoreFailures = false
- }
-
- apply plugin: 'com.github.spotbugs'
- spotbugs {
- toolVersion = libs.versions.spotbugs.tool.get()
- afterEvaluate {
- reportsDir = file("${project.reporting.baseDir}/findbugs")
- }
- excludeFilter = file("$rootDir/config/spotbugs/spotbugs_exclude.xml")
- ignoreFailures = false
- }
- }
-
- apply plugin: 'jacoco'
- jacoco {
- toolVersion = libs.versions.jacoco.get()
- }
- jacocoTestReport {
- dependsOn test
- }
-
- apply plugin: 'com.diffplug.spotless'
- spotless {
- java {
- endWithNewline()
- googleJavaFormat('1.22.0')
- importOrder(
- 'java',
- 'javax',
- 'scala',
- '',
- 'org.apache.spark',
- )
- toggleOffOn()
- targetExclude "**/BaseResource.java"
- trimTrailingWhitespace()
- removeUnusedImports()
- }
- format 'misc', {
- target '*.md', '*.gradle', '**/*.properties', '**/*.xml', '**/*.yaml', '**/*.yml'
- endWithNewline()
- trimTrailingWhitespace()
- }
- }
-}
-
-apply from: 'deploy.gradle'
diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml
deleted file mode 100644
index ee4f9bd..0000000
--- a/config/checkstyle/checkstyle.xml
+++ /dev/null
@@ -1,209 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!DOCTYPE module PUBLIC
- "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
- "https://checkstyle.org/dtds/configuration_1_3.dtd">
-
-<!--
-
- Checkstyle configuration based on the Google coding conventions from:
-
- - Google Java Style
- https://google.github.io/styleguide/javaguide.html
-
- with Spark-specific changes from:
-
- https://spark.apache.org/contributing.html#code-style-guide
-
- Checkstyle is very configurable. Be sure to read the documentation at
- http://checkstyle.sf.net (or in your downloaded distribution).
-
- Most Checks are configurable, be sure to consult the documentation.
-
- To completely disable a check, just comment it out or delete it from the file.
-
- Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
-
- -->
-
-<module name="Checker">
- <property name="charset" value="UTF-8"/>
-
- <property name="severity" value="error"/>
-
- <property name="fileExtensions" value="java, properties, xml"/>
-
- <!-- Checks for whitespace -->
- <!-- See http://checkstyle.sf.net/config_whitespace.html -->
- <module name="FileTabCharacter">
- <property name="eachLine" value="true"/>
- </module>
-
- <module name="RegexpSingleline">
- <!-- \s matches whitespace character, $ matches end of line. -->
- <property name="format" value="\s+$"/>
- <property name="message" value="No trailing whitespace allowed."/>
- </module>
-
- <module name="LineLength">
- <property name="max" value="100"/>
- <property name="ignorePattern"
- value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
- </module>
-
- <module name="NewlineAtEndOfFile"/>
-
- <module name="TreeWalker">
- <!--
- If you wish to turn off checking for a section of code, you can put a comment in the source
- before and after the section, with the following syntax:
-
- // checkstyle.off: XXX (such as checkstyle.off: NoFinalizer)
- ... // stuff that breaks the styles
- // checkstyle.on: XXX (such as checkstyle.on: NoFinalizer)
- -->
- <module name="SuppressionCommentFilter">
- <property name="offCommentFormat" value="checkstyle\.off\: ([\w\|]+)"/>
- <property name="onCommentFormat" value="checkstyle\.on\: ([\w\|]+)"/>
- <property name="checkFormat" value="$1"/>
- </module>
- <module name="OuterTypeFilename"/>
- <module name="IllegalTokenText">
- <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
- <property name="format"
- value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
- <property name="message" value="Avoid using corresponding octal or Unicode escape."/>
- </module>
- <module name="AvoidEscapedUnicodeCharacters">
- <property name="allowEscapesForControlCharacters" value="true"/>
- <property name="allowByTailComment" value="true"/>
- <property name="allowNonPrintableEscapes" value="true"/>
- </module>
- <module name="NoLineWrap"/>
- <module name="EmptyBlock">
- <property name="option" value="TEXT"/>
- <property name="tokens"
- value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
- </module>
- <module name="NeedBraces">
- <property name="allowSingleLineStatement" value="true"/>
- </module>
- <module name="OneStatementPerLine"/>
- <module name="ArrayTypeStyle"/>
- <module name="FallThrough"/>
- <module name="UpperEll"/>
- <module name="ModifierOrder"/>
- <module name="SeparatorWrap">
- <property name="tokens" value="DOT"/>
- <property name="option" value="nl"/>
- </module>
- <module name="SeparatorWrap">
- <property name="tokens" value="COMMA"/>
- <property name="option" value="EOL"/>
- </module>
- <module name="PackageName">
- <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
- <message key="name.invalidPattern"
- value="Package name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="ClassTypeParameterName">
- <property name="format" value="([A-Z][a-zA-Z0-9]*$)"/>
- <message key="name.invalidPattern"
- value="Class type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="MethodTypeParameterName">
- <property name="format" value="([A-Z][a-zA-Z0-9]*)"/>
- <message key="name.invalidPattern"
- value="Method type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="GenericWhitespace">
- <message key="ws.followed"
- value="GenericWhitespace ''{0}'' is followed by whitespace."/>
- <message key="ws.preceded"
- value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
- <message key="ws.illegalFollow"
- value="GenericWhitespace ''{0}'' should followed by whitespace."/>
- <message key="ws.notPreceded"
- value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
- </module>
- <module name="MethodParamPad"/>
- <module name="AnnotationLocation">
- <property name="tokens"
- value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
- </module>
- <module name="AnnotationLocation">
- <property name="tokens" value="VARIABLE_DEF"/>
- <property name="allowSamelineMultipleAnnotations" value="true"/>
- </module>
- <module name="MethodName">
- <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
- <message key="name.invalidPattern"
- value="Method name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="EmptyCatchBlock">
- <property name="exceptionVariableName" value="expected"/>
- </module>
- <module name="CommentsIndentation"/>
- <module name="UnusedImports"/>
- <module name="RedundantImport"/>
- <module name="RedundantModifier"/>
- <module name="RegexpSinglelineJava">
- <property name="format" value="throw new \w+Error\("/>
- <property name="message" value="Avoid throwing error in application code."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="Objects\.toStringHelper"/>
- <property name="message"
- value="Avoid using Object.toStringHelper. Use ToStringBuilder instead."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="new (java\.lang\.)?(Byte|Integer|Long|Short)\("/>
- <property name="message"
- value="Use static factory 'valueOf' or 'parseXXX' instead of the deprecated constructors."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="Files\.createTempDir\("/>
- <property name="message"
- value="Avoid using com.google.common.io.Files.createTempDir() due to CVE-2020-8908.
- Use org.apache.spark.network.util.JavaUtils.createTempDir() instead."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="FileBackedOutputStream"/>
- <property name="message"
- value="Avoid using FileBackedOutputStream due to CVE-2023-2976."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="AtomicDoubleArray"/>
- <property name="message" value="Avoid using AtomicDoubleArray due to CVE-2018-10237."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="CompoundOrdering"/>
- <property name="message" value="Avoid using CompoundOrdering due to CVE-2018-10237."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="@Test\(expected"/>
- <property name="message"
- value="Please use the `assertThrows` method to test for exceptions."/>
- </module>
- <module name="IllegalImport">
- <property name="illegalPkgs" value="org.apache.log4j"/>
- <property name="illegalPkgs" value="org.apache.commons.lang"/>
- <property name="illegalPkgs" value="org.apache.commons.collections"/>
- </module>
- </module>
-</module>
diff --git a/config/pmd/ruleset.xml b/config/pmd/ruleset.xml
deleted file mode 100644
index 3f75485..0000000
--- a/config/pmd/ruleset.xml
+++ /dev/null
@@ -1,81 +0,0 @@
-<?xml version="1.0"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- ~
- -->
-
-<ruleset name="pmd ruleset">
- <description>
- Apache Spark Kubernetes Operator Ruleset
- </description>
- <rule ref="category/java/bestpractices.xml">
- <exclude name="JUnitAssertionsShouldIncludeMessage" />
- <exclude name="JUnitTestContainsTooManyAsserts" />
- </rule>
- <rule ref="category/java/codestyle.xml">
- <exclude name="AtLeastOneConstructor" />
- <exclude name="CommentDefaultAccessModifier" />
- <exclude name="FieldNamingConventions" />
- <exclude name="GenericsNaming" />
- <exclude name="LocalVariableCouldBeFinal" />
- <exclude name="LongVariable" />
- <exclude name="MethodArgumentCouldBeFinal" />
- <exclude name="OnlyOneReturn" />
- <exclude name="ShortVariable" />
- <exclude name="TooManyStaticImports" />
- <exclude name="UseUnderscoresInNumericLiterals" />
- </rule>
- <rule ref="category/java/design.xml">
- <exclude name="AbstractClassWithoutAnyMethod" />
- <exclude name="AvoidCatchingGenericException" />
- <exclude name="CognitiveComplexity" />
- <exclude name="CyclomaticComplexity" />
- <exclude name="ExcessiveImports" />
- <exclude name="LawOfDemeter" />
- <exclude name="NPathComplexity" />
- <exclude name="TooManyMethods" />
- </rule>
- <rule ref="category/java/documentation.xml">
- <exclude name="CommentRequired" />
- <exclude name="CommentSize" />
- </rule>
- <rule ref="category/java/errorprone.xml">
- <exclude name="AssignmentInOperand" />
- <exclude name="AvoidCatchingThrowable" />
- <exclude name="AvoidDuplicateLiterals" />
- <exclude name="AvoidFieldNameMatchingMethodName" />
- <exclude name="AvoidLiteralsInIfCondition" />
- <exclude name="CloseResource" />
- <exclude name="ConstructorCallsOverridableMethod" />
- <exclude name="MissingSerialVersionUID" />
- <exclude name="NullAssignment" />
- <exclude name="TestClassWithoutTestCases" />
- <exclude name="UseLocaleWithCaseConversions" />
- <exclude name="UseProperClassLoader" />
- </rule>
- <rule ref="category/java/multithreading.xml">
- <exclude name="DoNotUseThreads" />
- <exclude name="UseConcurrentHashMap" />
- </rule>
- <rule ref="category/java/performance.xml">
- <exclude name="AvoidFileStream" />
- <exclude name="AvoidInstantiatingObjectsInLoops" />
- <exclude name="RedundantFieldInitializer" />
- </rule>
- <rule ref="category/java/security.xml"/>
- <!-- exclude on generated files -->
- <exclude-pattern>.*/src/generated/.*</exclude-pattern>
-</ruleset>
diff --git a/config/spotbugs/spotbugs_exclude.xml b/config/spotbugs/spotbugs_exclude.xml
deleted file mode 100644
index c492bc2..0000000
--- a/config/spotbugs/spotbugs_exclude.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<FindBugsFilter>
- <Bug pattern="EI_EXPOSE_REP"/>
- <Bug pattern="EI_EXPOSE_REP2"/>
- <Bug pattern="MS_EXPOSE_REP"/>
- <Bug pattern="CT_CONSTRUCTOR_THROW"/>
-</FindBugsFilter>
diff --git a/deploy.gradle b/deploy.gradle
deleted file mode 100644
index be2e151..0000000
--- a/deploy.gradle
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-if (project.hasProperty('release') && JavaVersion.current().getMajorVersion().toString() != '17') {
- throw new GradleException("Releases must be built with Java 17")
-}
-
-subprojects {
- apply plugin: 'maven-publish'
- apply plugin: 'signing'
- afterEvaluate {
- task sourceJar(type: Jar, dependsOn: classes) {
- archiveClassifier.set('sources')
- from sourceSets.main.allSource
- group 'build'
- }
-
- task javadocJar(type: Jar, dependsOn: javadoc) {
- archiveClassifier.set('javadoc')
- from javadoc.destinationDir
- group 'build'
- }
-
- task testJar(type: Jar) {
- archiveClassifier.set('tests')
- from sourceSets.test.output
- }
-
- artifacts {
- archives sourceJar
- archives javadocJar
- archives testJar
- }
-
- // add LICENSE and NOTICE
- [jar, sourceJar, javadocJar, testJar].each { task ->
- task.from(rootDir) {
- include 'LICENSE'
- include 'NOTICE'
- }
- }
-
- publishing {
- publications {
- apache(MavenPublication) {
- if (tasks.matching({task -> task.name == 'shadowJar'}).isEmpty()) {
- from components.java
- } else {
- project.shadow.component(it)
- }
-
- artifact sourceJar
- artifact javadocJar
- artifact testJar
-
- versionMapping {
- allVariants {
- fromResolutionResult()
- }
- }
-
- groupId = 'org.apache.spark'
- pom {
- name = 'Apache Spark'
- description = 'A fast and general purpose engine for large-scale data processin'
- url = 'https://spark.apache.org'
- licenses {
- license {
- name = 'The Apache Software License, Version 2.0'
- url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
- }
- }
- mailingLists {
- mailingList {
- name = 'Dev Mailing List'
- post = 'dev@spark.apache.org'
- subscribe = 'dev-subscribe@spark.apache.org'
- unsubscribe = 'dev-unsubscribe@spark.apache.org'
- }
- }
- issueManagement {
- system = 'JIRA'
- url = 'https://issues.apache.org/jira/browse/SPARK'
- }
- }
- }
- }
-
- repositories {
- maven {
- credentials {
- username project.hasProperty('mavenUser') ? "$mavenUser" : ""
- password project.hasProperty('mavenPassword') ? "$mavenPassword" : ""
- }
- // upload to the releases repository using ./gradlew -Prelease publish
- def apacheSnapshotsRepoUrl = 'https://repository.apache.org/content/repositories/snapshots'
- def apacheReleasesRepoUrl = 'https://repository.apache.org/service/local/staging/deploy/maven2'
- def snapshotsRepoUrl = project.hasProperty('mavenSnapshotsRepo') ? "$mavenSnapshotsRepo" : "$apacheSnapshotsRepoUrl"
- def releasesRepoUrl = project.hasProperty('mavenReleasesRepo') ? "$mavenReleasesRepo" : "$apacheReleasesRepoUrl"
- url = project.hasProperty('release') ? releasesRepoUrl : snapshotsRepoUrl
- }
- }
- }
-
- if (project.hasProperty('release')) {
- signing {
- useGpgCmd()
- sign publishing.publications.apache
- }
- }
- }
-}
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
deleted file mode 100644
index 4234634..0000000
--- a/dev/.rat-excludes
+++ /dev/null
@@ -1,18 +0,0 @@
-target
-.gitignore
-.gitattributes
-.project
-.classpath
-.rat-excludes
-.*md
-.java-version
-licenses/*
-licenses-binary/*
-LICENSE
-NOTICE
-TAGS
-RELEASE
-build
-.*gradle
-.*json
-.helmignore
diff --git a/dev/check-license b/dev/check-license
deleted file mode 100755
index bc7f493..0000000
--- a/dev/check-license
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-acquire_rat_jar () {
-
- URL="${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"
-
- JAR="$rat_jar"
-
- # Download rat launch jar if it hasn't been downloaded yet
- if [ ! -f "$JAR" ]; then
- # Download
- printf "Attempting to fetch rat\n"
- JAR_DL="${JAR}.part"
- if [ $(command -v curl) ]; then
- curl -L --silent "${URL}" > "$JAR_DL" && mv "$JAR_DL" "$JAR"
- elif [ $(command -v wget) ]; then
- wget --quiet ${URL} -O "$JAR_DL" && mv "$JAR_DL" "$JAR"
- else
- printf "You do not have curl or wget installed, please install rat manually.\n"
- exit -1
- fi
- fi
-
- unzip -tq "$JAR" &> /dev/null
- if [ $? -ne 0 ]; then
- # We failed to download
- rm "$JAR"
- printf "Our attempt to download rat locally to ${JAR} failed. Please install rat manually.\n"
- exit -1
- fi
-}
-
-# Go to the Spark project root directory
-FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
-cd "$FWDIR"
-
-if test -x "$JAVA_HOME/bin/java"; then
- declare java_cmd="$JAVA_HOME/bin/java"
-else
- declare java_cmd=java
-fi
-
-export RAT_VERSION=0.15
-export rat_jar="$FWDIR"/lib/apache-rat-${RAT_VERSION}.jar
-mkdir -p "$FWDIR"/lib
-
-[[ -f "$rat_jar" ]] || acquire_rat_jar || {
- echo "Download failed. Obtain the rat jar manually and place it at $rat_jar"
- exit 1
-}
-
-mkdir -p target
-$java_cmd -jar "$rat_jar" -E "$FWDIR"/dev/.rat-excludes -d "$FWDIR" > target/rat-results.txt
-
-if [ $? -ne 0 ]; then
- echo "RAT exited abnormally"
- exit 1
-fi
-
-ERRORS="$(cat target/rat-results.txt | grep -e "??")"
-
-if test ! -z "$ERRORS"; then
- echo "Could not find Apache license headers in the following files:"
- echo "$ERRORS"
- exit 1
-else
- echo -e "RAT checks passed."
-fi
diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py
deleted file mode 100755
index 9a8d39f..0000000
--- a/dev/merge_spark_pr.py
+++ /dev/null
@@ -1,719 +0,0 @@
-#!/usr/bin/env python3
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Utility for creating well-formed pull request merges and pushing them to Apache
-# Spark.
-# usage: ./merge_spark_pr.py (see config env vars below)
-#
-# This utility assumes you already have a local Spark git folder and that you
-# have added remotes corresponding to both (i) the github apache Spark
-# mirror and (ii) the apache git repo.
-
-import json
-import os
-import re
-import subprocess
-import sys
-import traceback
-from urllib.request import urlopen
-from urllib.request import Request
-from urllib.error import HTTPError
-
-try:
- import jira.client
-
- JIRA_IMPORTED = True
-except ImportError:
- JIRA_IMPORTED = False
-
-# Location of your Spark git development area
-SPARK_HOME = os.environ.get("SPARK_HOME", os.getcwd())
-# Remote name which points to the Github site
-PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github")
-# Remote name which points to Apache git
-PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache")
-# ASF JIRA username
-JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "")
-# ASF JIRA password
-JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", "")
-# ASF JIRA access token
-# If it is configured, username and password are dismissed
-# Go to https://issues.apache.org/jira/secure/ViewProfile.jspa -> Personal Access Tokens for
-# your own token management.
-JIRA_ACCESS_TOKEN = os.environ.get("JIRA_ACCESS_TOKEN")
-# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests
-# will be unauthenticated. You should only need to configure this if you find yourself regularly
-# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at
-# https://github.com/settings/tokens. This script only requires the "public_repo" scope.
-GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY")
-
-
-GITHUB_BASE = "https://github.com/apache/spark-kubernetes-operator/pull"
-GITHUB_API_BASE = "https://api.github.com/repos/apache/spark-kubernetes-operator"
-JIRA_BASE = "https://issues.apache.org/jira/browse"
-JIRA_API_BASE = "https://issues.apache.org/jira"
-# Prefix added to temporary branches
-BRANCH_PREFIX = "PR_TOOL"
-
-
-def print_error(msg):
- print("\033[91m%s\033[0m" % msg)
-
-
-def bold_input(prompt) -> str:
- return input("\033[1m%s\033[0m" % prompt)
-
-
-def get_json(url):
- try:
- request = Request(url)
- if GITHUB_OAUTH_KEY:
- request.add_header("Authorization", "token %s" % GITHUB_OAUTH_KEY)
- return json.load(urlopen(request))
- except HTTPError as e:
- if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] == "0":
- print_error(
- "Exceeded the GitHub API rate limit; see the instructions in "
- + "dev/merge_spark_pr.py to configure an OAuth token for making authenticated "
- + "GitHub requests."
- )
- elif e.code == 401:
- print_error(
- "GITHUB_OAUTH_KEY is invalid or expired. Please regenerate a new one with "
- + "at least the 'public_repo' scope on https://github.com/settings/tokens and "
- + "update your local settings before you try again."
- )
- else:
- print_error("Unable to fetch URL, exiting: %s" % url)
- sys.exit(-1)
-
-
-def fail(msg):
- print_error(msg)
- clean_up()
- sys.exit(-1)
-
-
-def run_cmd(cmd):
- print(cmd)
- if isinstance(cmd, list):
- return subprocess.check_output(cmd).decode("utf-8")
- else:
- return subprocess.check_output(cmd.split(" ")).decode("utf-8")
-
-
-def continue_maybe(prompt):
- result = bold_input("%s (y/N): " % prompt)
- if result.lower() != "y":
- fail("Okay, exiting")
-
-
-def clean_up():
- if "original_head" in globals():
- print("Restoring head pointer to %s" % original_head)
- run_cmd("git checkout %s" % original_head)
-
- branches = run_cmd("git branch").replace(" ", "").split("\n")
-
- for branch in list(filter(lambda x: x.startswith(BRANCH_PREFIX), branches)):
- print("Deleting local branch %s" % branch)
- run_cmd("git branch -D %s" % branch)
-
-
-# merge the requested PR and return the merge hash
-def merge_pr(pr_num, target_ref, title, body, pr_repo_desc):
- pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
- target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
- run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name))
- run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name))
- run_cmd("git checkout %s" % target_branch_name)
-
- had_conflicts = False
- try:
- run_cmd(["git", "merge", pr_branch_name, "--squash"])
- except Exception as e:
- msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
- continue_maybe(msg)
- msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
- continue_maybe(msg)
- had_conflicts = True
-
- # First commit author should be considered as the primary author when the rank is the same
- commit_authors = run_cmd(
- ["git", "log", "HEAD..%s" % pr_branch_name, "--pretty=format:%an <%ae>", "--reverse"]
- ).split("\n")
- distinct_authors = sorted(
- list(dict.fromkeys(commit_authors)), key=lambda x: commit_authors.count(x), reverse=True
- )
- primary_author = bold_input(
- 'Enter primary author in the format of "name <email>" [%s]: ' % distinct_authors[0]
- )
- if primary_author == "":
- primary_author = distinct_authors[0]
- else:
- # When primary author is specified manually, de-dup it from author list and
- # put it at the head of author list.
- distinct_authors = list(filter(lambda x: x != primary_author, distinct_authors))
- distinct_authors.insert(0, primary_author)
-
- merge_message_flags = []
-
- merge_message_flags += ["-m", title]
- if body is not None:
- # We remove @ symbols from the body to avoid triggering e-mails
- # to people every time someone creates a public fork of Spark.
- merge_message_flags += ["-m", body.replace("@", "")]
-
- committer_name = run_cmd("git config --get user.name").strip()
- committer_email = run_cmd("git config --get user.email").strip()
-
- if had_conflicts:
- message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % (
- committer_name,
- committer_email,
- )
- merge_message_flags += ["-m", message]
-
- # The string "Closes #%s" string is required for GitHub to correctly close the PR
- merge_message_flags += ["-m", "Closes #%s from %s." % (pr_num, pr_repo_desc)]
-
- authors = "Authored-by:" if len(distinct_authors) == 1 else "Lead-authored-by:"
- authors += " %s" % (distinct_authors.pop(0))
- if len(distinct_authors) > 0:
- authors += "\n" + "\n".join(["Co-authored-by: %s" % a for a in distinct_authors])
- authors += "\n" + "Signed-off-by: %s <%s>" % (committer_name, committer_email)
-
- merge_message_flags += ["-m", authors]
-
- run_cmd(["git", "commit", '--author="%s"' % primary_author] + merge_message_flags)
-
- continue_maybe(
- "Merge complete (local ref %s). Push to %s?" % (target_branch_name, PUSH_REMOTE_NAME)
- )
-
- try:
- run_cmd("git push %s %s:%s" % (PUSH_REMOTE_NAME, target_branch_name, target_ref))
- except Exception as e:
- clean_up()
- print_error("Exception while pushing: %s" % e)
-
- merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8]
- clean_up()
- print("Pull request #%s merged!" % pr_num)
- print("Merge hash: %s" % merge_hash)
- return merge_hash
-
-
-def cherry_pick(pr_num, merge_hash, default_branch):
- pick_ref = bold_input("Enter a branch name [%s]: " % default_branch)
- if pick_ref == "":
- pick_ref = default_branch
-
- pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper())
-
- run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
- run_cmd("git checkout %s" % pick_branch_name)
-
- try:
- run_cmd("git cherry-pick -sx %s" % merge_hash)
- except Exception as e:
- msg = "Error cherry-picking: %s\nWould you like to manually fix-up this merge?" % e
- continue_maybe(msg)
- msg = "Okay, please fix any conflicts and finish the cherry-pick. Finished?"
- continue_maybe(msg)
-
- continue_maybe(
- "Pick complete (local ref %s). Push to %s?" % (pick_branch_name, PUSH_REMOTE_NAME)
- )
-
- try:
- run_cmd("git push %s %s:%s" % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref))
- except Exception as e:
- clean_up()
- fail("Exception while pushing: %s" % e)
-
- pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8]
- clean_up()
-
- print("Pull request #%s picked into %s!" % (pr_num, pick_ref))
- print("Pick hash: %s" % pick_hash)
- return pick_ref
-
-
-def print_jira_issue_summary(issue):
- summary = issue.fields.summary
- assignee = issue.fields.assignee
- if assignee is not None:
- assignee = assignee.displayName
- status = issue.fields.status.name
- print("=== JIRA %s ===" % issue.key)
- print(
- "summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n"
- % (summary, assignee, status, JIRA_BASE, issue.key)
- )
-
-
-def get_jira_issue(prompt, default_jira_id=""):
- jira_id = bold_input("%s [%s]: " % (prompt, default_jira_id))
- if jira_id == "":
- jira_id = default_jira_id
- if jira_id == "":
- print("JIRA ID not found, skipping.")
- return None
- try:
- issue = asf_jira.issue(jira_id)
- print_jira_issue_summary(issue)
- status = issue.fields.status.name
- if status == "Resolved" or status == "Closed":
- print("JIRA issue %s already has status '%s'" % (jira_id, status))
- return None
- if bold_input("Check if the JIRA information is as expected (y/N): ").lower() == "y":
- return issue
- else:
- return get_jira_issue("Enter the revised JIRA ID again or leave blank to skip")
- except Exception as e:
- print_error("ASF JIRA could not find %s: %s" % (jira_id, e))
- return get_jira_issue("Enter the revised JIRA ID again or leave blank to skip")
-
-
-def resolve_jira_issue(merge_branches, comment, default_jira_id=""):
- issue = get_jira_issue("Enter a JIRA id", default_jira_id)
- if issue is None:
- return
-
- if issue.fields.assignee is None:
- choose_jira_assignee(issue)
-
- versions = asf_jira.project_versions("SPARK")
- # Consider only x.y.z, unreleased, unarchived versions
- versions = [
- x
- for x in versions
- if not x.raw["released"]
- and not x.raw["archived"]
- and re.match(r"kubernetes-operator-\d+\.\d+\.\d+", x.name)
- ]
- versions = sorted(versions, key=lambda x: x.name, reverse=True)
-
- default_fix_versions = []
- for b in merge_branches:
- if b == "main":
- default_fix_versions.append(versions[0].name)
- else:
- found = False
- found_versions = []
- for v in versions:
- if v.name.startswith(b.replace("branch-", "")):
- found_versions.append(v.name)
- found = True
- if found:
- # There might be several unreleased versions for specific branches
- # For example, assuming
- # versions = ['4.0.0', '3.5.1', '3.5.0', '3.4.2', '3.3.4', '3.3.3']
- # we've found two candidates for branch-3.5, we pick the last/smallest one
- default_fix_versions.append(found_versions[-1])
- else:
- print_error(
- "Target version for %s is not found on JIRA, it may be archived or "
- "not created. Skipping it." % b
- )
-
- for v in default_fix_versions:
- # Handles the case where we have forked a release branch but not yet made the release.
- # In this case, if the PR is committed to the main branch and the release branch, we
- # only consider the release branch to be the fix version. E.g. it is not valid to have
- # both 1.1.0 and 1.0.0 as fix versions.
- (major, minor, patch) = v.split(".")
- if patch == "0":
- previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
- if previous in default_fix_versions:
- default_fix_versions = list(filter(lambda x: x != v, default_fix_versions))
- default_fix_versions = ",".join(default_fix_versions)
-
- available_versions = set(list(map(lambda v: v.name, versions)))
- while True:
- try:
- fix_versions = bold_input(
- "Enter comma-separated fix version(s) [%s]: " % default_fix_versions
- )
- if fix_versions == "":
- fix_versions = default_fix_versions
- fix_versions = fix_versions.replace(" ", "").split(",")
- if set(fix_versions).issubset(available_versions):
- break
- else:
- print(
- "Specified version(s) [%s] not found in the available versions, try "
- "again (or leave blank and fix manually)." % (", ".join(fix_versions))
- )
- except KeyboardInterrupt:
- raise
- except BaseException:
- traceback.print_exc()
- print("Error setting fix version(s), try again (or leave blank and fix manually)")
-
- def get_version_json(version_str):
- return list(filter(lambda v: v.name == version_str, versions))[0].raw
-
- jira_fix_versions = list(map(lambda v: get_version_json(v), fix_versions))
-
- resolve = list(filter(lambda a: a["name"] == "Resolve Issue", asf_jira.transitions(issue.key)))[
- 0
- ]
- resolution = list(filter(lambda r: r.raw["name"] == "Fixed", asf_jira.resolutions()))[0]
- asf_jira.transition_issue(
- issue.key,
- resolve["id"],
- fixVersions=jira_fix_versions,
- comment=comment,
- resolution={"id": resolution.raw["id"]},
- )
-
- try:
- print_jira_issue_summary(asf_jira.issue(issue.key))
- except Exception:
- print("Unable to fetch JIRA issue %s after resolving" % issue.key)
- print("Successfully resolved %s with fixVersions=%s!" % (issue.key, fix_versions))
-
-
-def choose_jira_assignee(issue):
- """
- Prompt the user to choose who to assign the issue to in jira, given a list of candidates,
- including the original reporter and all commentators
- """
- while True:
- try:
- reporter = issue.fields.reporter
- commentators = list(map(lambda x: x.author, issue.fields.comment.comments))
- candidates = set(commentators)
- candidates.add(reporter)
- candidates = list(candidates)
- print("JIRA is unassigned, choose assignee")
- for idx, author in enumerate(candidates):
- if author.key == "apachespark":
- continue
- annotations = ["Reporter"] if author == reporter else []
- if author in commentators:
- annotations.append("Commentator")
- print("[%d] %s (%s)" % (idx, author.displayName, ",".join(annotations)))
- raw_assignee = bold_input(
- "Enter number of user, or userid, to assign to (blank to leave unassigned):"
- )
- if raw_assignee == "":
- return None
- else:
- try:
- id = int(raw_assignee)
- assignee = candidates[id]
- except BaseException:
- # assume it's a user id, and try to assign (might fail, we just prompt again)
- assignee = asf_jira.user(raw_assignee)
- try:
- assign_issue(issue.key, assignee.name)
- except Exception as e:
- if (
- e.__class__.__name__ == "JIRAError"
- and ("'%s' cannot be assigned" % assignee.name)
- in getattr(e, "response").text
- ):
- continue_maybe(
- "User '%s' cannot be assigned, add to contributors role and try again?"
- % assignee.name
- )
- grant_contributor_role(assignee.name)
- assign_issue(issue.key, assignee.name)
- else:
- raise e
- return assignee
- except KeyboardInterrupt:
- raise
- except BaseException:
- traceback.print_exc()
- print("Error assigning JIRA, try again (or leave blank and fix manually)")
-
-
-def grant_contributor_role(user: str):
- role = asf_jira.project_role("SPARK", 10010)
- role.add_user(user)
- print("Successfully added user '%s' to contributors role" % user)
-
-
-def assign_issue(issue: int, assignee: str) -> bool:
- """
- Assign an issue to a user, which is a shorthand for jira.client.JIRA.assign_issue.
- The original one has an issue that it will search users again and only choose the assignee
- from 20 candidates. If it's unmatched, it picks the head blindly. In our case, the assignee
- is already resolved.
- """
- url = getattr(asf_jira, "_get_latest_url")(f"issue/{issue}/assignee")
- payload = {"name": assignee}
- getattr(asf_jira, "_session").put(url, data=json.dumps(payload))
- return True
-
-
-def resolve_jira_issues(title, merge_branches, comment):
- jira_ids = re.findall("SPARK-[0-9]{4,5}", title)
-
- if len(jira_ids) == 0:
- resolve_jira_issue(merge_branches, comment)
- for jira_id in jira_ids:
- resolve_jira_issue(merge_branches, comment, jira_id)
-
-
-def standardize_jira_ref(text):
- """
- Standardize the [SPARK-XXXXX] [MODULE] prefix
- Converts "[SPARK-XXX][mllib] Issue", "[MLLib] SPARK-XXX. Issue" or "SPARK XXX [MLLIB]: Issue" to
- "[SPARK-XXX][MLLIB] Issue"
-
- >>> standardize_jira_ref(
- ... "[SPARK-5821] [SQL] ParquetRelation2 CTAS should check if delete is successful")
- '[SPARK-5821][SQL] ParquetRelation2 CTAS should check if delete is successful'
- >>> standardize_jira_ref(
- ... "[SPARK-4123][Project Infra][WIP]: Show new dependencies added in pull requests")
- '[SPARK-4123][PROJECT INFRA][WIP] Show new dependencies added in pull requests'
- >>> standardize_jira_ref("[MLlib] Spark 5954: Top by key")
- '[SPARK-5954][MLLIB] Top by key'
- >>> standardize_jira_ref("[SPARK-979] a LRU scheduler for load balancing in TaskSchedulerImpl")
- '[SPARK-979] a LRU scheduler for load balancing in TaskSchedulerImpl'
- >>> standardize_jira_ref(
- ... "SPARK-1094 Support MiMa for reporting binary compatibility across versions.")
- '[SPARK-1094] Support MiMa for reporting binary compatibility across versions.'
- >>> standardize_jira_ref("[WIP] [SPARK-1146] Vagrant support for Spark")
- '[SPARK-1146][WIP] Vagrant support for Spark'
- >>> standardize_jira_ref(
- ... "SPARK-1032. If Yarn app fails before registering, app master stays aroun...")
- '[SPARK-1032] If Yarn app fails before registering, app master stays aroun...'
- >>> standardize_jira_ref(
- ... "[SPARK-6250][SPARK-6146][SPARK-5911][SQL] Types are now reserved words in DDL parser.")
- '[SPARK-6250][SPARK-6146][SPARK-5911][SQL] Types are now reserved words in DDL parser.'
- >>> standardize_jira_ref("Additional information for users building from source code")
- 'Additional information for users building from source code'
- """
- jira_refs = []
- components = []
-
- # If the string is compliant, no need to process any further
- if re.search(r"^\[SPARK-[0-9]{3,6}\](\[[A-Z0-9_\s,]+\] )+\S+", text):
- return text
-
- # Extract JIRA ref(s):
- pattern = re.compile(r"(SPARK[-\s]*[0-9]{3,6})+", re.IGNORECASE)
- for ref in pattern.findall(text):
- # Add brackets, replace spaces with a dash, & convert to uppercase
- jira_refs.append("[" + re.sub(r"\s+", "-", ref.upper()) + "]")
- text = text.replace(ref, "")
-
- # Extract spark component(s):
- # Look for alphanumeric chars, spaces, dashes, periods, and/or commas
- pattern = re.compile(r"(\[[\w\s,.-]+\])", re.IGNORECASE)
- for component in pattern.findall(text):
- components.append(component.upper())
- text = text.replace(component, "")
-
- # Cleanup any remaining symbols:
- pattern = re.compile(r"^\W+(.*)", re.IGNORECASE)
- if pattern.search(text) is not None:
- text = pattern.search(text).groups()[0]
-
- # Assemble full text (JIRA ref(s), module(s), remaining text)
- clean_text = "".join(jira_refs).strip() + "".join(components).strip() + " " + text.strip()
-
- # Replace multiple spaces with a single space, e.g. if no jira refs and/or components were
- # included
- clean_text = re.sub(r"\s+", " ", clean_text.strip())
-
- return clean_text
-
-
-def get_current_ref():
- ref = run_cmd("git rev-parse --abbrev-ref HEAD").strip()
- if ref == "HEAD":
- # The current ref is a detached HEAD, so grab its SHA.
- return run_cmd("git rev-parse HEAD").strip()
- else:
- return ref
-
-
-def initialize_jira():
- global asf_jira
- jira_server = {"server": JIRA_API_BASE}
-
- if not JIRA_IMPORTED:
- print_error("ERROR finding jira library. Run 'pip3 install jira' to install.")
- continue_maybe("Continue without jira?")
- elif JIRA_ACCESS_TOKEN:
- client = jira.client.JIRA(jira_server, token_auth=JIRA_ACCESS_TOKEN)
- try:
- # Eagerly check if the token is valid to align with the behavior of username/password
- # authn
- client.current_user()
- asf_jira = client
- except Exception as e:
- if e.__class__.__name__ == "JIRAError" and getattr(e, "status_code", None) == 401:
- msg = (
- "ASF JIRA could not authenticate with the invalid or expired token '%s'"
- % JIRA_ACCESS_TOKEN
- )
- fail(msg)
- else:
- raise e
- elif JIRA_USERNAME and JIRA_PASSWORD:
- print("You can use JIRA_ACCESS_TOKEN instead of JIRA_USERNAME/JIRA_PASSWORD.")
- print("Visit https://issues.apache.org/jira/secure/ViewProfile.jspa ")
- print("and click 'Personal Access Tokens' menu to manage your own tokens.")
- asf_jira = jira.client.JIRA(jira_server, basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
- else:
- print("Neither JIRA_ACCESS_TOKEN nor JIRA_USERNAME/JIRA_PASSWORD are set.")
- continue_maybe("Continue without jira?")
-
-
-def main():
- initialize_jira()
- global original_head
-
- os.chdir(SPARK_HOME)
- original_head = get_current_ref()
-
- branches = get_json("%s/branches" % GITHUB_API_BASE)
- branch_names = list(filter(lambda x: x.startswith("branch-"), [x["name"] for x in branches]))
- # Assumes branch names can be sorted lexicographically
- branch_names = sorted(branch_names, reverse=True)
- branch_iter = iter(branch_names)
-
- pr_num = bold_input("Which pull request would you like to merge? (e.g. 34): ")
- pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num))
- pr_events = get_json("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num))
-
- url = pr["url"]
-
- # Warn if the PR is WIP
- if "[WIP]" in pr["title"]:
- msg = "The PR title has `[WIP]`:\n%s\nContinue?" % pr["title"]
- continue_maybe(msg)
-
- # Decide whether to use the modified title or not
- modified_title = standardize_jira_ref(pr["title"]).rstrip(".")
- if modified_title != pr["title"]:
- print("I've re-written the title as follows to match the standard format:")
- print("Original: %s" % pr["title"])
- print("Modified: %s" % modified_title)
- result = bold_input("Would you like to use the modified title? (y/N): ")
- if result.lower() == "y":
- title = modified_title
- print("Using modified title:")
- else:
- title = pr["title"]
- print("Using original title:")
- print(title)
- else:
- title = pr["title"]
-
- body = pr["body"]
- if body is None:
- body = ""
- modified_body = re.sub(re.compile(r"<!--[^>]*-->\n?", re.DOTALL), "", body).lstrip()
- if modified_body != body:
- print("=" * 80)
- print(modified_body)
- print("=" * 80)
- print("I've removed the comments from PR template like the above:")
- result = bold_input("Would you like to use the modified body? (y/N): ")
- if result.lower() == "y":
- body = modified_body
- print("Using modified body:")
- else:
- print("Using original body:")
- print("=" * 80)
- print(body)
- print("=" * 80)
- target_ref = pr["base"]["ref"]
- user_login = pr["user"]["login"]
- base_ref = pr["head"]["ref"]
- pr_repo_desc = "%s/%s" % (user_login, base_ref)
-
- # Merged pull requests don't appear as merged in the GitHub API;
- # Instead, they're closed by committers.
- merge_commits = [e for e in pr_events if e["event"] == "closed" and e["commit_id"] is not None]
-
- if merge_commits and pr["state"] == "closed":
- # A PR might have multiple merge commits, if it's reopened and merged again. We shall
- # cherry-pick PRs in closed state with the latest merge hash.
- # If the PR is still open(reopened), we shall not cherry-pick it but perform the normal
- # merge as it could have been reverted earlier.
- merge_commits = sorted(merge_commits, key=lambda x: x["created_at"])
- merge_hash = merge_commits[-1]["commit_id"]
- message = get_json("%s/commits/%s" % (GITHUB_API_BASE, merge_hash))["commit"]["message"]
-
- print("Pull request %s has already been merged, assuming you want to backport" % pr_num)
- commit_is_downloaded = (
- run_cmd(["git", "rev-parse", "--quiet", "--verify", "%s^{commit}" % merge_hash]).strip()
- != ""
- )
- if not commit_is_downloaded:
- fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num)
-
- print("Found commit %s:\n%s" % (merge_hash, message))
- cherry_pick(pr_num, merge_hash, next(branch_iter, branch_names[0]))
- sys.exit(0)
-
- if not bool(pr["mergeable"]):
- msg = (
- "Pull request %s is not mergeable in its current form.\n" % pr_num
- + "Continue? (experts only!)"
- )
- continue_maybe(msg)
-
- print("\n=== Pull Request #%s ===" % pr_num)
- print("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % (title, pr_repo_desc, target_ref, url))
- continue_maybe("Proceed with merging pull request #%s?" % pr_num)
-
- merged_refs = [target_ref]
-
- merge_hash = merge_pr(pr_num, target_ref, title, body, pr_repo_desc)
-
- pick_prompt = "Would you like to pick %s into another branch?" % merge_hash
- while bold_input("\n%s (y/N): " % pick_prompt).lower() == "y":
- merged_refs = merged_refs + [
- cherry_pick(pr_num, merge_hash, next(branch_iter, branch_names[0]))
- ]
-
- if asf_jira is not None:
- continue_maybe("Would you like to update an associated JIRA?")
- jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (
- pr_num,
- GITHUB_BASE,
- pr_num,
- )
- resolve_jira_issues(title, merged_refs, jira_comment)
- else:
- print("Exiting without trying to close the associated JIRA.")
-
-
-if __name__ == "__main__":
- import doctest
-
- (failure_count, test_count) = doctest.testmod()
- if failure_count:
- sys.exit(-1)
- try:
- main()
- except BaseException:
- clean_up()
- raise
diff --git a/docs/architecture.md b/docs/architecture.md
deleted file mode 100644
index 0539355..0000000
--- a/docs/architecture.md
+++ /dev/null
@@ -1,81 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-# Design & Architecture
-
-**Spark-Kubernetes-Operator** (Operator) acts as a control plane to manage the complete
-deployment lifecycle of Spark applications and clusters. The Operator can be installed on Kubernetes
-cluster(s) using Helm. In most production environments it is typically deployed in a designated
-namespace and controls Spark workload in one or more managed namespaces.
-Spark Operator enables user to describe Spark application(s) or cluster(s) as
-[Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/).
-
-The Operator continuously tracks events related to the Spark custom resources in its reconciliation
-loops:
-
-For SparkApplications:
-
-* User submits a SparkApplication custom resource(CR) using kubectl / API
-* Operator launches driver and observes its status
-* Operator observes driver-spawn resources (e.g. executors) and record status till app terminates
-* Operator releases all Spark-app owned resources to cluster
-
-For SparkClusters:
-
-* User submits a SparkCluster custom resource(CR) using kubectl / API
-* Operator launches master and worker(s) based on CR spec and observes their status
-* Operator releases all Spark-cluster owned resources to cluster upon failure
-
-The Operator is built with the [Java Operator SDK](https://javaoperatorsdk.io/) for
-launching Spark deployments and submitting jobs under the hood. It also uses
-[fabric8](https://fabric8.io/) client to interact with Kubernetes API Server.
-
-## Application State Transition
-
-[<img src="resources/application_state_machine.png">](resources/application_state_machine.png)
-
-* Spark applications are expected to run from submitted to succeeded before releasing resources
-* User may configure the app CR to time-out after given threshold of time if it cannot reach healthy
- state after given threshold. The timeout can be configured for different lifecycle stages,
- when driver starting and when requesting executor pods. To update the default threshold,
- configure `.spec.applicationTolerations.applicationTimeoutConfig` for the application.
-* K8s resources created for an application would be deleted as the final stage of the application
- lifecycle by default. This is to ensure resource quota release for completed applications.
-* It is also possible to retain the created k8s resources for debug or audit purpose. To do so,
- user may set `.spec.applicationTolerations.resourceRetainPolicy` to `OnFailure` to retain
- resources upon application failure, or set to `Always` to retain resources regardless of
- application final state.
- - This controls the behavior of k8s resources created by Operator for the application, including
- driver pod, config map, service, and PVC(if enabled). This does not apply to resources created
- by driver (for example, executor pods). User may configure SparkConf to
- include `spark.kubernetes.executor.deleteOnTermination` for executor retention. Please refer
- [Spark docs](https://spark.apache.org/docs/latest/running-on-kubernetes.html) for details.
- - The created k8s resources have `ownerReference` to their related `SparkApplication` custom
- resource, such that they could be garbage collected when the `SparkApplication` is deleted.
- - Please be advised that k8s resources would not be retained if the application is configured to
- restart. This is to avoid resource quota usage increase unexpectedly or resource conflicts
- among multiple attempts.
-
-## Cluster State Transition
-
-[<img src="resources/cluster_state_machine.png">](resources/application_state_machine.png)
-
-* Spark clusters are expected to be always running after submitted.
-* Similar to Spark applications, K8s resources created for a cluster would be deleted as the final
- stage of the cluster lifecycle by default.
diff --git a/docs/config_properties.md b/docs/config_properties.md
deleted file mode 100644
index c86ada2..0000000
--- a/docs/config_properties.md
+++ /dev/null
@@ -1,40 +0,0 @@
-[//]: # (This doc is automatically generated by gradle task, manual updates would be overridden.)
-# Spark Operator Config Properties
- | Key | Type | Default Value | Allow Hot Reloading | Description |
- | --- | --- | --- | --- | --- |
- | spark.kubernetes.operator.name | String | spark-kubernetes-operator | false | Name of the operator. |
- | spark.kubernetes.operator.namespace | String | default | false | Namespace that operator is deployed within. |
- | spark.kubernetes.operator.watchedNamespaces | String | default | true | Comma-separated list of namespaces that the operator would be watching for Spark resources. If set to '*', operator would watch all namespaces. |
- | spark.kubernetes.operator.terminateOnInformerFailureEnabled | Boolean | false | false | Enable to indicate informer errors should stop operator startup. If disabled, operator startup will ignore recoverable errors, caused for example by RBAC issues and will retry periodically. |
- | spark.kubernetes.operator.reconciler.terminationTimeoutSeconds | Integer | 30 | false | Grace period for operator shutdown before reconciliation threads are killed. |
- | spark.kubernetes.operator.reconciler.parallelism | Integer | 50 | false | Thread pool size for Spark Operator reconcilers. Unbounded pool would be used if set to non-positive number. |
- | spark.kubernetes.operator.reconciler.foregroundRequestTimeoutSeconds | Long | 30 | true | Timeout (in seconds) to for requests made to API server. This applies only to foreground requests. |
- | spark.kubernetes.operator.reconciler.intervalSeconds | Long | 120 | true | Interval (in seconds, non-negative) to reconcile Spark applications. Note that reconciliation is always expected to be triggered when app spec / status is updated. This interval controls the reconcile behavior of operator reconciliation even when there's no update on SparkApplication, e.g. to determine whether a hanging app needs to be proactively terminated. Thus this is recommended to set to above 2 minutes t [...]
- | spark.kubernetes.operator.reconciler.trimStateTransitionHistoryEnabled | Boolean | true | true | When enabled, operator would trim state transition history when a new attempt starts, keeping previous attempt summary only. |
- | spark.kubernetes.operator.reconciler.appStatusListenerClassNames | String | | false | Comma-separated names of SparkAppStatusListener class implementations |
- | spark.kubernetes.operator.reconciler.clusterStatusListenerClassNames | String | | false | Comma-separated names of SparkClusterStatusListener class implementations |
- | spark.kubernetes.operator.dynamicConfig.enabled | Boolean | false | false | When enabled, operator would use config map as source of truth for config property override. The config map need to be created in spark.kubernetes.operator.namespace, and labeled with operator name. |
- | spark.kubernetes.operator.dynamicConfig.selector | String | app.kubernetes.io/name=spark-kubernetes-operator,app.kubernetes.io/component=operator-dynamic-config-overrides | false | The selector str applied to dynamic config map. |
- | spark.kubernetes.operator.dynamicConfig.reconcilerParallelism | Integer | 1 | false | Parallelism for dynamic config reconciler. Unbounded pool would be used if set to non-positive number. |
- | spark.kubernetes.operator.reconciler.rateLimiter.refreshPeriodSeconds | Integer | 15 | false | Operator rate limiter refresh period(in seconds) for each resource. |
- | spark.kubernetes.operator.reconciler.rateLimiter.maxLoopForPeriod | Integer | 5 | false | Max number of reconcile loops triggered within the rate limiter refresh period for each resource. Setting the limit <= 0 disables the limiter. |
- | spark.kubernetes.operator.reconciler.retry.initialIntervalSeconds | Integer | 5 | false | Initial interval(in seconds) of retries on unhandled controller errors. |
- | spark.kubernetes.operator.reconciler.retry.intervalMultiplier | Double | 1.5 | false | Interval multiplier of retries on unhandled controller errors. Setting this to 1 for linear retry. |
- | spark.kubernetes.operator.reconciler.retry.maxIntervalSeconds | Integer | -1 | false | Max interval(in seconds) of retries on unhandled controller errors. Set to non-positive for unlimited. |
- | spark.kubernetes.operator.api.retryMaxAttempts | Integer | 15 | false | Max attempts of retries on unhandled controller errors. Setting this to non-positive value means no retry. |
- | spark.kubernetes.operator.api.retryAttemptAfterSeconds | Long | 1 | false | Default time (in seconds) to wait till next request. This would be used if server does not set Retry-After in response. Setting this to non-positive number means immediate retry. |
- | spark.kubernetes.operator.api.statusPatchMaxAttempts | Long | 3 | false | Maximal number of retry attempts of requests to k8s server for resource status update. This would be performed on top of k8s client spark.kubernetes.operator.retry.maxAttempts to overcome potential conflicting update on the same SparkApplication. This should be positive number. |
- | spark.kubernetes.operator.api.secondaryResourceCreateMaxAttempts | Long | 3 | false | Maximal number of retry attempts of requesting secondary resource for Spark application. This would be performed on top of k8s client spark.kubernetes.operator.retry.maxAttempts to overcome potential conflicting reconcile on the same SparkApplication. This should be positive number |
- | spark.kubernetes.operator.metrics.josdkMetricsEnabled | Boolean | true | false | When enabled, the josdk metrics will be added in metrics source and configured for operator. |
- | spark.kubernetes.operator.metrics.clientMetricsEnabled | Boolean | true | false | Enable KubernetesClient metrics for measuring the HTTP traffic to the Kubernetes API Server. Since the metrics is collected via Okhttp interceptors, can be disabled when opt in customized interceptors. |
- | spark.kubernetes.operator.metrics.clientMetricsGroupByResponseCodeEnabled | Boolean | true | false | When enabled, additional metrics group by http response code group(1xx, 2xx, 3xx, 4xx, 5xx) received from API server will be added. Users can disable it when their monitoring system can combine lower level kubernetes.client.http.response.<3-digit-response-code> metrics. |
- | spark.kubernetes.operator.metrics.port | Integer | 19090 | false | The port used for checking metrics |
- | spark.kubernetes.operator.health.probePort | Integer | 19091 | false | The port used for health/readiness check probe status. |
- | spark.kubernetes.operator.health.sentinelExecutorPoolSize | Integer | 3 | false | Size of executor service in Sentinel Managers to check the health of sentinel resources. |
- | spark.kubernetes.operator.health.sentinelResourceReconciliationDelaySeconds | Integer | 60 | true | Allowed max time(seconds) between spec update and reconciliation for sentinel resources. |
- | spark.kubernetes.operator.leaderElection.enabled | Boolean | false | false | Enable leader election for the operator to allow running standby instances. When this is disabled, only one operator instance is expected to be up and running at any time (replica = 1) to avoid race condition. |
- | spark.kubernetes.operator.leaderElection.leaseName | String | spark-operator-lease | false | Leader election lease name, must be unique for leases in the same namespace. |
- | spark.kubernetes.operator.leaderElection.leaseDurationSeconds | Integer | 180 | false | Leader election lease duration in seconds, non-negative. |
- | spark.kubernetes.operator.leaderElection.renewDeadlineSeconds | Integer | 120 | false | Leader election renew deadline in seconds, non-negative. This needs to be smaller than the lease duration to allow current leader renew the lease before lease expires. |
- | spark.kubernetes.operator.leaderElection.retryPeriodSeconds | Integer | 5 | false | Leader election retry period in seconds, non-negative. |
-
diff --git a/docs/configuration.md b/docs/configuration.md
deleted file mode 100644
index bafd3c5..0000000
--- a/docs/configuration.md
+++ /dev/null
@@ -1,144 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-# Configuration
-
-## Configure Operator
-
-Spark Operator supports different ways to configure the behavior:
-
-* **spark-operator.properties** provided when deploying the operator. In addition to the
- [property file](../build-tools/helm/spark-kubernetes-operator/conf/spark-operator.
- properties), it is also possible to override or append config properties in helm [Values
- files](../build-tools/helm/spark-kubernetes-operator/values.yaml).
-* **System Properties** : when provided as system properties (e.g. via -D options to the
- operator JVM), it overrides the values provided in property file.
-* **Hot property loading** : when enabled, a
- [configmap](https://kubernetes.io/docs/concepts/configuration/configmap/) would be created with
- the operator in the same namespace. Operator can monitor updates performed on the configmap. Hot
- properties reloading takes higher precedence comparing with default properties override.
- - An example use case: operator use hot properties to figure the list of namespace(s) to
- operate Spark applications. The hot properties config map can be updated and
- maintained by user or additional microservice to tune the operator behavior without
- rebooting it.
- - Please be advised that not all properties can be hot-loaded and honored at runtime.
- Refer the list of [supported properties](./config_properties.md) for more details.
-
-To enable hot properties loading, update the **helm chart values file** with
-
-```
-operatorConfiguration:
- spark-operator.properties: |+
- spark.operator.dynamic.config.enabled=true
- # ... all other config overides...
- dynamicConfig:
- create: true
-```
-
-## Metrics
-
-Spark operator,
-following [Apache Spark](https://spark.apache.org/docs/latest/monitoring.html#metrics),
-has a configurable metrics system based on
-the [Dropwizard Metrics Library](https://metrics.dropwizard.io/4.2.25/). Note that Spark Operator
-does not have Spark UI, MetricsServlet
-and PrometheusServlet from org.apache.spark.metrics.sink package are not supported. If you are
-interested in Prometheus metrics exporting, please take a look at below
-section [Forward Metrics to Prometheus](#Forward-Metrics-to-Prometheus)
-
-### JVM Metrics
-
-Spark Operator collects JVM metrics
-via [Codahale JVM Metrics](https://javadoc.io/doc/com.codahale.metrics/metrics-jvm/latest/index.html)
-
-- BufferPoolMetricSet
-- FileDescriptorRatioGauge
-- GarbageCollectorMetricSet
-- MemoryUsageGaugeSet
-- ThreadStatesGaugeSet
-
-### Kubernetes Client Metrics
-
-| Metrics Name | Type | Description |
-|-----------------------------------------------------------|------------|--------------------------------------------------------------------------------------------------------------------------|
-| kubernetes.client.http.request | Meter | Tracking the rates of HTTP request sent to the Kubernetes API Server |
-| kubernetes.client.http.response | Meter | Tracking the rates of HTTP response from the Kubernetes API Server |
-| kubernetes.client.http.response.failed | Meter | Tracking the rates of HTTP requests which have no response from the Kubernetes API Server |
-| kubernetes.client.http.response.latency.nanos | Histograms | Measures the statistical distribution of HTTP response latency from the Kubernetes API Server |
-| kubernetes.client.http.response.<ResponseCode> | Meter | Tracking the rates of HTTP response based on response code from the Kubernetes API Server |
-| kubernetes.client.http.request.<RequestMethod> | Meter | Tracking the rates of HTTP request based type of method to the Kubernetes API Server |
-| kubernetes.client.http.response.1xx | Meter | Tracking the rates of HTTP Code 1xx responses (informational) received from the Kubernetes API Server per response code. |
-| kubernetes.client.http.response.2xx | Meter | Tracking the rates of HTTP Code 2xx responses (success) received from the Kubernetes API Server per response code. |
-| kubernetes.client.http.response.3xx | Meter | Tracking the rates of HTTP Code 3xx responses (redirection) received from the Kubernetes API Server per response code. |
-| kubernetes.client.http.response.4xx | Meter | Tracking the rates of HTTP Code 4xx responses (client error) received from the Kubernetes API Server per response code. |
-| kubernetes.client.http.response.5xx | Meter | Tracking the rates of HTTP Code 5xx responses (server error) received from the Kubernetes API Server per response code. |
-| kubernetes.client.<ResourceName>.<Method> | Meter | Tracking the rates of HTTP request for a combination of one Kubernetes resource and one http method |
-| kubernetes.client.<NamespaceName>.<ResourceName>.<Method> | Meter | Tracking the rates of HTTP request for a combination of one namespace-scoped Kubernetes resource and one http method |
-
-### Forward Metrics to Prometheus
-
-In this section, we will show you how to forward Spark Operator metrics
-to [Prometheus](https://prometheus.io).
-
-* Modify the metrics properties section in the file
- `build-tools/helm/spark-kubernetes-operator/values.yaml`:
-
-```properties
-metrics.properties:|+
- spark.metrics.conf.operator.sink.prometheus.class=org.apache.spark.kubernetes.operator.metrics.
-sink.PrometheusPullModelSink
-```
-
-* Install Spark Operator
-
-```bash
-helm install spark-kubernetes-operator -f build-tools/helm/spark-kubernetes-operator/values.yaml build-tools/helm/spark-kubernetes-operator/
-```
-
-* Install Prometheus via Helm Chart
-
-```bash
-helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-helm install prometheus prometheus-community/prometheus
-```
-
-* Find and Annotate Spark Operator Pods
-
-```bash
-kubectl get pods -l app.kubernetes.io/name=spark-kubernetes-operator
-NAME READY STATUS RESTARTS AGE
-spark-kubernetes-operator-598cb5d569-bvvd2 1/1 Running 0 24m
-
-kubectl annotate pods spark-kubernetes-operator-598cb5d569-bvvd2 prometheus.io/scrape=true
-kubectl annotate pods spark-kubernetes-operator-598cb5d569-bvvd2 prometheus.io/path=/prometheus
-kubectl annotate pods spark-kubernetes-operator-598cb5d569-bvvd2 prometheus.io/port=19090
-```
-
-* Check Metrics via Prometheus UI
-
-```bash
-kubectl get pods | grep "prometheus-server"
-prometheus-server-654bc74fc9-8hgkb 2/2 Running 0 59m
-
-kubectl port-forward --address 0.0.0.0 pod/prometheus-server-654bc74fc9-8hgkb 8080:9090
-```
-
-open your browser with address `localhost:8080`. Click on Status Targets tab, you should be able
-to find target as below.
-[<img src="resources/prometheus.png">](resources/prometheus.png)
diff --git a/docs/operations.md b/docs/operations.md
deleted file mode 100644
index 86ebf11..0000000
--- a/docs/operations.md
+++ /dev/null
@@ -1,151 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-### Compatibility
-
-- Java 17, 21 and 24
-- Kubernetes version compatibility:
- + k8s version >= 1.30 is recommended. Operator attempts to be API compatible as possible, but
- patch support will not be performed on k8s versions that reached EOL.
-- Spark versions 3.5 or above.
-
-### Spark Application Namespaces
-
-By default, Spark applications are created in the same namespace as the operator deployment.
-You many also configure the chart deployment to add necessary RBAC resources for
-applications to enable them running in additional namespaces.
-
-## Overriding configuration parameters during Helm install
-
-Helm provides different ways to override the default installation parameters (contained
-in `values.yaml`) for the Helm chart.
-
-To override single parameters you can use `--set`, for example:
-
-```
-helm install --set image.repository=<my_registory>/spark-kubernetes-operator \
- -f build-tools/helm/spark-kubernetes-operator/values.yaml \
- build-tools/helm/spark-kubernetes-operator/
-```
-
-You can also provide multiple custom values file by using the `-f` flag, the latest takes
-higher precedence:
-
-```
-helm install spark-kubernetes-operator \
- -f build-tools/helm/spark-kubernetes-operator/values.yaml \
- -f my_values.yaml \
- build-tools/helm/spark-kubernetes-operator/
-```
-
-The configurable parameters of the Helm chart and which default values as detailed in the
-following table:
-
-| Parameters | Description | Default value |
-|---------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|
-| image.repository | The image repository of spark-kubernetes-operator. | spark-kubernetes-operator |
-| image.pullPolicy | The image pull policy of spark-kubernetes-operator. | IfNotPresent |
-| image.tag | The image tag of spark-kubernetes-operator. | 0.2.0-SNAPSHOT |
-| image.digest | The image digest of spark-kubernetes-operator. If set then it takes precedence and the image tag will be ignored. | |
-| imagePullSecrets | The image pull secrets of spark-kubernetes-operator. | |
-| operatorDeployment.replica | Operator replica count. Must be 1 unless leader election is configured. | 1 |
-| operatorDeployment.strategy.type | Operator pod upgrade strategy. Must be Recreate unless leader election is configured. | Recreate |
-| operatorDeployment.operatorPod.annotations | Custom annotations to be added to the operator pod | |
-| operatorDeployment.operatorPod.labels | Custom labels to be added to the operator pod | |
-| operatorDeployment.operatorPod.nodeSelector | Custom nodeSelector to be added to the operator pod. | |
-| operatorDeployment.operatorPod.topologySpreadConstraints | Custom topologySpreadConstraints to be added to the operator pod. | |
-| operatorDeployment.operatorPod.dnsConfig | DNS configuration to be used by the operator pod. | |
-| operatorDeployment.operatorPod.volumes | Additional volumes to be added to the operator pod. | |
-| operatorDeployment.operatorPod.priorityClassName | Priority class name to be used for the operator pod | |
-| operatorDeployment.operatorPod.securityContext | Security context overrides for the operator pod | |
-| operatorDeployment.operatorContainer.jvmArgs | JVM arg override for the operator container. | `"-Dfile.encoding=UTF8"` |
-| operatorDeployment.operatorContainer.env | Custom env to be added to the operator container. | |
-| operatorDeployment.operatorContainer.envFrom | Custom envFrom to be added to the operator container, e.g. for downward API. | |
-| operatorDeployment.operatorContainer.probes | Probe config for the operator container. | |
-| operatorDeployment.operatorContainer.securityContext | Security context overrides for the operator container. | run as non root for baseline secuirty standard compliance |
-| operatorDeployment.operatorContainer.resources | Resources for the operator container. | memory 4Gi, ephemeral storage 2Gi and 1 cpu |
-| operatorDeployment.additionalContainers | Additional containers to be added to the operator pod, e.g. sidecar. | |
-| operatorRbac.serviceAccount.create | Whether to create service account for operator to use. | true |
-| operatorRbac.serviceAccount.name | Name of the operator Role. | `"spark-operator"` |
-| operatorRbac.clusterRole.create | Whether to create ClusterRole for operator to use. | true |
-| operatorRbac.clusterRole.name | Name of the operator ClusterRole. | `"spark-operator-clusterrole"` |
-| operatorRbac.clusterRoleBinding.create | Whether to create ClusterRoleBinding for operator to use. | true |
-| operatorRbac.clusterRoleBinding.name | Name of the operator ClusterRoleBinding. | `"spark-operator-clusterrolebinding"` |
-| operatorRbac.role.create | Whether to create Role for operator to use in each workload namespace(s). At least one of `clusterRole.create` or `role.create` should be enabled | false |
-| operatorRbac.role.name | Name of the operator Role | `"spark-operator-role"` |
-| operatorRbac.roleBinding.create | Whether to create RoleBinding for operator to use. At least one of `clusterRoleBinding.create` or `roleBinding.create` should be enabled | false |
-| operatorRbac.roleBinding.name | Name of the operator RoleBinding in each workload namespace(s). | `"spark-operator-rolebinding"` |
-| operatorRbac.roleBinding.roleRef | RoleRef for the created Operator RoleBinding. Override this when you want the created RoleBinding refer to ClusterRole / Role that's different from the default operator Role. | Refers to default `operatorRbac.role.name` |
-| operatorRbac.configManagement.create | Enable this to create a Role for operator configuration management (hot property loading and leader election). | true |
-| operatorRbac.configManagement.roleName | Role name for operator configuration management. | `spark-operator-config-role` |
-| operatorRbac.configManagement.roleBinding | RoleBinding name for operator configuration management. | `"spark-operator-config-monitor-role-binding"` |
-| operatorRbac.labels | Labels to be applied on all created `operatorRbac` resources. | `"app.kubernetes.io/component": "operator-rbac"` |
-| workloadResources.namespaces.create | Whether to create dedicated namespaces for Spark workload. | true |
-| workloadResources.namespaces.overrideWatchedNamespaces | When enabled, operator would by default only watch namespace(s) provided in data field. | true |
-| workloadResources.namespaces.data | List of namespaces to create for Spark workload. The chart namespace would be used if this is empty. | |
-| workloadResources.clusterRole.create | When enabled, a ClusterRole would be created for Spark workload to use. | true |
-| workloadResources.clusterRole.name | Name of the Spark workload ClusterRole. | "spark-workload-clusterrole" |
-| workloadResources.role.create | When enabled, a Role would be created in each namespace for Spark workload. At least one of `clusterRole.create` or `role.create` should be enabled. | false |
-| workloadResources.role.name | Name for Spark workload Role. | "spark-workload-role" |
-| workloadResources.roleBinding.create | When enabled, a RoleBinding would be created in each namespace for Spark workload. This shall be enabled unless access is configured from 3rd party. | true |
-| workloadResources.roleBinding.name | Name of the Spark workload RoleBinding. | "spark-workload-rolebinding" |
-| workloadResources.serviceAccounts.create | Whether to create a service account for Spark workload. | true |
-| workloadResources.serviceAccounts.name | The name of Spark workload service account. | `spark` |
-| workloadResources.labels | Labels to be applied for all workload resources. | `"app.kubernetes.io/component": "spark-workload"` |
-| workloadResources.annotations | Annotations to be applied for all workload resources. | `"helm.sh/resource-policy": keep` |
-| workloadResources.sparkApplicationSentinel.create | If enabled, sentinel resources will be created for operator to watch and reconcile for the health probe purpose. | false |
-| workloadResources.sparkApplicationSentinel.sentinelNamespaces | A list of namespaces where sentinel resources will be created in. Note that these namespaces have to be a subset of `workloadResources.namespaces.data`. | |
-| operatorConfiguration.append | If set to true, below conf file & properties would be appended to default conf. Otherwise, they would override default properties. | true |
-| operatorConfiguration.log4j2.properties | The default log4j2 configuration. | Refer default [log4j2.properties](../build-tools/helm/spark-kubernetes-operator/conf/log4j2.properties) |
-| operatorConfiguration.spark-operator.properties | The default operator configuration. | |
-| operatorConfiguration.metrics.properties | The default operator metrics (sink) configuration. | |
-| operatorConfiguration.dynamicConfig.create | If set to true, a config map would be created & watched by operator as source of truth for hot properties loading. | false |
-| operatorConfiguration.dynamicConfig.enable | If set to true, operator would honor the created config mapas source of truth for hot properties loading. | false |
-| operatorConfiguration.dynamicConfig.annotations | Annotations to be applied for the dynamicConfig resources. | `"helm.sh/resource-policy": keep` |
-| operatorConfiguration.dynamicConfig.data | Data field (key-value pairs) that acts as hot properties in the config map. | `spark.kubernetes.operator.reconciler.intervalSeconds: "60"` |
-
-For more information check the [Helm documentation](https://helm.sh/docs/helm/helm_install/).
-
-__Notice__: The pod resources should be set as your workload in different environments to
-archive a matched K8s pod QoS. See
-also [Pod Quality of Service Classes](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/#quality-of-service-classes).
-
-## Operator Health(Liveness) Probe with Sentinel Resource
-
-Learning
-from [Apache Flink Operator](https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main/docs/operations/health/#canary-resources),
-a dummy spark application resource in any watched namespace can help Spark operator health
-probe monitor.
-
-Here is a Spark Sentinel resource example with the label `"spark.operator/sentinel": "true"`
-and it will not result in creation of any other kubernetes resources. Controlled by
-property `health.sentinel.resource.reconciliation.delay.seconds`, by default, the timeout to
-reconcile the sentinel resources is 60 seconds. If the operator cannot reconcile these
-resources within limited time, the operator health probe will return HTTP code 500 when kubelet
-send the HTTP Get to the liveness endpoint, and the
-kubelet will then kill the spark operator container and restart it.
-
-```yaml
-apiVersion: org.apache.spark/v1alpha1
-kind: SparkApplication
-metadata:
- name: spark-sentinel-resources
- labels:
- "spark.operator/sentinel": "true"
-```
diff --git a/docs/resources/application_state_machine.png b/docs/resources/application_state_machine.png
deleted file mode 100644
index 3b3df3d..0000000
Binary files a/docs/resources/application_state_machine.png and /dev/null differ
diff --git a/docs/resources/cluster_state_machine.png b/docs/resources/cluster_state_machine.png
deleted file mode 100644
index 2a8dcdd..0000000
Binary files a/docs/resources/cluster_state_machine.png and /dev/null differ
diff --git a/docs/resources/prometheus.png b/docs/resources/prometheus.png
deleted file mode 100644
index 5507d57..0000000
Binary files a/docs/resources/prometheus.png and /dev/null differ
diff --git a/docs/spark_custom_resources.md b/docs/spark_custom_resources.md
deleted file mode 100644
index 6a8751d..0000000
--- a/docs/spark_custom_resources.md
+++ /dev/null
@@ -1,322 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-## Spark Operator API
-
-The core user facing API of the Spark Kubernetes Operator is the `SparkApplication` and
-`SparkCluster` Custom Resources Definition (CRD). Spark custom resource extends
-standard k8s API, defines Spark Application spec and tracks status.
-
-Once the Spark Operator is installed and running in your Kubernetes environment, it will
-continuously watch SparkApplication(s) and SparkCluster(s) submitted, via k8s API client or
-kubectl by the user, orchestrate secondary resources (pods, configmaps .etc).
-
-Please check out the [quickstart](../README.md) as well for installing operator.
-
-## SparkApplication
-
-SparkApplication can be defined in YAML format. User may configure the application entrypoint
-and configurations. Let's start with the [Spark-Pi example](../examples/pi.yaml):
-
-```yaml
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi
-spec:
- # Entry point for the app
- mainClass: "org.apache.spark.examples.SparkPi"
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- runtimeVersions:
- scalaVersion: "2.13"
- sparkVersion: "4.0.0-preview2"
-```
-
-After application is submitted, Operator will add status information to your application based on
-the observed state:
-
-```
-kubectl get sparkapp pi -o yaml
-```
-
-### Write and build your SparkApplication
-
-It's straightforward to convert your spark-submit application to `SparkApplication` yaml.
-Operators constructs driver spec in the similar approach. To submit Java / scala application,
-use `.spec.jars` and `.spec.mainClass`. Similarly, set `pyFiles` for Python applications.
-
-While building images to use by driver and executor, it's recommended to use official
-[Spark Docker](https://github.com/apache/spark-docker) as base images. Check the pod template
-support (`.spec.driverSpec.podTemplateSpec` and `.spec.executorSpec.podTemplateSpec`) as well for
-setting custom Spark home and work dir.
-
-### Pod Template Support
-
-It is possible to configure pod template for driver & executor pods for configure spec that are
-not configurable from SparkConf.
-
-Spark Operator supports defining pod template for driver and executor pods in two ways:
-
-1. Set `PodTemplateSpec` in `SparkApplication`
-2. Config `spark.kubernetes.[driver/executor].podTemplateFile`
-
-If pod template spec is set in application spec (option 1), it would take higher precedence
-than option 2. Also `spark.kubernetes.[driver/executor].podTemplateFile` would be unset to
-avoid multiple override.
-
-When pod template is set as remote file in conf properties (option 2), please ensure Spark
-Operator has necessary permission to access the remote file location, e.g. deploy operator
-with proper workload identity with target S3 / Cloud Storage bucket access. Similar permission
-requirements are also needed driver pod: operator needs template file access to create driver,
-and driver needs the same for creating executors.
-
-Please be advised that Spark still overrides necessary pod configuration in both options. For
-more details,
-refer [Spark doc](https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template).
-
-## Enable Additional Ingress for Driver
-
-Operator may create [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) for
-Spark driver of running applications on demand. For example, to expose Spark UI - which is by
-default enabled on driver port 4040, you may configure
-
-```yaml
-spec:
- driverServiceIngressList:
- - serviceMetadata:
- name: "spark-ui-service"
- serviceSpec:
- ports:
- - protocol: TCP
- port: 80
- targetPort: 4040
- ingressMetadata:
- name: "spark-ui-ingress"
- annotations:
- nginx.ingress.kubernetes.io/rewrite-target: /
- ingressSpec:
- ingressClassName: nginx-example
- rules:
- - http:
- paths:
- - path: "/"
- pathType: Prefix
- backend:
- service:
- name: spark-ui-service
- port:
- number: 80
-```
-
-Spark Operator by default would populate the `.spec.selector` field of the created Service to match
-the driver labels. If `.ingressSpec.rules` is not provided, Spark Operator would also populate one
-default rule backed by the associated Service. It's recommended to always provide the ingress spec
-to make sure it's compatible with your
-[IngressController](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/).
-
-## Create and Mount ConfigMap
-
-It is possible to ask operator to create configmap so they can be used by driver and/or executor
-pods on the fly. `configMapSpecs` allows you to specify the desired metadata and data as string
-literals for the configmap(s) to be created.
-
-```yaml
-spec:
- configMapSpecs:
- - name: "example-config-map"
- data:
- foo: "bar"
-```
-
-Like other app-specific resources, the created configmap has owner reference to Spark driver and
-therefore shares the same lifecycle and garbage collection mechanism with the associated app.
-
-This feature can be used to create lightweight override config files for given Spark app. For
-example, below snippet would create and mount a configmap with metrics property file, then use it
-in SparkConf:
-
-```yaml
-spec:
- sparkConf:
- spark.metrics.conf: "/etc/metrics/metrics.properties"
- driverSpec:
- podTemplateSpec:
- spec:
- containers:
- - volumeMounts:
- - name: "config-override"
- mountPath: "/etc/metrics"
- readOnly: true
- volumes:
- - name: config-override
- configMap:
- name: metrics-configmap
- executorSpec:
- podTemplateSpec:
- spec:
- containers:
- - volumeMounts:
- - name: "config-override"
- mountPath: "/etc/metrics"
- readOnly: true
- volumes:
- - name: config-override
- configMap:
- name: metrics-configmap
- configMapSpecs:
- - name: "metrics-configmap"
- data:
- metrics.properties: "*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink\n"
-
-```
-
-## Understanding Failure Types
-
-In addition to the general `Failed` state (that driver pod fails or driver container exits
-with non-zero code), Spark Operator introduces a few different failure state for ease of
-app status monitoring at high level, and for ease of setting up different handlers if users
-are creating / managing SparkApplications with external microservices or workflow engines.
-
-
-Spark Operator recognizes "infrastructure failure" in the best effort way. It is possible to
-configure different restart policy on general failure(s) vs. on potential infrastructure
-failure(s). For example, you may configure the app to restart only upon infrastructure
-failures. If Spark application fails as a result of
-
-```
-DriverStartTimedOut
-ExecutorsStartTimedOut
-SchedulingFailure
-```
-
-It is more likely that the app failed as a result of infrastructure reason(s), including
-scenarios like driver or executors cannot be scheduled or cannot initialize in configured
-time window for scheduler reasons, as a result of insufficient capacity, cannot get IP
-allocated, cannot pull images, or k8s API server issue at scheduling .etc.
-
-Please be advised that this is a best-effort failure identification. You may still need to
-debug actual failure from the driver pods. Spark Operator would stage the last observed
-driver pod status with the stopping state for audit purposes.
-
-## Configure the Tolerations for SparkApplication
-
-### Restart
-
-Spark Operator enables configure app restart behavior for different failure types. Here's a
-sample restart config snippet:
-
-``` yaml
-restartConfig:
- # accptable values are 'Never', 'Always', 'OnFailure' and 'OnInfrastructureFailure'
- restartPolicy: Never
- # operator would retry the application if configured. All resources from current attepmt
- # would be deleted before starting next attempt
- maxRestartAttempts: 3
- # backoff time (in millis) that operator would wait before next attempt
- restartBackoffMillis: 30000
-```
-
-### Timeouts
-
-It's possible to configure applications to be proactively terminated and resubmitted in particular
-cases to avoid resource deadlock.
-
-
-| Field | Type | Default Value | Descritpion |
-|-----------------------------------------------------------------------------------------|---------|---------------|--------------------------------------------------------------------------------------------------------------------|
-| .spec.applicationTolerations.applicationTimeoutConfig.driverStartTimeoutMillis | integer | 300000 | Time to wait for driver reaches running state after requested driver. |
-| .spec.applicationTolerations.applicationTimeoutConfig.executorStartTimeoutMillis | integer | 300000 | Time to wait for driver to acquire minimal number of running executors. |
-| .spec.applicationTolerations.applicationTimeoutConfig.forceTerminationGracePeriodMillis | integer | 300000 | Time to wait for force delete resources at the end of attempt. |
-| .spec.applicationTolerations.applicationTimeoutConfig.driverReadyTimeoutMillis | integer | 300000 | Time to wait for driver reaches ready state. |
-| .spec.applicationTolerations.applicationTimeoutConfig.terminationRequeuePeriodMillis | integer | 2000 | Back-off time when releasing resource need to be re-attempted for application. |
-
-
-### Instance Config
-
-Instance Config helps operator to decide whether an application is running healthy. When
-the underlying cluster has batch scheduler enabled, you may configure the apps to be
-started if and only if there are sufficient resources. If, however, the cluster does not
-have a batch scheduler, operator may help avoid app hanging with `InstanceConfig` that
-describes the bare minimal tolerable scenario.
-
-For example, with below spec:
-
-```yaml
-applicationTolerations:
- instanceConfig:
- minExecutors: 3
- initExecutors: 5
- maxExecutors: 10
-sparkConf:
- spark.executor.instances: "10"
-```
-
-Spark would try to bring up 10 executors as defined in SparkConf. In addition, from
-operator perspective,
-
-* If Spark app acquires less than 5 executors in given tine window (.spec.
- applicationTolerations.applicationTimeoutConfig.executorStartTimeoutMillis) after
- submitted, it would be shut down proactively in order to avoid resource deadlock.
-* Spark app would be marked as 'RunningWithBelowThresholdExecutors' if it loses executors after
- successfully start up.
-* Spark app would be marked as 'RunningHealthy' if it has at least min executors after
- successfully started up.
-
-### Delete Resources On Termination
-
-Operator by default would delete all created resources at the end of an attempt. It would
-try to record the last observed driver status in `status` field of the application for
-troubleshooting purpose.
-
-On the other hand, when developing an application, it's possible to configure
-
-```yaml
-applicationTolerations:
- # Acceptable values are 'Always', 'OnFailure', 'Never'
- resourceRetentionPolicy: OnFailure
-```
-
-to avoid operator attempt to delete driver pod and driver resources if app fails. Similarly,
-if resourceRetentionPolicy is set to `Always`, operator would not delete driver resources
-when app ends. Note that this applies only to operator-created resources (driver pod, SparkConf
-configmap .etc). You may also want to tune `spark.kubernetes.driver.service.deleteOnTermination`
-and `spark.kubernetes.executor.deleteOnTermination` to control the behavior of driver-created
-resources.
-
-## Spark Cluster
-
-Spark Operator also supports launching Spark clusters in k8s via `SparkCluster` custom resource,
-which takes minimal effort to specify desired master and worker instances spec.
-
-To deploy a Spark cluster, you may start with specifying the desired Spark version, worker count as
-well as the SparkConf as in the [example](../examples/qa-cluster-with-one-worker.yaml). Master &
-worker instances would be deployed as [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
-and exposed via k8s [service(s)](https://kubernetes.io/docs/concepts/services-networking/service/).
-
-Like Pod Template Support for Applications, it's also possible to submit template(s) for the Spark
-instances for `SparkCluster` to configure spec that's not supported via SparkConf. It's worth notice
-that Spark may overwrite certain fields.
diff --git a/examples/cluster-java21.yaml b/examples/cluster-java21.yaml
deleted file mode 100644
index abc4826..0000000
--- a/examples/cluster-java21.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkCluster
-metadata:
- name: cluster-java21
-spec:
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
- clusterTolerations:
- instanceConfig:
- initWorkers: 3
- minWorkers: 3
- maxWorkers: 3
- sparkConf:
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2-java21"
- spark.master.ui.title: "Prod Spark Cluster (Java 21)"
- spark.master.rest.enabled: "true"
- spark.master.rest.host: "0.0.0.0"
- spark.ui.reverseProxy: "true"
diff --git a/examples/cluster-on-yunikorn.yaml b/examples/cluster-on-yunikorn.yaml
deleted file mode 100644
index 41a0f4a..0000000
--- a/examples/cluster-on-yunikorn.yaml
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkCluster
-metadata:
- name: cluster-on-yunikorn
-spec:
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
- clusterTolerations:
- instanceConfig:
- initWorkers: 1
- minWorkers: 1
- maxWorkers: 1
- sparkConf:
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- spark.kubernetes.scheduler.name: "yunikorn"
- spark.master.ui.title: "Spark Cluster on YuniKorn Scheduler"
- spark.master.rest.enabled: "true"
- spark.master.rest.host: "0.0.0.0"
- spark.ui.reverseProxy: "true"
diff --git a/examples/cluster-with-hpa-template.yaml b/examples/cluster-with-hpa-template.yaml
deleted file mode 100644
index 5220690..0000000
--- a/examples/cluster-with-hpa-template.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkCluster
-metadata:
- name: cluster-with-hpa-template
-spec:
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
- clusterTolerations:
- instanceConfig:
- initWorkers: 1
- minWorkers: 1
- maxWorkers: 2
- workerSpec:
- statefulSetSpec:
- template:
- spec:
- containers:
- - name: worker
- resources:
- requests:
- cpu: "3"
- memory: "3Gi"
- limits:
- cpu: "3"
- memory: "3Gi"
- horizontalPodAutoscalerSpec:
- metrics:
- - type: ContainerResource
- containerResource:
- name: cpu
- container: worker
- target:
- type: Utilization
- averageUtilization: 10
- behavior:
- scaleUp:
- policies:
- - type: Pods
- value: 1
- periodSeconds: 10
- scaleDown:
- policies:
- - type: Pods
- value: 1
- periodSeconds: 1200
- sparkConf:
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2-java21"
- spark.master.ui.title: "Cluster with HorizontalPodAutoscaler Template"
- spark.master.rest.enabled: "true"
- spark.master.rest.host: "0.0.0.0"
- spark.ui.reverseProxy: "true"
diff --git a/examples/cluster-with-hpa.yaml b/examples/cluster-with-hpa.yaml
deleted file mode 100644
index ca82088..0000000
--- a/examples/cluster-with-hpa.yaml
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkCluster
-metadata:
- name: cluster-with-hpa
-spec:
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
- clusterTolerations:
- instanceConfig:
- initWorkers: 3
- minWorkers: 1
- maxWorkers: 3
- workerSpec:
- statefulSetSpec:
- template:
- spec:
- containers:
- - name: worker
- resources:
- requests:
- cpu: "3"
- memory: "3Gi"
- limits:
- cpu: "3"
- memory: "3Gi"
- sparkConf:
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2-java21"
- spark.master.ui.title: "Cluster with HorizontalPodAutoscaler"
- spark.master.rest.enabled: "true"
- spark.master.rest.host: "0.0.0.0"
- spark.ui.reverseProxy: "true"
diff --git a/examples/cluster-with-template.yaml b/examples/cluster-with-template.yaml
deleted file mode 100644
index 1209500..0000000
--- a/examples/cluster-with-template.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkCluster
-metadata:
- name: cluster-with-template
-spec:
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
- clusterTolerations:
- instanceConfig:
- initWorkers: 1
- minWorkers: 1
- maxWorkers: 1
- masterSpec:
- statefulSetMetadata:
- annotations:
- customAnnotation: "annotation"
- statefulSetSpec:
- template:
- metadata:
- annotations:
- cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
- spec:
- priorityClassName: system-cluster-critical
- securityContext:
- runAsUser: 0
- containers:
- - name: master
- resources:
- requests:
- cpu: "2"
- memory: "2Gi"
- limits:
- cpu: "2"
- memory: "2Gi"
- - name: sidecar
- image: registry.k8s.io/pause:3.9
- resources:
- requests:
- cpu: "0.1"
- memory: "10Mi"
- limits:
- cpu: "0.1"
- memory: "10Mi"
- serviceMetadata:
- annotations:
- customAnnotation: "svc1"
- workerSpec:
- statefulSetMetadata:
- annotations:
- customAnnotation: "annotation"
- statefulSetSpec:
- template:
- metadata:
- annotations:
- cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
- spec:
- priorityClassName: system-cluster-critical
- securityContext:
- runAsUser: 0
- containers:
- - name: worker
- resources:
- requests:
- cpu: "2"
- memory: "2Gi"
- limits:
- cpu: "2"
- memory: "2Gi"
- - name: sidecar
- image: registry.k8s.io/pause:3.9
- resources:
- requests:
- cpu: "0.1"
- memory: "10Mi"
- limits:
- cpu: "0.1"
- memory: "10Mi"
- serviceMetadata:
- annotations:
- customAnnotation: "annotation"
- sparkConf:
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- spark.master.ui.title: "Spark Cluster with Template"
- spark.master.rest.enabled: "true"
- spark.master.rest.host: "0.0.0.0"
- spark.ui.reverseProxy: "true"
diff --git a/examples/pi-java21.yaml b/examples/pi-java21.yaml
deleted file mode 100644
index a0015a0..0000000
--- a/examples/pi-java21.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi-java21
-spec:
- mainClass: "org.apache.spark.examples.SparkPi"
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2-java21-scala"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- runtimeVersions:
- scalaVersion: "2.13"
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/pi-on-yunikorn.yaml b/examples/pi-on-yunikorn.yaml
deleted file mode 100644
index 8fda63a..0000000
--- a/examples/pi-on-yunikorn.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi-on-yunikorn
-spec:
- mainClass: "org.apache.spark.examples.SparkPi"
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- driverArgs: [ "20000" ]
- sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- spark.kubernetes.scheduler.name: "yunikorn"
- spark.kubernetes.driver.label.queue: "root.default"
- spark.kubernetes.executor.label.queue: "root.default"
- spark.kubernetes.driver.annotation.yunikorn.apache.org/app-id: "{{APP_ID}}"
- spark.kubernetes.executor.annotation.yunikorn.apache.org/app-id: "{{APP_ID}}"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- runtimeVersions:
- scalaVersion: "2.13"
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/pi-scala.yaml b/examples/pi-scala.yaml
deleted file mode 100644
index 6041822..0000000
--- a/examples/pi-scala.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi-scala
-spec:
- mainClass: "org.apache.spark.examples.SparkPi"
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2-scala"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- runtimeVersions:
- scalaVersion: "2.13"
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/pi-with-one-pod.yaml b/examples/pi-with-one-pod.yaml
deleted file mode 100644
index a83560e..0000000
--- a/examples/pi-with-one-pod.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi-with-one-pod
-spec:
- mainClass: "org.apache.spark.examples.SparkPi"
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- sparkConf:
- spark.kubernetes.driver.master: "local[10]"
- spark.kubernetes.driver.request.cores: "5"
- spark.kubernetes.driver.limit.cores: "5"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/pi-with-spark-connect-plugin.yaml b/examples/pi-with-spark-connect-plugin.yaml
deleted file mode 100644
index 0ab676f..0000000
--- a/examples/pi-with-spark-connect-plugin.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi-with-spark-connect-plugin
-spec:
- mainClass: "org.apache.spark.examples.SparkPi"
- driverArgs: ["200000"]
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- sparkConf:
- spark.plugins: "org.apache.spark.sql.connect.SparkConnectPlugin"
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- runtimeVersions:
- scalaVersion: "2.13"
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/pi.yaml b/examples/pi.yaml
deleted file mode 100644
index bfcd73d..0000000
--- a/examples/pi.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi
-spec:
- mainClass: "org.apache.spark.examples.SparkPi"
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- runtimeVersions:
- scalaVersion: "2.13"
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/prod-cluster-with-three-workers.yaml b/examples/prod-cluster-with-three-workers.yaml
deleted file mode 100644
index ffcd41d..0000000
--- a/examples/prod-cluster-with-three-workers.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkCluster
-metadata:
- name: prod
-spec:
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
- clusterTolerations:
- instanceConfig:
- initWorkers: 3
- minWorkers: 3
- maxWorkers: 3
- sparkConf:
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- spark.master.ui.title: "Prod Spark Cluster"
- spark.master.rest.enabled: "true"
- spark.master.rest.host: "0.0.0.0"
- spark.ui.reverseProxy: "true"
diff --git a/examples/pyspark-pi.yaml b/examples/pyspark-pi.yaml
deleted file mode 100644
index 7c9f46d..0000000
--- a/examples/pyspark-pi.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: pi-python
-spec:
- pyFiles: "local:///opt/spark/examples/src/main/python/pi.py"
- sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- applicationTolerations:
- resourceRetainPolicy: OnFailure
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/qa-cluster-with-one-worker.yaml b/examples/qa-cluster-with-one-worker.yaml
deleted file mode 100644
index 30bc18c..0000000
--- a/examples/qa-cluster-with-one-worker.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkCluster
-metadata:
- name: qa
-spec:
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
- clusterTolerations:
- instanceConfig:
- initWorkers: 1
- minWorkers: 1
- maxWorkers: 1
- sparkConf:
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- spark.master.ui.title: "QA Spark Cluster"
- spark.master.rest.enabled: "true"
- spark.master.rest.host: "0.0.0.0"
- spark.ui.reverseProxy: "true"
diff --git a/examples/spark-connect-server-with-spark-cluster.yaml b/examples/spark-connect-server-with-spark-cluster.yaml
deleted file mode 100644
index a26ff3d..0000000
--- a/examples/spark-connect-server-with-spark-cluster.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: spark-connect-server
-spec:
- mainClass: "org.apache.spark.sql.connect.service.SparkConnectServer"
- sparkConf:
- spark.master: "spark://prod-master-svc:7077"
- spark.submit.deployMode: "cluster"
- spark.executor.cores: "1"
- spark.cores.max: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- spark.ui.reverseProxy: "true"
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/sql.yaml b/examples/sql.yaml
deleted file mode 100644
index 0f5cce6..0000000
--- a/examples/sql.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
-kind: SparkApplication
-metadata:
- name: sql
-spec:
- mainClass: "org.apache.spark.examples.sql.JavaSparkSQLCli"
- jars: "local:///opt/spark/examples/jars/spark-examples.jar"
- driverArgs: [ "SHOW DATABASES", "SHOW TABLES", "SELECT VERSION()" ]
- sparkConf:
- spark.dynamicAllocation.enabled: "true"
- spark.dynamicAllocation.shuffleTracking.enabled: "true"
- spark.dynamicAllocation.maxExecutors: "3"
- spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
- spark.kubernetes.container.image: "apache/spark:4.0.0-preview2"
- runtimeVersions:
- sparkVersion: "4.0.0-preview2"
diff --git a/examples/submit-pi-to-prod.sh b/examples/submit-pi-to-prod.sh
deleted file mode 100755
index b3b4707..0000000
--- a/examples/submit-pi-to-prod.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-curl -XPOST http://localhost:6066/v1/submissions/create \
---data '{
- "appResource": "",
- "sparkProperties": {
- "spark.master": "spark://prod-master-svc:7077",
- "spark.submit.deployMode": "cluster",
- "spark.app.name": "SparkPi",
- "spark.driver.cores": "1",
- "spark.driver.memory": "1g",
- "spark.executor.cores": "1",
- "spark.executor.memory": "1g",
- "spark.cores.max": "2",
- "spark.ui.reverseProxy": "true",
- "spark.jars": "local:///opt/spark/examples/jars/spark-examples.jar"
- },
- "clientSparkVersion": "",
- "mainClass": "org.apache.spark.examples.SparkPi",
- "environmentVariables": { },
- "action": "CreateSubmissionRequest",
- "appArgs": [ "2000" ]
-}'
diff --git a/examples/submit-spark-connect-server.sh b/examples/submit-spark-connect-server.sh
deleted file mode 100755
index c31423c..0000000
--- a/examples/submit-spark-connect-server.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-curl -XPOST http://localhost:6066/v1/submissions/create \
---data '{
- "appResource": "",
- "sparkProperties": {
- "spark.master": "spark://prod-master-svc:7077",
- "spark.submit.deployMode": "cluster",
- "spark.app.name": "SparkConnectServer",
- "spark.driver.cores": "1",
- "spark.driver.memory": "1g",
- "spark.executor.cores": "1",
- "spark.executor.memory": "1g",
- "spark.cores.max": "2",
- "spark.ui.reverseProxy": "true"
- },
- "clientSparkVersion": "",
- "mainClass": "org.apache.spark.sql.connect.service.SparkConnectServer",
- "environmentVariables": { },
- "action": "CreateSubmissionRequest",
- "appArgs": [ ]
-}'
diff --git a/gradle.properties b/gradle.properties
deleted file mode 100644
index d407788..0000000
--- a/gradle.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Gradle
-org.gradle.jvmargs=-Xmx4g
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
deleted file mode 100644
index b08523e..0000000
--- a/gradle/libs.versions.toml
+++ /dev/null
@@ -1,70 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-[versions]
-fabric8 = "7.2.0"
-lombok = "1.18.38"
-operator-sdk = "4.9.0"
-okhttp = "4.12.0"
-dropwizard-metrics = "4.2.30"
-spark = "4.0.1-SNAPSHOT"
-log4j = "2.24.3"
-
-# Test
-junit = "5.12.2"
-jacoco = "0.8.13"
-mockito = "5.17.0"
-powermock = "2.0.9"
-
-# Build Analysis
-checkstyle = "10.17.0"
-pmd = "6.55.0"
-spotbugs-tool = "4.8.6"
-spotbugs-plugin = "6.0.17"
-spotless-plugin = "6.25.0"
-
-# Packaging
-shadow-jar-plugin = "8.3.6"
-
-[libraries]
-kubernetes-client = { group = "io.fabric8", name = "kubernetes-client", version.ref = "fabric8" }
-kubernetes-httpclient-okhttp = { group = "io.fabric8", name = "kubernetes-httpclient-okhttp", version.ref = "fabric8" }
-kubernetes-server-mock = { group = "io.fabric8", name = "kubernetes-server-mock", version.ref = "fabric8" }
-crd-generator-apt = { group = "io.fabric8", name = "crd-generator-apt", version.ref = "fabric8" }
-okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" }
-mockwebserver = { group = "com.squareup.okhttp3", name = "mockwebserver", version.ref = "okhttp" }
-logging-interceptor = { group = "com.squareup.okhttp3", name = "logging-interceptor", version.ref = "okhttp" }
-lombok = { group = "org.projectlombok", name = "lombok", version.ref = "lombok" }
-log4j-api = { group = "org.apache.logging.log4j", name = "log4j-api", version.ref = "log4j" }
-log4j-core = { group = "org.apache.logging.log4j", name = "log4j-core", version.ref = "log4j" }
-log4j-slf4j-impl = { group = "org.apache.logging.log4j", name = "log4j-slf4j-impl", version.ref = "log4j" }
-log4j-api12 = { group = "org.apache.logging.log4j", name = "log4j-1.2-api", version.ref = "log4j" }
-log4j-layout-template-json = { group = "org.apache.logging.log4j", name = "log4j-layout-template-json", version.ref = "log4j" }
-operator-framework = { group = "io.javaoperatorsdk", name = "operator-framework", version.ref = "operator-sdk"}
-operator-framework-junit5 = { group = "io.javaoperatorsdk", name = "operator-framework-junit-5", version.ref = "operator-sdk"}
-spotbugs-annotations = { group = "com.github.spotbugs", name = "spotbugs-annotations", version.ref = "spotbugs-tool"}
-metrics-core = { group = "io.dropwizard.metrics", name = "metrics-core", version.ref = "dropwizard-metrics"}
-metrics-jvm = { group = "io.dropwizard.metrics", name = "metrics-jvm", version.ref = "dropwizard-metrics"}
-spark-core = { group = "org.apache.spark", name = "spark-core_2.13", version.ref = "spark"}
-spark-kubernetes = { group = "org.apache.spark", name = "spark-kubernetes_2.13", version.ref = "spark"}
-mockito-core = { group = "org.mockito", name = "mockito-core", version.ref = "mockito"}
-junit-bom = { group = "org.junit", name = "junit-bom", version.ref = "junit"}
-junit-jupiter = { group = "org.junit.jupiter", name = "junit-jupiter", version.ref = "junit"}
-junit-platform-launcher = { group = "org.junit.platform", name = "junit-platform-launcher"}
-powermock-core = { group = "org.powermock", name = "powermock-core", version.ref = "powermock"}
-spotbugs-gradle-plugin = { group = "com.github.spotbugs.snom", name = "spotbugs-gradle-plugin", version.ref = "spotbugs-plugin" }
-spotless-plugin-gradle = { group = "com.diffplug.spotless", name = "spotless-plugin-gradle", version.ref = "spotless-plugin" }
-shadow = { group = "com.gradleup.shadow", name = "shadow-gradle-plugin", version.ref = "shadow-jar-plugin"}
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
deleted file mode 100644
index aecb62d..0000000
--- a/gradle/wrapper/gradle-wrapper.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-distributionSha256Sum=61ad310d3c7d3e5da131b76bbf22b5a4c0786e9d892dae8c1658d4b484de3caa
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip
-networkTimeout=10000
-validateDistributionUrl=true
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
deleted file mode 100755
index 118b4ab..0000000
--- a/gradlew
+++ /dev/null
@@ -1,259 +0,0 @@
-#!/bin/sh
-
-#
-# Copyright © 2015-2021 the original authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-##############################################################################
-#
-# Gradle start up script for POSIX generated by Gradle.
-#
-# Important for running:
-#
-# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
-# noncompliant, but you have some other compliant shell such as ksh or
-# bash, then to run this script, type that shell name before the whole
-# command line, like:
-#
-# ksh Gradle
-#
-# Busybox and similar reduced shells will NOT work, because this script
-# requires all of these POSIX shell features:
-# * functions;
-# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
-# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
-# * compound commands having a testable exit status, especially «case»;
-# * various built-in commands including «command», «set», and «ulimit».
-#
-# Important for patching:
-#
-# (2) This script targets any POSIX shell, so it avoids extensions provided
-# by Bash, Ksh, etc; in particular arrays are avoided.
-#
-# The "traditional" practice of packing multiple parameters into a
-# space-separated string is a well documented source of bugs and security
-# problems, so this is (mostly) avoided, by progressively accumulating
-# options in "$@", and eventually passing that to Java.
-#
-# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
-# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
-# see the in-line comments for details.
-#
-# There are tweaks for specific operating systems such as AIX, CygWin,
-# Darwin, MinGW, and NonStop.
-#
-# (3) This script is generated from the Groovy template
-# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
-# within the Gradle project.
-#
-# You can find Gradle at https://github.com/gradle/gradle/.
-#
-##############################################################################
-
-# Attempt to set APP_HOME
-
-# Resolve links: $0 may be a link
-app_path=$0
-
-# Need this for daisy-chained symlinks.
-while
- APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
- [ -h "$app_path" ]
-do
- ls=$( ls -ld "$app_path" )
- link=${ls#*' -> '}
- case $link in #(
- /*) app_path=$link ;; #(
- *) app_path=$APP_HOME$link ;;
- esac
-done
-
-# This is normally unused
-# shellcheck disable=SC2034
-APP_BASE_NAME=${0##*/}
-# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
-' "$PWD" ) || exit
-if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar -a "$(command -v curl)" ]; then
- curl -o $APP_HOME/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/gradle/gradle/v8.14.0/gradle/wrapper/gradle-wrapper.jar
-fi
-# If the file still doesn't exist, let's try `wget` and cross our fingers
-if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar -a "$(command -v wget)" ]; then
- wget -O $APP_HOME/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/gradle/gradle/v8.14.0/gradle/wrapper/gradle-wrapper.jar
-fi
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD=maximum
-
-warn () {
- echo "$*"
-} >&2
-
-die () {
- echo
- echo "$*"
- echo
- exit 1
-} >&2
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-nonstop=false
-case "$( uname )" in #(
- CYGWIN* ) cygwin=true ;; #(
- Darwin* ) darwin=true ;; #(
- MSYS* | MINGW* ) msys=true ;; #(
- NONSTOP* ) nonstop=true ;;
-esac
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
- if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
- # IBM's JDK on AIX uses strange locations for the executables
- JAVACMD=$JAVA_HOME/jre/sh/java
- else
- JAVACMD=$JAVA_HOME/bin/java
- fi
- if [ ! -x "$JAVACMD" ] ; then
- die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
- fi
-else
- JAVACMD=java
- if ! command -v java >/dev/null 2>&1
- then
- die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
- fi
-fi
-
-# Increase the maximum file descriptors if we can.
-if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
- case $MAX_FD in #(
- max*)
- # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
- # shellcheck disable=SC2039,SC3045
- MAX_FD=$( ulimit -H -n ) ||
- warn "Could not query maximum file descriptor limit"
- esac
- case $MAX_FD in #(
- '' | soft) :;; #(
- *)
- # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
- # shellcheck disable=SC2039,SC3045
- ulimit -n "$MAX_FD" ||
- warn "Could not set maximum file descriptor limit to $MAX_FD"
- esac
-fi
-
-# Collect all arguments for the java command, stacking in reverse order:
-# * args from the command line
-# * the main class name
-# * -classpath
-# * -D...appname settings
-# * --module-path (only if needed)
-# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
-
-# For Cygwin or MSYS, switch paths to Windows format before running java
-if "$cygwin" || "$msys" ; then
- APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
- CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
-
- JAVACMD=$( cygpath --unix "$JAVACMD" )
-
- # Now convert the arguments - kludge to limit ourselves to /bin/sh
- for arg do
- if
- case $arg in #(
- -*) false ;; # don't mess with options #(
- /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
- [ -e "$t" ] ;; #(
- *) false ;;
- esac
- then
- arg=$( cygpath --path --ignore --mixed "$arg" )
- fi
- # Roll the args list around exactly as many times as the number of
- # args, so each arg winds up back in the position where it started, but
- # possibly modified.
- #
- # NB: a `for` loop captures its iteration list before it begins, so
- # changing the positional parameters here affects neither the number of
- # iterations, nor the values presented in `arg`.
- shift # remove old arg
- set -- "$@" "$arg" # push replacement arg
- done
-fi
-
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED"'
-
-# Collect all arguments for the java command:
-# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
-# and any embedded shellness will be escaped.
-# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
-# treated as '${Hostname}' itself on the command line.
-
-set -- \
- "-Dorg.gradle.appname=$APP_BASE_NAME" \
- -classpath "$CLASSPATH" \
- org.gradle.wrapper.GradleWrapperMain \
- "$@"
-
-# Stop when "xargs" is not available.
-if ! command -v xargs >/dev/null 2>&1
-then
- die "xargs is not available"
-fi
-
-# Use "xargs" to parse quoted args.
-#
-# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
-#
-# In Bash we could simply go:
-#
-# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
-# set -- "${ARGS[@]}" "$@"
-#
-# but POSIX shell has neither arrays nor command substitution, so instead we
-# post-process each arg (as a line of input to sed) to backslash-escape any
-# character that might be a shell metacharacter, then use eval to reverse
-# that process (while maintaining the separation between arguments), and wrap
-# the whole thing up as a single "set" statement.
-#
-# This will of course break if any of these variables contains a newline or
-# an unmatched quote.
-#
-
-eval "set -- $(
- printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
- xargs -n1 |
- sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
- tr '\n' ' '
- )" '"$@"'
-
-exec "$JAVACMD" "$@"
diff --git a/index.md b/index.md
new file mode 100644
index 0000000..2c10f43
--- /dev/null
+++ b/index.md
@@ -0,0 +1,87 @@
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+# Apache YuniKorn - A Universal Scheduler
+
+[](https://codecov.io/gh/apache/yunikorn-core)
+[](https://goreportcard.com/report/github.com/apache/yunikorn-core)
+[](https://opensource.org/licenses/Apache-2.0)
+[](https://img.shields.io/github/repo-size/apache/yunikorn-core)
+
+<img src="https://raw.githubusercontent.com/apache/yunikorn-site/master/static/img/logo/yunikorn-logo-blue.png" width="200">
+
+----
+
+[Apache YuniKorn](https://yunikorn.apache.org/) is a light-weight, universal resource scheduler for container orchestrator systems.
+It was created to achieve fine-grained resource sharing for various workloads efficiently on a large scale, multi-tenant,
+and cloud-native environment. YuniKorn brings a unified, cross-platform, scheduling experience for mixed workloads that consist
+of stateless batch workloads and stateful services.
+
+YuniKorn now supports K8s and can be deployed as a custom K8s scheduler. YuniKorn's architecture design also allows adding different
+shim layer and adopt to different ResourceManager implementation including Apache Hadoop YARN, or any other systems.
+
+## Feature highlights
+
+- Gang scheduling
+- Application and queue aware preemption
+- Hierarchy Resource Queues
+- Application aware scheduling
+- Job Ordering and Queuing
+- High throughput scheduling
+- Default Kubernetes workload compatible (daemonsets, jobs, deployments etc)
+
+Read the complete list of features from [here](https://yunikorn.apache.org/docs/get_started/core_features).
+
+## Web UI
+
+YuniKorn has builtin web UIs for queue hierarchies and apps. See below:
+
+
+
+## Supported K8s versions
+
+| K8s Version | Supported from version | Support ended |
+|---------------------|:----------------------:|:-------------:|
+| 1.12.x (or earlier) | - | - |
+| 1.13.x | 0.8.0 | 0.10.0 |
+| 1.14.x | 0.8.0 | 0.10.0 |
+| 1.15.x | 0.8.0 | 0.10.0 |
+| 1.16.x | 0.10.0 | 0.11.0 |
+| 1.17.x | 0.10.0 | 0.11.0 |
+| 1.18.x | 0.10.0 | 0.11.0 |
+| 1.19.x | 0.11.0 | 1.0.0 |
+| 1.20.x | 0.12.1 | 1.2.0 |
+| 1.21.x | 0.12.1 | 1.3.0 |
+| 1.22.x | 0.12.2 | 1.3.0 |
+| 1.23.x | 0.12.2 | 1.3.0 |
+| 1.24.x | 1.0.0 | - |
+| 1.25.x | 1.2.0 | - |
+| 1.26.x | 1.2.0 | - |
+| 1.27.x | 1.4.0 | - |
+| 1.28.x | 1.4.0 | - |
+| 1.29.x | 1.5.0 | - |
+| 1.30.x | 1.6.0 | - |
+| 1.31.x | 1.6.0 | - |
+
+## Useful links
+
+- [Get started](https://yunikorn.apache.org/docs/)
+- [Project roadmap](https://yunikorn.apache.org/community/roadmap)
+- [Performance](https://yunikorn.apache.org/docs/performance/evaluate_perf_function_with_kubemark)
+- [Get involved](https://yunikorn.apache.org/community/get_involved)
+- [Other resources](https://yunikorn.apache.org/community/events)
+
diff --git a/index.yaml b/index.yaml
new file mode 100644
index 0000000..149101d
--- /dev/null
+++ b/index.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+entries:
+ spark-kubernetes-operator:
+ - apiVersion: v2
+ appVersion: 0.1.0
+ created: "2025-05-13T00:30:50.885979468Z"
+ description: A Helm chart for the Apache Spark Kubernetes Operator
+ digest: 67605e5d7a9b4b32f7906e727b8a7f2f8ffd5f61f92690fc38e6e538c7da7641
+ icon: https://spark.apache.org/favicon.ico
+ name: spark-kubernetes-operator
+ type: application
+ urls:
+ - https://github.com/apache/spark-kubernetes-operator/releases/download/v0.1.0/spark-kubernetes-operator-0.1.0.tgz
+ version: 0.1.0
+generated: "2025-05-13T00:30:50.883608234Z"
diff --git a/settings.gradle b/settings.gradle
deleted file mode 100644
index bf5289f..0000000
--- a/settings.gradle
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-rootProject.name = 'apache-spark-kubernetes-operator'
-include 'spark-operator-api'
-include 'spark-submission-worker'
-include 'spark-operator'
-include "build-tools-docs-utils"
-project(':build-tools-docs-utils').projectDir = file('build-tools/docs-utils')
diff --git a/spark-operator-api/build.gradle b/spark-operator-api/build.gradle
deleted file mode 100644
index e07a1ff..0000000
--- a/spark-operator-api/build.gradle
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-dependencies {
- // fabric8
- implementation(libs.kubernetes.client) {
- exclude group: 'com.squareup.okhttp3'
- }
- compileOnly(libs.crd.generator.apt)
- annotationProcessor(libs.crd.generator.apt)
-
- // utils
- implementation(libs.lombok)
- annotationProcessor(libs.lombok)
-
- // logging
- implementation(libs.log4j.slf4j.impl)
- implementation(libs.log4j.core)
-
- testImplementation(platform(libs.junit.bom))
- testImplementation(libs.junit.jupiter)
- testRuntimeOnly(libs.junit.platform.launcher)
-}
-
-test {
- useJUnitPlatform()
-}
-
-// Adds additional printer columns to generated yaml
-// This requires yq
-tasks.register('finalizeGeneratedCRD', Exec) {
- dependsOn jar
- println "Updating PrinterColumns for generated CRD"
- commandLine 'sh', './src/main/resources/printer-columns.sh'
-}
-
-// Copy generated yaml to Helm charts
-tasks.register('relocateGeneratedCRD', Copy) {
- dependsOn finalizeGeneratedCRD
- from "build/classes/java/main/META-INF/fabric8/"
- include "*.spark.apache.org-v1.yml"
- into "../build-tools/helm/spark-kubernetes-operator/crds"
- rename '(.+).yml', '$1.yaml'
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/BaseResource.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/BaseResource.java
deleted file mode 100644
index 10b218d..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/BaseResource.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator;
-
-import io.fabric8.kubernetes.api.model.Namespaced;
-import io.fabric8.kubernetes.client.CustomResource;
-
-import org.apache.spark.k8s.operator.spec.BaseSpec;
-import org.apache.spark.k8s.operator.status.BaseAttemptSummary;
-import org.apache.spark.k8s.operator.status.BaseState;
-import org.apache.spark.k8s.operator.status.BaseStatus;
-
-public class BaseResource<
- S,
- AS extends BaseAttemptSummary,
- STATE extends BaseState<S>,
- SPEC extends BaseSpec,
- STATUS extends BaseStatus<S, STATE, AS>>
- extends CustomResource<SPEC, STATUS> implements Namespaced {}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java
deleted file mode 100644
index 72513ff..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator;
-
-public class Constants {
- public static final String API_GROUP = "spark.apache.org";
- public static final String API_VERSION = "v1alpha1";
- public static final String LABEL_SPARK_APPLICATION_NAME = "spark.operator/spark-app-name";
- public static final String LABEL_SPARK_CLUSTER_NAME = "spark.operator/spark-cluster-name";
- public static final String LABEL_SPARK_OPERATOR_NAME = "spark.operator/name";
- public static final String LABEL_SENTINEL_RESOURCE = "spark.operator/sentinel";
- public static final String LABEL_RESOURCE_NAME = "app.kubernetes.io/name";
- public static final String LABEL_COMPONENT_NAME = "app.kubernetes.io/component";
- public static final String LABEL_SPARK_ROLE_NAME = "spark-role";
- public static final String LABEL_SPARK_ROLE_DRIVER_VALUE = "driver";
- public static final String LABEL_SPARK_ROLE_EXECUTOR_VALUE = "executor";
- public static final String LABEL_SPARK_ROLE_CLUSTER_VALUE = "cluster";
- public static final String LABEL_SPARK_ROLE_MASTER_VALUE = "master";
- public static final String LABEL_SPARK_ROLE_WORKER_VALUE = "worker";
- public static final String LABEL_SPARK_VERSION_NAME = "spark-version";
- public static final String SENTINEL_RESOURCE_DUMMY_FIELD = "sentinel.dummy.number";
-
- public static final String DRIVER_SPARK_CONTAINER_PROP_KEY =
- "spark.kubernetes.driver.podTemplateContainerName";
- public static final String DRIVER_SPARK_TEMPLATE_FILE_PROP_KEY =
- "spark.kubernetes.driver.podTemplateFile";
- public static final String EXECUTOR_SPARK_TEMPLATE_FILE_PROP_KEY =
- "spark.kubernetes.executor.podTemplateFile";
-
- // Default state messages
- public static final String DRIVER_REQUESTED_MESSAGE = "Requested driver from resource scheduler.";
- public static final String DRIVER_COMPLETED_MESSAGE = "Spark application completed successfully.";
- public static final String DRIVER_TERMINATED_BEFORE_INITIALIZATION_MESSAGE =
- "Driver container is terminated without SparkContext / SparkSession initialization.";
- public static final String DRIVER_FAILED_INIT_CONTAINERS_MESSAGE =
- "Driver has failed init container(s). Refer last observed status for details.";
- public static final String DRIVER_FAILED_MESSAGE =
- "Driver has one or more failed critical container(s), refer last observed status for "
- + "details.";
- public static final String DRIVER_SUCCEEDED_MESSAGE =
- "Driver has critical container(s) exited with 0.";
- public static final String DRIVER_RESTARTED_MESSAGE =
- "Driver has one or more critical container(s) restarted unexpectedly, refer last "
- + "observed status for details.";
- public static final String APP_CANCELLED_MESSAGE =
- "Spark application has been shutdown as requested.";
- public static final String DRIVER_UNEXPECTED_REMOVED_MESSAGE =
- "Driver removed. This could caused by 'exit' called in driver process with non-zero "
- + "code, involuntary disruptions or unintentional destroy behavior, check "
- + "Kubernetes events for more details.";
- public static final String DRIVER_LAUNCH_TIMEOUT_MESSAGE =
- "The driver has not responded to the initial health check request within the "
- + "allotted start-up time. This can be configured by setting "
- + ".spec.applicationTolerations.applicationTimeoutConfig.";
- public static final String DRIVER_RUNNING_MESSAGE = "Driver has started running.";
- public static final String DRIVER_READY_MESSAGE = "Driver has reached ready state.";
- public static final String SUBMITTED_STATE_MESSAGE =
- "Spark application has been created on Kubernetes Cluster.";
- public static final String UNKNOWN_STATE_MESSAGE = "Cannot process application status.";
- public static final String EXCEED_MAX_RETRY_ATTEMPT_MESSAGE =
- "The maximum number of restart attempts (%d) has been exceeded.";
- public static final String SCHEDULE_FAILURE_MESSAGE =
- "Failed to request driver from scheduler backend.";
- public static final String RUNNING_HEALTHY_MESSAGE = "Application is running healthy.";
- public static final String INITIALIZED_WITH_BELOW_THRESHOLD_EXECUTORS_MESSAGE =
- "The application is running with less than minimal number of requested initial executors.";
- public static final String RUNNING_WITH_BELOW_THRESHOLD_EXECUTORS_MESSAGE =
- "The Spark application is running with less than minimal number of requested executors.";
- public static final String EXECUTOR_LAUNCH_TIMEOUT_MESSAGE =
- "The Spark application failed to get enough executors in the given time threshold.";
-
- // Spark Cluster Messages
- public static final String CLUSTER_SCHEDULE_FAILURE_MESSAGE =
- "Failed to request Spark cluster from scheduler backend.";
- public static final String CLUSTER_SUBMITTED_STATE_MESSAGE =
- "Spark cluster has been submitted to Kubernetes Cluster.";
- public static final String CLUSTER_READY_MESSAGE = "Cluster has reached ready state.";
- public static final String UNKNOWN_CLUSTER_STATE_MESSAGE = "Cannot process cluster status.";
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplication.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplication.java
deleted file mode 100644
index 3717ee7..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplication.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import io.fabric8.kubernetes.model.annotation.Group;
-import io.fabric8.kubernetes.model.annotation.ShortNames;
-import io.fabric8.kubernetes.model.annotation.Version;
-
-import org.apache.spark.k8s.operator.spec.ApplicationSpec;
-import org.apache.spark.k8s.operator.status.ApplicationAttemptSummary;
-import org.apache.spark.k8s.operator.status.ApplicationState;
-import org.apache.spark.k8s.operator.status.ApplicationStateSummary;
-import org.apache.spark.k8s.operator.status.ApplicationStatus;
-
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonDeserialize()
-@Group(Constants.API_GROUP)
-@Version(Constants.API_VERSION)
-@ShortNames({"sparkapp"})
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class SparkApplication
- extends BaseResource<
- ApplicationStateSummary,
- ApplicationAttemptSummary,
- ApplicationState,
- ApplicationSpec,
- ApplicationStatus> {
- @Override
- public ApplicationStatus initStatus() {
- return new ApplicationStatus();
- }
-
- @Override
- public ApplicationSpec initSpec() {
- return new ApplicationSpec();
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplicationList.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplicationList.java
deleted file mode 100644
index 9da21e6..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplicationList.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator;
-
-import io.fabric8.kubernetes.api.model.DefaultKubernetesResourceList;
-import lombok.NoArgsConstructor;
-
-@NoArgsConstructor
-public class SparkApplicationList extends DefaultKubernetesResourceList<SparkApplication> {}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkCluster.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkCluster.java
deleted file mode 100644
index 4f0c689..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkCluster.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import io.fabric8.kubernetes.model.annotation.Group;
-import io.fabric8.kubernetes.model.annotation.ShortNames;
-import io.fabric8.kubernetes.model.annotation.Version;
-
-import org.apache.spark.k8s.operator.spec.ClusterSpec;
-import org.apache.spark.k8s.operator.status.ClusterAttemptSummary;
-import org.apache.spark.k8s.operator.status.ClusterState;
-import org.apache.spark.k8s.operator.status.ClusterStateSummary;
-import org.apache.spark.k8s.operator.status.ClusterStatus;
-
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonDeserialize()
-@Group(Constants.API_GROUP)
-@Version(Constants.API_VERSION)
-@ShortNames({"sparkcluster"})
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class SparkCluster
- extends BaseResource<
- ClusterStateSummary, ClusterAttemptSummary, ClusterState, ClusterSpec, ClusterStatus> {
- @Override
- public ClusterStatus initStatus() {
- return new ClusterStatus();
- }
-
- @Override
- public ClusterSpec initSpec() {
- return new ClusterSpec();
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkClusterList.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkClusterList.java
deleted file mode 100644
index 035b78c..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkClusterList.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator;
-
-import io.fabric8.kubernetes.api.model.DefaultKubernetesResourceList;
-import lombok.NoArgsConstructor;
-
-@NoArgsConstructor
-public class SparkClusterList extends DefaultKubernetesResourceList<SparkCluster> {}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/decorators/ResourceDecorator.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/decorators/ResourceDecorator.java
deleted file mode 100644
index 8c7a3bf..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/decorators/ResourceDecorator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.decorators;
-
-import io.fabric8.kubernetes.api.model.HasMetadata;
-
-public interface ResourceDecorator {
- <T extends HasMetadata> T decorate(T resource);
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/diff/Diffable.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/diff/Diffable.java
deleted file mode 100644
index f26a96a..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/diff/Diffable.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.diff;
-
-public interface Diffable<T> {}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationSpec.java
deleted file mode 100644
index 9d7722f..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationSpec.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.generator.annotation.Required;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@EqualsAndHashCode(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ApplicationSpec extends BaseSpec {
- protected String mainClass;
- @Required protected RuntimeVersions runtimeVersions;
- protected String jars;
- protected String pyFiles;
- protected String sparkRFiles;
- protected String files;
- @Builder.Default protected DeploymentMode deploymentMode = DeploymentMode.ClusterMode;
- protected String proxyUser;
- @Builder.Default protected List<String> driverArgs = new ArrayList<>();
-
- @Builder.Default
- protected ApplicationTolerations applicationTolerations = new ApplicationTolerations();
-
- protected BaseApplicationTemplateSpec driverSpec;
- protected BaseApplicationTemplateSpec executorSpec;
- protected List<DriverServiceIngressSpec> driverServiceIngressList;
- protected List<ConfigMapSpec> configMapSpecs;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTimeoutConfig.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTimeoutConfig.java
deleted file mode 100644
index 2bf9632..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTimeoutConfig.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ApplicationTimeoutConfig {
- /**
- * Operator may proactively terminate apps that cannot reach running healthy state to avoid
- * resource deadlock when batch scheduler is not integrated in the cluster.
- */
-
- /*
- * Time to wait for driver reaches 'Running' state after requested driver, default to 5min
- * https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
- */
- @Builder.Default protected Long driverStartTimeoutMillis = 300 * 1000L;
-
- /*
- * Time to wait for driver becomes 'Ready', default to 5min
- * https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions
- */
- @Builder.Default protected Long driverReadyTimeoutMillis = 300 * 1000L;
- /*
- * Time to wait for minimal desired number of executor pods become 'Ready', default to 5min
- * https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions
- */
- @Builder.Default protected Long executorStartTimeoutMillis = 300 * 1000L;
- /*
- * Time to wait for force delete resources at the end of attempt, default to 5min
- */
- @Builder.Default protected Long forceTerminationGracePeriodMillis = 300 * 1000L;
- /*
- * Backoff time before operator requeues the reconcile request when it cannot delete app
- * resource when cleaning up, default to 2s
- */
- @Builder.Default protected Long terminationRequeuePeriodMillis = 2 * 1000L;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTolerations.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTolerations.java
deleted file mode 100644
index f1fadf7..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTolerations.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ApplicationTolerations {
- @Builder.Default protected RestartConfig restartConfig = new RestartConfig();
-
- @Builder.Default
- protected ApplicationTimeoutConfig applicationTimeoutConfig = new ApplicationTimeoutConfig();
-
- /** Determine the toleration behavior for executor / worker instances. */
- @Builder.Default protected ExecutorInstanceConfig instanceConfig = new ExecutorInstanceConfig();
-
- @Builder.Default protected ResourceRetainPolicy resourceRetainPolicy = ResourceRetainPolicy.Never;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseApplicationTemplateSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseApplicationTemplateSpec.java
deleted file mode 100644
index 7fd7e3a..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseApplicationTemplateSpec.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.kubernetes.api.model.PodTemplateSpec;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class BaseApplicationTemplateSpec {
- protected PodTemplateSpec podTemplateSpec;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java
deleted file mode 100644
index 3522ad3..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.Data;
-
-import org.apache.spark.k8s.operator.diff.Diffable;
-
-@Data
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class BaseSpec implements Diffable<BaseSpec> {
- protected Map<String, String> sparkConf = new HashMap<>();
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterSpec.java
deleted file mode 100644
index 408f8d6..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterSpec.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.generator.annotation.Required;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@EqualsAndHashCode(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ClusterSpec extends BaseSpec {
- @Required @Builder.Default protected RuntimeVersions runtimeVersions = new RuntimeVersions();
-
- @Required @Builder.Default
- protected ClusterTolerations clusterTolerations = new ClusterTolerations();
-
- @Builder.Default protected MasterSpec masterSpec = new MasterSpec.MasterSpecBuilder().build();
- @Builder.Default protected WorkerSpec workerSpec = new WorkerSpec.WorkerSpecBuilder().build();
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterTolerations.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterTolerations.java
deleted file mode 100644
index d7e546b..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterTolerations.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.generator.annotation.Required;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ClusterTolerations {
- /** Determine the toleration behavior for worker instances. */
- @Required @Builder.Default
- protected WorkerInstanceConfig instanceConfig = new WorkerInstanceConfig();
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ConfigMapSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ConfigMapSpec.java
deleted file mode 100644
index 8cce3fd..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ConfigMapSpec.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import java.util.Map;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.generator.annotation.Required;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ConfigMapSpec {
- @Required protected String name;
- protected Map<String, String> labels;
- protected Map<String, String> annotations;
- protected Map<String, String> data;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DeploymentMode.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DeploymentMode.java
deleted file mode 100644
index 47e6b3c..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DeploymentMode.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-public enum DeploymentMode {
- ClusterMode,
- ClientMode
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DriverServiceIngressSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DriverServiceIngressSpec.java
deleted file mode 100644
index 9712a65..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DriverServiceIngressSpec.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.generator.annotation.Required;
-import io.fabric8.kubernetes.api.model.ObjectMeta;
-import io.fabric8.kubernetes.api.model.ServiceSpec;
-import io.fabric8.kubernetes.api.model.networking.v1.IngressSpec;
-import lombok.Builder;
-import lombok.Data;
-
-@Data
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class DriverServiceIngressSpec {
- @Required protected ObjectMeta serviceMetadata;
- @Required protected ServiceSpec serviceSpec;
-
- @Required protected ObjectMeta ingressMetadata;
- protected IngressSpec ingressSpec;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ExecutorInstanceConfig.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ExecutorInstanceConfig.java
deleted file mode 100644
index 3325445..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ExecutorInstanceConfig.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-/**
- * Config tolerations of executor instances for the application. Used then the target cluster is
- * lack of batch / gang scheduling This is different from SparkConf: spark.executor.instances
- *
- * <p>For example, with below spec:
- *
- * <pre>{@code
- * spec:
- * applicationTolerations:
- * instanceConfig:
- * minExecutors: 3
- * initExecutors: 5
- * maxExecutors: 10
- * sparkConf:
- * spark.executor.instances: "10"
- * }</pre>
- *
- * Spark would try to bring up 10 executors as defined in SparkConf. In addition, from SparkApp
- * perspective,
- *
- * <ul>
- * <li>If Spark app acquires less than 5 executors in given tine window
- * (.spec.applicationTolerations.applicationTimeoutConfig.executorStartTimeoutMillis) after
- * submitted, it would be shut down proactively in order to avoid resource deadlock.
- * <li>Spark app would be marked as 'RUNNING_WITH_PARTIAL_CAPACITY' if it loses executors after
- * successfully start up.
- * <li>Spark app would be marked as 'RunningHealthy' if it has at least min executors after
- * successfully start up.
- * </ul>
- */
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ExecutorInstanceConfig {
- @Builder.Default protected int initExecutors = 0;
- @Builder.Default protected int minExecutors = 0;
- @Builder.Default protected int maxExecutors = 0;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/MasterSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/MasterSpec.java
deleted file mode 100644
index c04a2be..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/MasterSpec.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.kubernetes.api.model.ObjectMeta;
-import io.fabric8.kubernetes.api.model.ServiceSpec;
-import io.fabric8.kubernetes.api.model.apps.StatefulSetSpec;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class MasterSpec {
- protected StatefulSetSpec statefulSetSpec;
- protected ObjectMeta statefulSetMetadata;
- protected ServiceSpec serviceSpec;
- protected ObjectMeta serviceMetadata;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ResourceRetainPolicy.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ResourceRetainPolicy.java
deleted file mode 100644
index 322d82f..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ResourceRetainPolicy.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-public enum ResourceRetainPolicy {
- /**
- * Configure operator to delete / retain resources for an app after it terminates. This controls
- * the resources created directly by Operator (spark-submit), including driver pod and its
- * configmap. Resources created by driver are still controlled by driver (via SparkConf). Tuning
- * 'spark.kubernetes.driver.service.deleteOnTermination' and `spark.kubernetes.executor
- * .deleteOnTermination`in application for resource retaining policy is still necessary. Default
- * to 'Never' so operator deletes all created Spark resources after app terminates. Setting this
- * to 'Always' / 'OnFailure' to indicate operator not to delete resources based on app state, this
- * can be helpful in dev phase to debug failed Spark pod behavior. Driver has owner reference
- * configured to make sure it's garbage collected by k8s. Please be advised that resource would
- * not be retained if application is configured to restart, this is to avoid resource quota usage
- * increase unexpectedly or resource conflicts among multiple attempts.
- */
- Always,
- OnFailure,
- Never
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartConfig.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartConfig.java
deleted file mode 100644
index 2c19cd2..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartConfig.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class RestartConfig {
- @Builder.Default protected RestartPolicy restartPolicy = RestartPolicy.Never;
- @Builder.Default protected Long maxRestartAttempts = 3L;
- @Builder.Default protected Long restartBackoffMillis = 30000L;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartPolicy.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartPolicy.java
deleted file mode 100644
index 8eac7ff..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartPolicy.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import org.apache.spark.k8s.operator.status.BaseStateSummary;
-
-public enum RestartPolicy {
- Always,
- Never,
- OnFailure,
- OnInfrastructureFailure;
-
- public static boolean attemptRestartOnState(
- final RestartPolicy policy, final BaseStateSummary stateSummary) {
- return switch (policy) {
- case Always -> true;
- case Never -> false;
- case OnFailure -> stateSummary.isFailure();
- case OnInfrastructureFailure -> stateSummary.isInfrastructureFailure();
- };
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RuntimeVersions.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RuntimeVersions.java
deleted file mode 100644
index 819cca8..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RuntimeVersions.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.generator.annotation.Required;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class RuntimeVersions {
- @Required protected String sparkVersion;
- protected String scalaVersion;
- protected String jdkVersion;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerInstanceConfig.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerInstanceConfig.java
deleted file mode 100644
index 5adca3e..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerInstanceConfig.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.generator.annotation.Required;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class WorkerInstanceConfig {
- @Required @Builder.Default protected int initWorkers = 0;
- @Required @Builder.Default protected int minWorkers = 0;
- @Required @Builder.Default protected int maxWorkers = 0;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerSpec.java
deleted file mode 100644
index 78b86e3..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerSpec.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.kubernetes.api.model.ObjectMeta;
-import io.fabric8.kubernetes.api.model.ServiceSpec;
-import io.fabric8.kubernetes.api.model.apps.StatefulSetSpec;
-import io.fabric8.kubernetes.api.model.autoscaling.v2.HorizontalPodAutoscalerSpec;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class WorkerSpec {
- protected StatefulSetSpec statefulSetSpec;
- protected ObjectMeta statefulSetMetadata;
- protected ServiceSpec serviceSpec;
- protected ObjectMeta serviceMetadata;
- protected HorizontalPodAutoscalerSpec horizontalPodAutoscalerSpec;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationAttemptSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationAttemptSummary.java
deleted file mode 100644
index a901935..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationAttemptSummary.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import java.util.SortedMap;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.ToString;
-
-@Getter
-@EqualsAndHashCode(callSuper = true)
-@ToString(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ApplicationAttemptSummary extends BaseAttemptSummary {
- // The state transition history for given attempt
- // This is used when state history trimming is enabled
- protected final SortedMap<Long, ApplicationState> stateTransitionHistory;
-
- public ApplicationAttemptSummary(
- AttemptInfo attemptInfo, SortedMap<Long, ApplicationState> stateTransitionHistory) {
- super(attemptInfo);
- this.stateTransitionHistory = stateTransitionHistory;
- }
-
- public ApplicationAttemptSummary() {
- this(new AttemptInfo(), null);
- }
-
- public ApplicationAttemptSummary(AttemptInfo attemptInfo) {
- this(attemptInfo, null);
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationState.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationState.java
deleted file mode 100644
index 33614f8..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationState.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import static org.apache.spark.k8s.operator.Constants.SUBMITTED_STATE_MESSAGE;
-
-import java.io.Serializable;
-import java.time.Instant;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import io.fabric8.kubernetes.api.model.PodStatus;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.ToString;
-
-@ToString(callSuper = true)
-@EqualsAndHashCode(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ApplicationState extends BaseState<ApplicationStateSummary> implements Serializable {
-
- @Getter @Setter PodStatus lastObservedDriverStatus;
-
- public ApplicationState() {
- super(ApplicationStateSummary.Submitted, Instant.now().toString(), SUBMITTED_STATE_MESSAGE);
- }
-
- public ApplicationState(ApplicationStateSummary currentStateSummary, String message) {
- super(currentStateSummary, Instant.now().toString(), message);
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStateSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStateSummary.java
deleted file mode 100644
index fc5fed1..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStateSummary.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import java.util.Set;
-
-public enum ApplicationStateSummary implements BaseStateSummary {
- /** Spark application is submitted to the cluster but yet scheduled */
- Submitted,
-
- /** Spark application will be restarted with same configuration */
- ScheduledToRestart,
-
- /** A request has been made to start driver pod in the cluster */
- DriverRequested,
-
- /**
- * Driver pod has reached 'Running' state and thus bound to a node Refer
- * https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
- */
- DriverStarted,
-
- /**
- * Driver pod is ready to serve connections from executors Refer Refer
- * https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
- */
- DriverReady,
-
- /**
- * Less that minimal required executor pods reached condition 'Ready' during starting up Note that
- * reaching 'Ready' does not necessarily mean that the executor has successfully registered with
- * driver. This is a best-effort from operator to detect executor status
- */
- InitializedBelowThresholdExecutors,
-
- /**
- * All required executor pods started reached condition 'Ready' Note that reaching 'Ready' does
- * not necessarily mean that the executor has successfully registered with driver. This is a
- * best-effort from operator to detect executor status
- */
- RunningHealthy,
-
- /** The application has lost a fraction of executors for external reasons */
- RunningWithBelowThresholdExecutors,
-
- /** The request timed out for driver */
- DriverStartTimedOut,
-
- /** The request timed out for executors */
- ExecutorsStartTimedOut,
-
- /** Timed out waiting for driver to become ready */
- DriverReadyTimedOut,
-
- /**
- * The application completed successfully, or System.exit() is called explicitly with zero state
- */
- Succeeded,
-
- /**
- * The application has failed, JVM exited abnormally, or System.exit is called explicitly with
- * non-zero state
- */
- Failed,
-
- /**
- * Operator failed to orchestrate Spark application in cluster. For example, the given pod
- * template is rejected by API server because it's invalid or does not meet cluster security
- * standard; operator is not able to schedule pods due to insufficient quota or missing RBAC
- */
- SchedulingFailure,
-
- /** The driver pod was failed with Evicted reason */
- DriverEvicted,
-
- /** all resources (pods, services .etc have been cleaned up) */
- ResourceReleased,
-
- /**
- * If configured, operator may mark app as terminated without releasing resources. While this can
- * be helpful in dev phase, it shall not be enabled for prod use cases
- */
- TerminatedWithoutReleaseResources;
-
- public static final Set<ApplicationStateSummary> infrastructureFailures =
- Set.of(DriverStartTimedOut, ExecutorsStartTimedOut, SchedulingFailure);
-
- public static final Set<ApplicationStateSummary> failures =
- Set.of(
- DriverStartTimedOut,
- ExecutorsStartTimedOut,
- SchedulingFailure,
- DriverEvicted,
- Failed,
- DriverReadyTimedOut);
-
- public boolean isInitializing() {
- return Submitted.equals(this) || ScheduledToRestart.equals(this);
- }
-
- public boolean isStarting() {
- return ScheduledToRestart.ordinal() < this.ordinal()
- && RunningHealthy.ordinal() > this.ordinal();
- }
-
- /**
- * A state is 'terminated' if and only if no further actions are needed to reconcile it
- *
- * @return true if the state indicates app has terminated
- */
- public boolean isTerminated() {
- return ResourceReleased.equals(this) || TerminatedWithoutReleaseResources.equals(this);
- }
-
- /**
- * When state is 'stopping', operator releases its resources based on retain policy, and perform
- * retry based on retry policy
- *
- * @return true if an app is stopping
- */
- public boolean isStopping() {
- return RunningWithBelowThresholdExecutors.ordinal() < this.ordinal() && !isTerminated();
- }
-
- @Override
- public boolean isFailure() {
- return failures.contains(this);
- }
-
- @Override
- public boolean isInfrastructureFailure() {
- return infrastructureFailures.contains(this);
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStatus.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStatus.java
deleted file mode 100644
index 2813440..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStatus.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import static org.apache.spark.k8s.operator.Constants.EXCEED_MAX_RETRY_ATTEMPT_MESSAGE;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-import org.apache.spark.k8s.operator.spec.ResourceRetainPolicy;
-import org.apache.spark.k8s.operator.spec.RestartConfig;
-import org.apache.spark.k8s.operator.spec.RestartPolicy;
-
-@EqualsAndHashCode(callSuper = true)
-@ToString(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ApplicationStatus
- extends BaseStatus<ApplicationStateSummary, ApplicationState, ApplicationAttemptSummary> {
-
- public ApplicationStatus() {
- super(new ApplicationState(), new ApplicationAttemptSummary());
- }
-
- public ApplicationStatus(
- ApplicationState currentState,
- Map<Long, ApplicationState> stateTransitionHistory,
- ApplicationAttemptSummary previousAttemptSummary,
- ApplicationAttemptSummary currentAttemptSummary) {
- super(currentState, stateTransitionHistory, previousAttemptSummary, currentAttemptSummary);
- }
-
- /**
- * Create a new ApplicationStatus, set the given latest state as current and update state history
- */
- public ApplicationStatus appendNewState(ApplicationState state) {
- return new ApplicationStatus(
- state,
- createUpdatedHistoryWithNewState(state),
- previousAttemptSummary,
- currentAttemptSummary);
- }
-
- /**
- * Create ApplicationStatus to be updated upon termination of current attempt, with respect to
- * current state and restart config.
- *
- * @param restartConfig restart config for the app
- * @param resourceRetainPolicy resourceRetainPolicy for the app
- * @param stateMessageOverride state message to be applied
- * @param trimStateTransitionHistory if enabled, operator would trim the state history, keeping
- * only previous and current attempt.
- * @return updated ApplicationStatus
- */
- public ApplicationStatus terminateOrRestart(
- final RestartConfig restartConfig,
- final ResourceRetainPolicy resourceRetainPolicy,
- String stateMessageOverride,
- boolean trimStateTransitionHistory) {
- if (!currentState.currentStateSummary.isStopping()) {
- // application is not stopping, skip
- throw new IllegalStateException(
- "Spark application cannot be directly terminated unless in stopping "
- + "state, current state is: "
- + currentState);
- }
-
- if (!RestartPolicy.attemptRestartOnState(
- restartConfig.getRestartPolicy(), currentState.getCurrentStateSummary())) {
- // no restart configured
- ApplicationState state =
- new ApplicationState(ApplicationStateSummary.ResourceReleased, stateMessageOverride);
- if (ResourceRetainPolicy.Always.equals(resourceRetainPolicy)
- || ResourceRetainPolicy.OnFailure.equals(resourceRetainPolicy)
- && currentState.currentStateSummary.isFailure()) {
- state = terminateAppWithoutReleaseResource(stateMessageOverride);
- }
- return new ApplicationStatus(
- state,
- createUpdatedHistoryWithNewState(state),
- previousAttemptSummary,
- currentAttemptSummary);
- }
-
- if (currentAttemptSummary.getAttemptInfo().getId() >= restartConfig.getMaxRestartAttempts()) {
- String stateMessage =
- String.format(EXCEED_MAX_RETRY_ATTEMPT_MESSAGE, restartConfig.getMaxRestartAttempts());
- if (stateMessageOverride != null && !stateMessageOverride.isEmpty()) {
- stateMessage += stateMessageOverride;
- }
- // max number of restart attempt reached
- ApplicationState state =
- new ApplicationState(ApplicationStateSummary.ResourceReleased, stateMessage);
- if (ResourceRetainPolicy.Always.equals(resourceRetainPolicy)
- || ResourceRetainPolicy.OnFailure.equals(resourceRetainPolicy)
- && currentState.currentStateSummary.isFailure()) {
- state = terminateAppWithoutReleaseResource(stateMessage);
- }
- // still use previous & current attempt summary - they are to be updated only upon
- // new restart
- return new ApplicationStatus(
- state,
- createUpdatedHistoryWithNewState(state),
- previousAttemptSummary,
- currentAttemptSummary);
- }
-
- AttemptInfo nextAttemptInfo = currentAttemptSummary.getAttemptInfo().createNextAttemptInfo();
- ApplicationAttemptSummary nextAttemptSummary = new ApplicationAttemptSummary(nextAttemptInfo);
- ApplicationState state =
- new ApplicationState(ApplicationStateSummary.ScheduledToRestart, stateMessageOverride);
-
- if (trimStateTransitionHistory) {
- // when truncating, put all previous history entries into previous attempt summary
- ApplicationAttemptSummary newPrevSummary =
- new ApplicationAttemptSummary(
- currentAttemptSummary.getAttemptInfo(), stateTransitionHistory);
- return new ApplicationStatus(
- state,
- Collections.singletonMap(getCurrentStateId() + 1, state),
- newPrevSummary,
- nextAttemptSummary);
- } else {
- // when truncating is disabled, currentAttempt becomes the new 'previous'
- return new ApplicationStatus(
- state,
- createUpdatedHistoryWithNewState(state),
- currentAttemptSummary,
- nextAttemptSummary);
- }
- }
-
- private ApplicationState terminateAppWithoutReleaseResource(String stateMessageOverride) {
- String stateMessage =
- "Application is terminated without releasing resources as configured."
- + stateMessageOverride;
- return new ApplicationState(
- ApplicationStateSummary.TerminatedWithoutReleaseResources, stateMessage);
- }
-
- private Map<Long, ApplicationState> createUpdatedHistoryWithNewState(ApplicationState state) {
- TreeMap<Long, ApplicationState> updatedHistory = new TreeMap<>(stateTransitionHistory);
- updatedHistory.put(updatedHistory.lastKey() + 1L, state);
- return updatedHistory;
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/AttemptInfo.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/AttemptInfo.java
deleted file mode 100644
index 84c8b76..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/AttemptInfo.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.ToString;
-
-@NoArgsConstructor
-@AllArgsConstructor
-@Builder
-@EqualsAndHashCode
-@ToString
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class AttemptInfo {
- @Getter @Builder.Default protected final long id = 0L;
-
- public AttemptInfo createNextAttemptInfo() {
- return new AttemptInfo(id + 1L);
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseAttemptSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseAttemptSummary.java
deleted file mode 100644
index 8078529..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseAttemptSummary.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.RequiredArgsConstructor;
-import lombok.ToString;
-
-@EqualsAndHashCode
-@ToString
-@Getter
-@RequiredArgsConstructor
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class BaseAttemptSummary {
- protected final AttemptInfo attemptInfo;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseState.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseState.java
deleted file mode 100644
index e73cd35..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseState.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class BaseState<T> implements Serializable {
- protected T currentStateSummary;
- protected String lastTransitionTime;
- protected String message;
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStateSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStateSummary.java
deleted file mode 100644
index f398dc8..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStateSummary.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-/** State for Spark Custom Resource */
-public interface BaseStateSummary {
- /** The CustomResource has failed */
- boolean isFailure();
-
- /** The CustomResource has failed for infrastructure reasons */
- boolean isInfrastructureFailure();
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStatus.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStatus.java
deleted file mode 100644
index ab1d642..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStatus.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.ToString;
-
-@ToString
-@EqualsAndHashCode
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class BaseStatus<S, STATE extends BaseState<S>, AS extends BaseAttemptSummary> {
- @Getter final STATE currentState;
- @Getter final SortedMap<Long, STATE> stateTransitionHistory;
- @Getter final AS previousAttemptSummary;
- @Getter final AS currentAttemptSummary;
-
- public BaseStatus(STATE initState, AS currentAttemptSummary) {
- this.currentState = initState;
- this.stateTransitionHistory = new TreeMap<>();
- this.stateTransitionHistory.put(0L, initState);
- this.previousAttemptSummary = null;
- this.currentAttemptSummary = currentAttemptSummary;
- }
-
- public BaseStatus(
- STATE currentState,
- Map<Long, STATE> stateTransitionHistory,
- AS previousAttemptSummary,
- AS currentAttemptSummary) {
- this.currentState = currentState;
- this.stateTransitionHistory = new TreeMap<>(stateTransitionHistory);
- this.previousAttemptSummary = previousAttemptSummary;
- this.currentAttemptSummary = currentAttemptSummary;
- }
-
- protected long getCurrentStateId() {
- return stateTransitionHistory.lastKey();
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterAttemptSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterAttemptSummary.java
deleted file mode 100644
index e0fb7cd..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterAttemptSummary.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import java.util.SortedMap;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.ToString;
-
-@Getter
-@EqualsAndHashCode(callSuper = true)
-@ToString(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ClusterAttemptSummary extends BaseAttemptSummary {
- protected final SortedMap<Long, ClusterState> stateTransitionHistory;
-
- public ClusterAttemptSummary(
- AttemptInfo attemptInfo, SortedMap<Long, ClusterState> stateTransitionHistory) {
- super(attemptInfo);
- this.stateTransitionHistory = stateTransitionHistory;
- }
-
- public ClusterAttemptSummary() {
- this(new AttemptInfo(), null);
- }
-
- public ClusterAttemptSummary(AttemptInfo attemptInfo) {
- this(attemptInfo, null);
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterState.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterState.java
deleted file mode 100644
index 2d6a832..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterState.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import static org.apache.spark.k8s.operator.Constants.CLUSTER_SUBMITTED_STATE_MESSAGE;
-
-import java.io.Serializable;
-import java.time.Instant;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-@ToString(callSuper = true)
-@EqualsAndHashCode(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ClusterState extends BaseState<ClusterStateSummary> implements Serializable {
- public ClusterState() {
- super(ClusterStateSummary.Submitted, Instant.now().toString(), CLUSTER_SUBMITTED_STATE_MESSAGE);
- }
-
- public ClusterState(ClusterStateSummary currentStateSummary, String message) {
- super(currentStateSummary, Instant.now().toString(), message);
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStateSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStateSummary.java
deleted file mode 100644
index b71a529..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStateSummary.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-public enum ClusterStateSummary implements BaseStateSummary {
- /** Spark cluster is submitted but yet scheduled */
- Submitted,
-
- /** Spark cluster fails to schedule. */
- SchedulingFailure,
-
- /** Cluster is running healthy */
- RunningHealthy,
-
- /** Cluster failed */
- Failed,
-
- /** all resources (pods, services .etc have been cleaned up) */
- ResourceReleased;
-
- public boolean isInitializing() {
- return Submitted.equals(this);
- }
-
- public boolean isStarting() {
- return RunningHealthy.ordinal() > this.ordinal();
- }
-
- /**
- * A state is 'terminated' if and only if no further actions are needed to reconcile it.
- *
- * @return true if the state indicates the cluster has terminated
- */
- public boolean isTerminated() {
- return ResourceReleased.equals(this);
- }
-
- @Override
- public boolean isFailure() {
- return SchedulingFailure.equals(this) || Failed.equals(this);
- }
-
- @Override
- public boolean isInfrastructureFailure() {
- return SchedulingFailure.equals(this);
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStatus.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStatus.java
deleted file mode 100644
index 73c9207..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStatus.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-@EqualsAndHashCode(callSuper = true)
-@ToString(callSuper = true)
-@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ClusterStatus
- extends BaseStatus<ClusterStateSummary, ClusterState, ClusterAttemptSummary> {
-
- public ClusterStatus() {
- super(new ClusterState(), new ClusterAttemptSummary());
- }
-
- public ClusterStatus(
- ClusterState currentState,
- Map<Long, ClusterState> stateTransitionHistory,
- ClusterAttemptSummary previousAttemptSummary,
- ClusterAttemptSummary currentAttemptSummary) {
- super(currentState, stateTransitionHistory, previousAttemptSummary, currentAttemptSummary);
- }
-
- /** Create a new ClusterStatus, set the given latest state as current and update state history */
- public ClusterStatus appendNewState(ClusterState state) {
- return new ClusterStatus(
- state,
- createUpdatedHistoryWithNewState(state),
- previousAttemptSummary,
- currentAttemptSummary);
- }
-
- private Map<Long, ClusterState> createUpdatedHistoryWithNewState(ClusterState state) {
- TreeMap<Long, ClusterState> updatedHistory = new TreeMap<>(stateTransitionHistory);
- updatedHistory.put(updatedHistory.lastKey() + 1L, state);
- return updatedHistory;
- }
-}
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/utils/ModelUtils.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/utils/ModelUtils.java
deleted file mode 100644
index e9ed22e..0000000
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/utils/ModelUtils.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.utils;
-
-import static org.apache.spark.k8s.operator.Constants.DRIVER_SPARK_CONTAINER_PROP_KEY;
-
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import io.fabric8.kubernetes.api.model.ContainerStatus;
-import io.fabric8.kubernetes.api.model.HasMetadata;
-import io.fabric8.kubernetes.api.model.OwnerReference;
-import io.fabric8.kubernetes.api.model.OwnerReferenceBuilder;
-import io.fabric8.kubernetes.api.model.Pod;
-import io.fabric8.kubernetes.api.model.PodBuilder;
-import io.fabric8.kubernetes.api.model.PodTemplateSpec;
-
-import org.apache.spark.k8s.operator.SparkApplication;
-import org.apache.spark.k8s.operator.spec.ApplicationSpec;
-
-public final class ModelUtils {
-
- public static final ObjectMapper objectMapper = new ObjectMapper();
-
- private ModelUtils() {}
-
- public static Pod getPodFromTemplateSpec(PodTemplateSpec podTemplateSpec) {
- if (podTemplateSpec == null) {
- return new PodBuilder().withNewMetadata().endMetadata().withNewSpec().endSpec().build();
- }
- return new PodBuilder()
- .withMetadata(podTemplateSpec.getMetadata())
- .withSpec(podTemplateSpec.getSpec())
- .withAdditionalProperties(podTemplateSpec.getAdditionalProperties())
- .build();
- }
-
- /**
- * Find the Spark main container(s) in driver pod. If `spark.kubernetes.driver
- * .podTemplateContainerName` is not set, all containers are considered as main container from
- * health monitoring perspective
- */
- public static List<ContainerStatus> findDriverMainContainerStatus(
- final ApplicationSpec appSpec, final List<ContainerStatus> containerStatusList) {
- if (appSpec == null) {
- return containerStatusList;
- }
- Map<String, String> sparkConf = appSpec.getSparkConf();
- String key = sparkConf.get(DRIVER_SPARK_CONTAINER_PROP_KEY);
- if (key == null || key.isEmpty()) {
- return containerStatusList;
- }
- String mainContainerName = sparkConf.get(DRIVER_SPARK_CONTAINER_PROP_KEY);
- return containerStatusList.stream()
- .filter(c -> mainContainerName.equalsIgnoreCase(c.getName()))
- .collect(Collectors.toList());
- }
-
- /**
- * Build OwnerReference to the given resource
- *
- * @param owner the owner
- * @return OwnerReference to be used for subresources
- */
- public static OwnerReference buildOwnerReferenceTo(HasMetadata owner) {
- return new OwnerReferenceBuilder()
- .withName(owner.getMetadata().getName())
- .withApiVersion(owner.getApiVersion())
- .withKind(owner.getKind())
- .withUid(owner.getMetadata().getUid())
- .withBlockOwnerDeletion(true)
- .build();
- }
-
- public static <T extends HasMetadata> String asJsonString(T resource) {
- try {
- return objectMapper.writeValueAsString(resource);
- } catch (JsonProcessingException e) {
- throw new IllegalStateException(e);
- }
- }
-
- public static boolean overrideDriverTemplateEnabled(ApplicationSpec applicationSpec) {
- return applicationSpec != null
- && applicationSpec.getDriverSpec() != null
- && applicationSpec.getDriverSpec().getPodTemplateSpec() != null;
- }
-
- public static boolean overrideExecutorTemplateEnabled(ApplicationSpec applicationSpec) {
- return applicationSpec != null
- && applicationSpec.getExecutorSpec() != null
- && applicationSpec.getExecutorSpec().getPodTemplateSpec() != null;
- }
-
- public static long getAttemptId(final SparkApplication app) {
- long attemptId = 0L;
- if (app.getStatus() != null && app.getStatus().getCurrentAttemptSummary() != null) {
- attemptId = app.getStatus().getCurrentAttemptSummary().getAttemptInfo().getId();
- }
- return attemptId;
- }
-}
diff --git a/spark-operator-api/src/main/resources/printer-columns.sh b/spark-operator-api/src/main/resources/printer-columns.sh
deleted file mode 100755
index f49d2b2..0000000
--- a/spark-operator-api/src/main/resources/printer-columns.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This is a workaround. See https://github.com/fabric8io/kubernetes-client/issues/3069
-# We do a yq to add printer columns
-
-SCRIPT_PATH=$(cd "$(dirname "$0")"; pwd)
-for f in $(ls ${SCRIPT_PATH}/../../../build/classes/java/main/META-INF/fabric8/*.spark.apache.org-v1.yml); do
- yq -i '.spec.versions[0] += ({"additionalPrinterColumns": [{"jsonPath": ".status.currentState.currentStateSummary", "name": "Current State", "type": "string"}, {"jsonPath": ".metadata.creationTimestamp", "name": "Age", "type": "date"}]})' $f
-done
-
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ApplicationSpecTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ApplicationSpecTest.java
deleted file mode 100644
index 31693c9..0000000
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ApplicationSpecTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import static org.apache.spark.k8s.operator.spec.DeploymentMode.ClusterMode;
-import static org.junit.jupiter.api.Assertions.*;
-
-import org.junit.jupiter.api.Test;
-
-class ApplicationSpecTest {
- @Test
- void testBuilder() {
- ApplicationSpec spec1 = new ApplicationSpec();
- ApplicationSpec spec2 = new ApplicationSpec.ApplicationSpecBuilder().build();
- assertEquals(spec1, spec2);
- }
-
- @Test
- void testInitSpecWithDefaults() {
- ApplicationSpec spec = new ApplicationSpec();
- assertEquals(ClusterMode, spec.getDeploymentMode());
- assertNull(spec.getDriverSpec());
- assertNull(spec.getExecutorSpec());
- ApplicationTolerations tolerations = spec.getApplicationTolerations();
- assertNotNull(tolerations);
- assertEquals(RestartPolicy.Never, tolerations.getRestartConfig().getRestartPolicy());
- assertEquals(ResourceRetainPolicy.Never, tolerations.getResourceRetainPolicy());
- assertEquals(0, tolerations.instanceConfig.initExecutors);
- assertEquals(0, tolerations.instanceConfig.minExecutors);
- assertEquals(0, tolerations.instanceConfig.maxExecutors);
- }
-}
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ClusterSpecTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ClusterSpecTest.java
deleted file mode 100644
index b96455a..0000000
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ClusterSpecTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import org.junit.jupiter.api.Test;
-
-class ClusterSpecTest {
- @Test
- void testBuilder() {
- ClusterSpec spec1 = new ClusterSpec();
- ClusterSpec spec2 = new ClusterSpec.ClusterSpecBuilder().build();
- assertEquals(spec1, spec2);
- }
-
- @Test
- void testInitSpecWithDefaults() {
- ClusterSpec spec1 = new ClusterSpec();
- assertEquals(null, spec1.runtimeVersions.jdkVersion);
- assertEquals(null, spec1.runtimeVersions.scalaVersion);
- assertEquals(null, spec1.runtimeVersions.sparkVersion);
- assertEquals(0, spec1.clusterTolerations.instanceConfig.initWorkers);
- assertEquals(0, spec1.clusterTolerations.instanceConfig.minWorkers);
- assertEquals(0, spec1.clusterTolerations.instanceConfig.maxWorkers);
- }
-}
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/RestartPolicyTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/RestartPolicyTest.java
deleted file mode 100644
index 104796d..0000000
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/RestartPolicyTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.spec;
-
-import static org.apache.spark.k8s.operator.spec.RestartPolicy.Always;
-import static org.apache.spark.k8s.operator.spec.RestartPolicy.Never;
-import static org.apache.spark.k8s.operator.spec.RestartPolicy.OnFailure;
-import static org.apache.spark.k8s.operator.spec.RestartPolicy.OnInfrastructureFailure;
-import static org.apache.spark.k8s.operator.spec.RestartPolicy.attemptRestartOnState;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.DriverEvicted;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.DriverReadyTimedOut;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.DriverStartTimedOut;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.ExecutorsStartTimedOut;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.Failed;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.SchedulingFailure;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.Succeeded;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.spark.k8s.operator.status.ApplicationStateSummary;
-
-class RestartPolicyTest {
-
- @Test
- void testAlways() {
- for (ApplicationStateSummary stateSummary : ApplicationStateSummary.values()) {
- assertTrue(attemptRestartOnState(Always, stateSummary));
- }
- }
-
- @Test
- void testNever() {
- for (ApplicationStateSummary stateSummary : ApplicationStateSummary.values()) {
- assertFalse(attemptRestartOnState(Never, stateSummary));
- }
- }
-
- @Test
- void testOnFailure() {
- for (ApplicationStateSummary stateSummary : ApplicationStateSummary.values()) {
- if (!stateSummary.isStopping()) {
- assertFalse(attemptRestartOnState(OnFailure, stateSummary));
- }
- }
- assertFalse(attemptRestartOnState(OnFailure, Succeeded));
- assertTrue(attemptRestartOnState(OnFailure, Failed));
- assertTrue(attemptRestartOnState(OnFailure, DriverStartTimedOut));
- assertTrue(attemptRestartOnState(OnFailure, DriverReadyTimedOut));
- assertTrue(attemptRestartOnState(OnFailure, DriverEvicted));
- assertTrue(attemptRestartOnState(OnFailure, ExecutorsStartTimedOut));
- }
-
- @Test
- void testOnInfrastructureFailure() {
- assertTrue(attemptRestartOnState(OnInfrastructureFailure, DriverStartTimedOut));
- assertTrue(attemptRestartOnState(OnInfrastructureFailure, ExecutorsStartTimedOut));
- assertTrue(attemptRestartOnState(OnInfrastructureFailure, SchedulingFailure));
-
- for (ApplicationStateSummary stateSummary : ApplicationStateSummary.values()) {
- if (!stateSummary.isStopping()) {
- assertFalse(attemptRestartOnState(OnInfrastructureFailure, stateSummary));
- }
- }
- assertFalse(attemptRestartOnState(OnInfrastructureFailure, Succeeded));
- assertFalse(attemptRestartOnState(OnInfrastructureFailure, Failed));
- assertFalse(attemptRestartOnState(OnInfrastructureFailure, DriverReadyTimedOut));
- assertFalse(attemptRestartOnState(OnInfrastructureFailure, DriverEvicted));
- }
-}
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/status/ApplicationStatusTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/status/ApplicationStatusTest.java
deleted file mode 100644
index bcc0011..0000000
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/status/ApplicationStatusTest.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.Submitted;
-import static org.apache.spark.k8s.operator.status.ApplicationStateSummary.Succeeded;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.spark.k8s.operator.spec.ResourceRetainPolicy;
-import org.apache.spark.k8s.operator.spec.RestartConfig;
-import org.apache.spark.k8s.operator.spec.RestartPolicy;
-
-class ApplicationStatusTest {
-
- @Test
- void testInitStatus() {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- assertEquals(Submitted, applicationStatus.currentState.currentStateSummary);
- assertEquals(1, applicationStatus.getStateTransitionHistory().size());
- assertEquals(
- applicationStatus.currentState, applicationStatus.getStateTransitionHistory().get(0L));
- }
-
- @Test
- void testAppendNewState() {
- ApplicationStatus applicationStatus = new ApplicationStatus();
- ApplicationState newState = new ApplicationState(ApplicationStateSummary.RunningHealthy, "foo");
- ApplicationStatus newStatus = applicationStatus.appendNewState(newState);
- assertEquals(2, newStatus.getStateTransitionHistory().size());
- assertEquals(newState, newStatus.getStateTransitionHistory().get(1L));
- }
-
- @Test
- void testTerminateOrRestartWithoutRetry() {
- RestartConfig noRetryConfig = new RestartConfig();
- noRetryConfig.setRestartPolicy(RestartPolicy.Never);
- String messageOverride = "foo";
-
- // without retry
- ApplicationStatus status =
- new ApplicationStatus().appendNewState(new ApplicationState(Succeeded, "bar"));
- ApplicationStatus updatedStatusReleaseResource =
- status.terminateOrRestart(
- noRetryConfig, ResourceRetainPolicy.Never, messageOverride, false);
- assertEquals(
- ApplicationStateSummary.ResourceReleased,
- updatedStatusReleaseResource.getCurrentState().getCurrentStateSummary());
- assertTrue(
- updatedStatusReleaseResource.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(
- 0L, updatedStatusReleaseResource.getCurrentAttemptSummary().getAttemptInfo().getId());
-
- ApplicationStatus updatedStatusRetainResource =
- status.terminateOrRestart(
- noRetryConfig, ResourceRetainPolicy.Always, messageOverride, false);
- assertEquals(
- ApplicationStateSummary.TerminatedWithoutReleaseResources,
- updatedStatusRetainResource.getCurrentState().getCurrentStateSummary());
- assertTrue(
- updatedStatusRetainResource.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(
- 0L, updatedStatusRetainResource.getCurrentAttemptSummary().getAttemptInfo().getId());
- }
-
- @Test
- void testTerminateOrRestartWithRetry() {
- RestartConfig alwaysRetryConfig = new RestartConfig();
- alwaysRetryConfig.setRestartPolicy(RestartPolicy.Always);
- alwaysRetryConfig.setMaxRestartAttempts(1L);
- String messageOverride = "foo";
-
- ApplicationStatus status =
- new ApplicationStatus().appendNewState(new ApplicationState(Succeeded, "bar"));
-
- // retry policy set
- ApplicationStatus restartReleaseResource =
- status.terminateOrRestart(
- alwaysRetryConfig, ResourceRetainPolicy.Never, messageOverride, false);
- assertEquals(
- ApplicationStateSummary.ScheduledToRestart,
- restartReleaseResource.getCurrentState().getCurrentStateSummary());
- assertTrue(restartReleaseResource.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(1L, restartReleaseResource.getCurrentAttemptSummary().getAttemptInfo().getId());
-
- ApplicationStatus restartRetainResource =
- status.terminateOrRestart(
- alwaysRetryConfig, ResourceRetainPolicy.Always, messageOverride, false);
- assertEquals(
- ApplicationStateSummary.ScheduledToRestart,
- restartRetainResource.getCurrentState().getCurrentStateSummary());
- assertTrue(restartRetainResource.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(1L, restartRetainResource.getCurrentAttemptSummary().getAttemptInfo().getId());
-
- // trim state history for new restart
- ApplicationStatus restartTrimHistory =
- status.terminateOrRestart(
- alwaysRetryConfig, ResourceRetainPolicy.Never, messageOverride, true);
- assertEquals(
- ApplicationStateSummary.ScheduledToRestart,
- restartTrimHistory.getCurrentState().getCurrentStateSummary());
- assertTrue(restartTrimHistory.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(1L, restartTrimHistory.getCurrentAttemptSummary().getAttemptInfo().getId());
- assertEquals(1L, restartTrimHistory.getStateTransitionHistory().size());
- assertNotNull(restartTrimHistory.getPreviousAttemptSummary().getStateTransitionHistory());
- assertEquals(
- status.getStateTransitionHistory(),
- restartTrimHistory.getPreviousAttemptSummary().getStateTransitionHistory());
-
- ApplicationStatus restartRetainResourceTrimHistory =
- status.terminateOrRestart(
- alwaysRetryConfig, ResourceRetainPolicy.Always, messageOverride, true);
- assertEquals(
- ApplicationStateSummary.ScheduledToRestart,
- restartRetainResourceTrimHistory.getCurrentState().getCurrentStateSummary());
- assertTrue(
- restartRetainResourceTrimHistory.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(
- 1L, restartRetainResourceTrimHistory.getCurrentAttemptSummary().getAttemptInfo().getId());
- assertNotNull(
- restartRetainResourceTrimHistory.getPreviousAttemptSummary().getStateTransitionHistory());
- assertEquals(
- status.getStateTransitionHistory(),
- restartRetainResourceTrimHistory.getPreviousAttemptSummary().getStateTransitionHistory());
-
- // retry policy set but max retry attempt reached
- alwaysRetryConfig.setMaxRestartAttempts(1L);
- ApplicationStatus restartFailed =
- restartReleaseResource.appendNewState(
- new ApplicationState(ApplicationStateSummary.Failed, "bar"));
- ApplicationStatus maxRestartExceededReleaseResource =
- restartFailed.terminateOrRestart(
- alwaysRetryConfig, ResourceRetainPolicy.Never, messageOverride, false);
- assertEquals(
- ApplicationStateSummary.ResourceReleased,
- maxRestartExceededReleaseResource.getCurrentState().getCurrentStateSummary());
- assertTrue(
- maxRestartExceededReleaseResource.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(
- 1L, maxRestartExceededReleaseResource.getCurrentAttemptSummary().getAttemptInfo().getId());
-
- ApplicationStatus maxRestartExceededRetainResource =
- restartFailed.terminateOrRestart(
- alwaysRetryConfig, ResourceRetainPolicy.Always, messageOverride, false);
- assertEquals(
- ApplicationStateSummary.TerminatedWithoutReleaseResources,
- maxRestartExceededRetainResource.getCurrentState().getCurrentStateSummary());
- assertTrue(
- maxRestartExceededRetainResource.getCurrentState().getMessage().contains(messageOverride));
- assertEquals(
- 1L, maxRestartExceededRetainResource.getCurrentAttemptSummary().getAttemptInfo().getId());
- }
-}
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/status/ClusterStatusTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/status/ClusterStatusTest.java
deleted file mode 100644
index 6e18b3e..0000000
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/status/ClusterStatusTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.status;
-
-import static org.apache.spark.k8s.operator.status.ClusterStateSummary.Submitted;
-import static org.junit.jupiter.api.Assertions.*;
-
-import org.junit.jupiter.api.Test;
-
-class ClusterStatusTest {
- @Test
- void testInitStatus() {
- ClusterStatus status = new ClusterStatus();
- assertEquals(Submitted, status.currentState.currentStateSummary);
- assertEquals(1, status.getStateTransitionHistory().size());
- assertEquals(status.currentState, status.getStateTransitionHistory().get(0L));
- }
-
- @Test
- void testAppendNewState() {
- ClusterStatus status = new ClusterStatus();
- ClusterState newState = new ClusterState(ClusterStateSummary.RunningHealthy, "foo");
- ClusterStatus newStatus = status.appendNewState(newState);
- assertEquals(2, newStatus.getStateTransitionHistory().size());
- assertEquals(newState, newStatus.getStateTransitionHistory().get(1L));
- }
-}
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/utils/ModelUtilsTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/utils/ModelUtilsTest.java
deleted file mode 100644
index 87170be..0000000
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/utils/ModelUtilsTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.utils;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import io.fabric8.kubernetes.api.model.ContainerStatus;
-import io.fabric8.kubernetes.api.model.ContainerStatusBuilder;
-import io.fabric8.kubernetes.api.model.Pod;
-import io.fabric8.kubernetes.api.model.PodTemplateSpec;
-import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import org.apache.spark.k8s.operator.spec.ApplicationSpec;
-import org.apache.spark.k8s.operator.spec.BaseApplicationTemplateSpec;
-
-class ModelUtilsTest {
-
- private PodTemplateSpec buildSamplePodTemplateSpec() {
- return new PodTemplateSpecBuilder()
- .withNewMetadata()
- .withName("foo-pod-name")
- .withNamespace("foo-namespace-name")
- .withLabels(Map.of("foo", "bar"))
- .withAnnotations(Map.of("foo", "bar"))
- .endMetadata()
- .withNewSpec()
- .addNewInitContainer()
- .withName("foo-init-container")
- .withImage("foo-image")
- .endInitContainer()
- .addNewContainer()
- .withName("foo-container-1")
- .withImage("foo-image")
- .endContainer()
- .addNewContainer()
- .withName("foo-container-2")
- .withImage("bar-image")
- .endContainer()
- .endSpec()
- .addToAdditionalProperties("foo", "bar")
- .build();
- }
-
- @Test
- void testGetPodFromTemplateSpec() {
- PodTemplateSpec podTemplateSpec = buildSamplePodTemplateSpec();
- Pod pod = ModelUtils.getPodFromTemplateSpec(podTemplateSpec);
- Assertions.assertEquals(podTemplateSpec.getMetadata(), pod.getMetadata());
- Assertions.assertEquals(podTemplateSpec.getSpec(), pod.getSpec());
- Assertions.assertEquals(
- podTemplateSpec.getAdditionalProperties(), pod.getAdditionalProperties());
- }
-
- @Test
- void testFindDriverMainContainerStatus() {
- ContainerStatus containerStatus1 =
- new ContainerStatusBuilder().withName("foo-container-1").build();
- ContainerStatus containerStatus2 =
- new ContainerStatusBuilder().withName("foo-container-2").build();
- ApplicationSpec specWithoutSparkContainerNameProp = new ApplicationSpec();
- List<ContainerStatus> sparkContainerStatusList =
- ModelUtils.findDriverMainContainerStatus(
- specWithoutSparkContainerNameProp, List.of(containerStatus1, containerStatus2));
- Assertions.assertEquals(List.of(containerStatus1, containerStatus2), sparkContainerStatusList);
-
- ApplicationSpec specWithSparkContainerNameProp = new ApplicationSpec();
- Map<String, String> sparkProps = new HashMap<>();
- sparkProps.put("spark.kubernetes.driver.podTemplateContainerName", "foo-container-2");
- specWithSparkContainerNameProp.setSparkConf(sparkProps);
- sparkContainerStatusList =
- ModelUtils.findDriverMainContainerStatus(
- specWithSparkContainerNameProp, List.of(containerStatus1, containerStatus2));
- Assertions.assertEquals(List.of(containerStatus2), sparkContainerStatusList);
- }
-
- @Test
- void testOverrideDriverTemplateEnabled() {
- ApplicationSpec applicationSpec = new ApplicationSpec();
- assertFalse(ModelUtils.overrideDriverTemplateEnabled(applicationSpec));
-
- BaseApplicationTemplateSpec driverSpec = new BaseApplicationTemplateSpec();
- applicationSpec.setDriverSpec(driverSpec);
- assertFalse(ModelUtils.overrideDriverTemplateEnabled(applicationSpec));
-
- driverSpec.setPodTemplateSpec(buildSamplePodTemplateSpec());
- applicationSpec.setDriverSpec(driverSpec);
- assertTrue(ModelUtils.overrideDriverTemplateEnabled(applicationSpec));
- }
-
- @Test
- void testOverrideExecutorTemplateEnabled() {
- ApplicationSpec applicationSpec = new ApplicationSpec();
- assertFalse(ModelUtils.overrideDriverTemplateEnabled(applicationSpec));
-
- BaseApplicationTemplateSpec executorSpec = new BaseApplicationTemplateSpec();
- applicationSpec.setExecutorSpec(executorSpec);
- assertFalse(ModelUtils.overrideDriverTemplateEnabled(applicationSpec));
-
- executorSpec.setPodTemplateSpec(buildSamplePodTemplateSpec());
- applicationSpec.setDriverSpec(executorSpec);
- assertTrue(ModelUtils.overrideDriverTemplateEnabled(applicationSpec));
- }
-}
diff --git a/spark-operator/build.gradle b/spark-operator/build.gradle
deleted file mode 100644
index 1d41733..0000000
--- a/spark-operator/build.gradle
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-apply plugin: 'com.gradleup.shadow'
-
-dependencies {
- implementation project(":spark-operator-api")
- implementation project(":spark-submission-worker")
-
- implementation(libs.operator.framework) {
- exclude group: 'com.squareup.okio'
- exclude group: 'io.fabric8'
- }
- implementation(libs.kubernetes.httpclient.okhttp) {
- exclude group: 'com.squareup.okhttp3'
- }
- implementation(libs.okhttp)
- implementation(libs.logging.interceptor)
- implementation(libs.spotbugs.annotations)
- // logging
- implementation(libs.log4j.api)
- implementation(libs.log4j.core)
- implementation(libs.log4j.slf4j.impl)
- implementation(libs.log4j.api12)
- implementation(libs.log4j.layout.template.json)
-
- // metrics
- implementation(libs.metrics.core)
- implementation(libs.metrics.jvm)
- compileOnly(libs.spark.core) {
- exclude group: "com.github.luben"
- exclude group: "io.netty", module: "netty-tcnative-boringssl-static"
- exclude group: "io.netty", module: "netty-tcnative-classes"
- exclude group: "org.apache.logging.log4j"
- exclude group: "org.fusesource.leveldbjni"
- exclude group: "org.lz4"
- exclude group: "org.rocksdb"
- exclude group: "org.slf4j"
- exclude group: "org.xerial.snappy"
- exclude group: 'com.squareup.okhttp3'
- exclude group: 'com.squareup.okio'
- }
- compileOnly(libs.lombok)
-
- annotationProcessor(libs.lombok)
- annotationProcessor(libs.spotbugs.annotations)
-
- testImplementation(libs.operator.framework.junit5) {
- exclude group: 'com.squareup.okio'
- exclude group: 'io.fabric8'
- }
- testImplementation(libs.kubernetes.server.mock) {
- exclude group: 'junit'
- exclude group: 'com.squareup.okhttp3'
- }
- testImplementation(libs.spark.core) {
- exclude group: 'com.squareup.okio'
- exclude group: 'com.squareup.okhttp3'
- exclude group: "org.apache.logging.log4j"
- exclude group: "org.slf4j"
- }
- testImplementation(libs.mockwebserver)
- testImplementation platform(libs.junit.bom)
- testImplementation(libs.junit.jupiter)
- testImplementation(libs.powermock.core)
- testRuntimeOnly(libs.junit.platform.launcher)
-
- testImplementation(libs.mockito.core)
-}
-
-test {
- useJUnitPlatform()
-}
-
-jar.dependsOn shadowJar
-
-jar {
- zip64 = true
- archiveVersion.set('')
-}
-
-
-shadowJar {
- zip64 = true
- mergeServiceFiles()
- transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer)
-}
-
-base {
- description = "Spark Kubernetes Operator"
- def artifact = "spark-kubernetes-operator"
- archivesName = artifact
-}
diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java
deleted file mode 100644
index 00619d7..0000000
--- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator;
-
-import static org.apache.spark.k8s.operator.utils.Utils.getAppStatusListener;
-import static org.apache.spark.k8s.operator.utils.Utils.getClusterStatusListener;
-import static org.apache.spark.k8s.operator.utils.Utils.getWatchedNamespaces;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.javaoperatorsdk.operator.Operator;
-import io.javaoperatorsdk.operator.RegisteredController;
-import io.javaoperatorsdk.operator.api.config.ConfigurationServiceOverrider;
-import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider;
-import lombok.extern.slf4j.Slf4j;
-import okhttp3.Interceptor;
-
-import org.apache.spark.k8s.operator.client.KubernetesClientFactory;
-import org.apache.spark.k8s.operator.config.SparkOperatorConf;
-import org.apache.spark.k8s.operator.config.SparkOperatorConfigMapReconciler;
-import org.apache.spark.k8s.operator.metrics.MetricsService;
-import org.apache.spark.k8s.operator.metrics.MetricsSystem;
-import org.apache.spark.k8s.operator.metrics.MetricsSystemFactory;
-import org.apache.spark.k8s.operator.metrics.healthcheck.SentinelManager;
-import org.apache.spark.k8s.operator.metrics.source.KubernetesMetricsInterceptor;
-import org.apache.spark.k8s.operator.metrics.source.OperatorJosdkMetrics;
-import org.apache.spark.k8s.operator.probe.ProbeService;
-import org.apache.spark.k8s.operator.reconciler.SparkAppReconciler;
-import org.apache.spark.k8s.operator.reconciler.SparkClusterReconciler;
-import org.apache.spark.k8s.operator.utils.SparkAppStatusRecorder;
-import org.apache.spark.k8s.operator.utils.SparkClusterStatusRecorder;
-
-/**
- * Entry point for Spark Operator. Bootstrap the operator app by starting watch and reconciler for
- * SparkApps, starting watch for hot property loading, if enabled, and starting metrics server with
- * sentinel monitor if enabled
- */
-@Slf4j
-public class SparkOperator {
- private final List<Operator> registeredOperators;
- private final KubernetesClient client;
- private final SparkAppSubmissionWorker appSubmissionWorker;
- private final SparkClusterSubmissionWorker clusterSubmissionWorker;
- private final SparkAppStatusRecorder sparkAppStatusRecorder;
- private final SparkClusterStatusRecorder sparkClusterStatusRecorder;
- protected Set<RegisteredController<?>> registeredSparkControllers;
- protected Set<String> watchedNamespaces;
- private final MetricsSystem metricsSystem;
- private final SentinelManager<SparkApplication> sparkApplicationSentinelManager;
- private final SentinelManager<SparkCluster> sparkClusterSentinelManager;
- private final ProbeService probeService;
- private final MetricsService metricsService;
- private final ExecutorService metricsResourcesSingleThreadPool;
-
- public SparkOperator() {
- this.metricsSystem = MetricsSystemFactory.createMetricsSystem();
- this.client =
- KubernetesClientFactory.buildKubernetesClient(getClientInterceptors(metricsSystem));
- this.appSubmissionWorker = new SparkAppSubmissionWorker();
- this.clusterSubmissionWorker = new SparkClusterSubmissionWorker();
- this.sparkAppStatusRecorder = new SparkAppStatusRecorder(getAppStatusListener());
- this.sparkClusterStatusRecorder = new SparkClusterStatusRecorder(getClusterStatusListener());
- this.registeredSparkControllers = new HashSet<>();
- this.watchedNamespaces = getWatchedNamespaces();
- this.sparkApplicationSentinelManager = new SentinelManager<>();
- this.sparkClusterSentinelManager = new SentinelManager<>();
- this.registeredOperators = new ArrayList<>();
- this.registeredOperators.add(registerSparkOperator());
- if (SparkOperatorConf.DYNAMIC_CONFIG_ENABLED.getValue()) {
- this.registeredOperators.add(registerSparkOperatorConfMonitor());
- }
- this.metricsResourcesSingleThreadPool = Executors.newSingleThreadExecutor();
- this.probeService =
- new ProbeService(
- registeredOperators,
- Arrays.asList(sparkApplicationSentinelManager, sparkClusterSentinelManager),
- null);
- this.metricsService = new MetricsService(metricsSystem, metricsResourcesSingleThreadPool);
- }
-
- protected Operator registerSparkOperator() {
- Operator op = new Operator(this::overrideOperatorConfigs);
- registeredSparkControllers.add(
- op.register(
- new SparkAppReconciler(
- appSubmissionWorker, sparkAppStatusRecorder, sparkApplicationSentinelManager),
- this::overrideControllerConfigs));
- registeredSparkControllers.add(
- op.register(
- new SparkClusterReconciler(
- clusterSubmissionWorker, sparkClusterStatusRecorder, sparkClusterSentinelManager),
- this::overrideControllerConfigs));
- return op;
- }
-
- protected Operator registerSparkOperatorConfMonitor() {
- Operator op = new Operator(this::overrideConfigMonitorConfigs);
- String operatorNamespace = SparkOperatorConf.OPERATOR_NAMESPACE.getValue();
- String confSelector = SparkOperatorConf.DYNAMIC_CONFIG_SELECTOR.getValue();
- log.info(
- "Starting conf monitor in namespace: {}, with selector: {}",
- operatorNamespace,
- confSelector);
- op.register(
- new SparkOperatorConfigMapReconciler(
- this::updateWatchingNamespaces,
- SparkOperatorConf.OPERATOR_NAMESPACE.getValue(),
- unused -> getWatchedNamespaces()),
- c -> {
- c.withRateLimiter(SparkOperatorConf.getOperatorRateLimiter());
- c.settingNamespaces(operatorNamespace);
- c.withLabelSelector(confSelector);
- });
- return op;
- }
-
- protected boolean updateWatchingNamespaces(Set<String> namespaces) {
- if (watchedNamespaces.equals(namespaces)) {
- log.info("No watched namespace change detected");
- return false;
- }
- if (watchedNamespaces.isEmpty()) {
- log.info("Cannot update watch namespaces for operator started at cluster level.");
- return false;
- }
- if (namespaces == null || namespaces.isEmpty()) {
- log.error("Cannot updating namespaces to empty");
- return false;
- }
- registeredSparkControllers.forEach(
- c -> {
- if (c.allowsNamespaceChanges()) {
- log.info("Updating operator namespaces to {}", namespaces);
- c.changeNamespaces(namespaces);
- } else {
- log.error("Controller does not allow namespace change, skipping namespace change.");
- }
- });
- this.watchedNamespaces = new HashSet<>(namespaces);
- return true;
- }
-
- protected void overrideOperatorConfigs(ConfigurationServiceOverrider overrider) {
- overrider.withKubernetesClient(client);
- overrider.withStopOnInformerErrorDuringStartup(
- SparkOperatorConf.TERMINATE_ON_INFORMER_FAILURE_ENABLED.getValue());
- overrider.withTerminationTimeoutSeconds(
- SparkOperatorConf.RECONCILER_TERMINATION_TIMEOUT_SECONDS.getValue());
- int parallelism = SparkOperatorConf.RECONCILER_PARALLELISM.getValue();
- if (parallelism > 0) {
- log.info("Configuring operator with {} reconciliation threads.", parallelism);
- overrider.withConcurrentReconciliationThreads(parallelism);
- } else {
- log.info("Configuring operator with unbounded reconciliation thread pool.");
- overrider.withExecutorService(Executors.newCachedThreadPool());
- }
- if (SparkOperatorConf.LEADER_ELECTION_ENABLED.getValue()) {
- overrider.withLeaderElectionConfiguration(SparkOperatorConf.getLeaderElectionConfig());
- }
- if (SparkOperatorConf.JOSDK_METRICS_ENABLED.getValue()) {
- log.info("Adding Operator JosdkMetrics to metrics system.");
- OperatorJosdkMetrics operatorJosdkMetrics = new OperatorJosdkMetrics();
- overrider.withMetrics(operatorJosdkMetrics);
- metricsSystem.registerSource(operatorJosdkMetrics);
- }
- }
-
- protected void overrideConfigMonitorConfigs(ConfigurationServiceOverrider overrider) {
- overrider.withKubernetesClient(client);
- overrider.withConcurrentReconciliationThreads(
- SparkOperatorConf.DYNAMIC_CONFIG_RECONCILER_PARALLELISM.getValue());
- overrider.withStopOnInformerErrorDuringStartup(true);
- overrider.withCloseClientOnStop(false);
- overrider.withInformerStoppedHandler(
- (informer, ex) ->
- log.error("Dynamic config informer stopped: operator will not accept config updates."));
- }
-
- protected void overrideControllerConfigs(ControllerConfigurationOverrider<?> overrider) {
- if (watchedNamespaces.isEmpty()) {
- log.info("Initializing operator watching at cluster level.");
- } else {
- log.info("Initializing with watched namespaces {}", watchedNamespaces);
- }
- overrider.settingNamespaces(watchedNamespaces);
- overrider.withRateLimiter(SparkOperatorConf.getOperatorRateLimiter());
- overrider.withRetry(SparkOperatorConf.getOperatorRetry());
- }
-
- protected List<Interceptor> getClientInterceptors(MetricsSystem metricsSystem) {
- List<Interceptor> clientInterceptors = new ArrayList<>();
- if (SparkOperatorConf.KUBERNETES_CLIENT_METRICS_ENABLED.getValue()) {
- KubernetesMetricsInterceptor metricsInterceptor = new KubernetesMetricsInterceptor();
- clientInterceptors.add(metricsInterceptor);
- metricsSystem.registerSource(metricsInterceptor);
- }
- return clientInterceptors;
- }
-
- /**
- * Bootstrap Spark Operator
- *
- * @param args not used - operator behavior are configured from SparkOperatorConf
- */
- public static void main(String[] args) {
- SparkOperator sparkOperator = new SparkOperator();
- for (Operator operator : sparkOperator.registeredOperators) {
- operator.start();
- }
- sparkOperator.probeService.start();
- // Single thread queue to ensure MetricsService starts after the MetricsSystem
- sparkOperator.metricsResourcesSingleThreadPool.submit(sparkOperator.metricsSystem::start);
- sparkOperator.metricsResourcesSingleThreadPool.submit(sparkOperator.metricsService::start);
- }
-}
diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/client/KubernetesClientFactory.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/client/KubernetesClientFactory.java
deleted file mode 100644
index 766c1c5..0000000
--- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/client/KubernetesClientFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.spark.k8s.operator.client;
-
-import java.util.List;
-
-import io.fabric8.kubernetes.client.Config;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.KubernetesClientBuilder;
-import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory;
-import okhttp3.Interceptor;
-import okhttp3.OkHttpClient;
-
-/** Build Kubernetes Client with metrics configured */
-public final class KubernetesClientFactory {
-
- private KubernetesClientFactory() {}
-
- public static KubernetesClient buildKubernetesClient(final List<Interceptor> interceptors) {
- return buildKubernetesClient(interceptors, null);
- }
-
- public static KubernetesClient buildKubernetesClient(
- final List<Interceptor> interceptors, final Config kubernetesClientConfig) {
- return new KubernetesClientBuilder()
- .withConfig(kubernetesClientConfig)
- .withHttpClientFactory(
- new OkHttpClientFactory() {
- @Override
- protected void additionalConfig(OkHttpClient.Builder builder) {
- for (Interceptor interceptor : interceptors) {
- builder.addInterceptor(interceptor);
- }
- }
- })
- .build();
- }
-}
diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java
deleted file mode 100644
index e0726c4..0000000
--- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.config;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.RequiredArgsConstructor;
-import lombok.ToString;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.spark.k8s.operator.utils.ModelUtils;
-
-/** Config options for Spark Operator. Supports primitive and serialized JSON */
-@RequiredArgsConstructor
-@AllArgsConstructor
-@EqualsAndHashCode
-@ToString
-@Builder
-@Slf4j
-public class ConfigOption<T> {
- @Getter @Builder.Default private final boolean enableDynamicOverride = true;
- @Getter private String key;
- @Getter private String description;
- @Getter private T defaultValue;
- @Getter private Class<T> typeParameterClass;
-
- public T getValue() {
- T resolvedValue = resolveValue();
- if (log.isDebugEnabled()) {
- log.debug("Resolved value for property {}={}", key, resolvedValue);
- }
- return resolveValue();
- }
-
- private T resolveValue() {
- try {
- String value = SparkOperatorConfManager.INSTANCE.getValue(key);
- if (!enableDynamicOverride) {
- value = SparkOperatorConfManager.INSTANCE.getInitialValue(key);
- }
- if (StringUtils.isNotEmpty(value)) {
- if (typeParameterClass.isPrimitive() || typeParameterClass == String.class) {
- return (T) resolveValueToPrimitiveType(typeParameterClass, value);
- } else {
- return ModelUtils.objectMapper.readValue(value, typeParameterClass);
- }
- } else {
- return defaultValue;
- }
- } catch (NumberFormatException | JsonProcessingException t) {
- log.error(
- "Failed to resolve value for config key {}, using default value {}",
- key,
- defaultValue,
- t);
- return defaultValue;
- }
- }
-
- public static Object resolveValueToPrimitiveType(Class<?> clazz, String value) {
- if (Boolean.class == clazz || Boolean.TYPE == clazz) {
- return Boolean.parseBoolean(value);
- }
- if (Byte.class == clazz || Byte.TYPE == clazz) {
- return Byte.parseByte(value);
- }
- if (Short.class == clazz || Short.TYPE == clazz) {
- return Short.parseShort(value);
- }
- if (Integer.class == clazz || Integer.TYPE == clazz) {
- return Integer.parseInt(value);
- }
- if (Long.class == clazz || Long.TYPE == clazz) {
- return Long.parseLong(value);
- }
- if (Float.class == clazz || Float.TYPE == clazz) {
- return Float.parseFloat(value);
- }
- if (Double.class == clazz || Double.TYPE == clazz) {
- return Double.parseDouble(value);
- }
- return value;
- }
-}
diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/SparkOperatorConf.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/SparkOperatorConf.java
deleted file mode 100644
index 3aca8cd..0000000
--- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/SparkOperatorConf.java
+++ /dev/null
@@ -1,479 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.k8s.operator.config;
-
-import java.time.Duration;
-
-import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration;
-import io.javaoperatorsdk.operator.processing.event.rate.LinearRateLimiter;
-import io.javaoperatorsdk.operator.processing.event.rate.RateLimiter;
-import io.javaoperatorsdk.operator.processing.retry.GenericRetry;
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.spark.k8s.operator.utils.Utils;
-
-/** Spark Operator Configuration options. */
-@Slf4j
-public final class SparkOperatorConf {
-
- public static final ConfigOption<String> OPERATOR_APP_NAME =
- ConfigOption.<String>builder()
- .key("spark.kubernetes.operator.name")
- .enableDynamicOverride(false)
- .description("Name of the operator.")
- .typeParameterClass(String.class)
- .defaultValue("spark-kubernetes-operator")
- .build();
-
- public static final ConfigOption<String> OPERATOR_NAMESPACE =
- ConfigOption.<String>builder()
- .key("spark.kubernetes.operator.namespace")
- .enableDynamicOverride(false)
- .description("Namespace that operator is deployed within.")
- .typeParameterClass(String.class)
- .defaultValue("default")
- .build();
-
- public static final ConfigOption<String> OPERATOR_WATCHED_NAMESPACES =
- ConfigOption.<String>builder()
- .key("spark.kubernetes.operator.watchedNamespaces")
- .enableDynamicOverride(true)
- .description(
- "Comma-separated list of namespaces that the operator would be watching for "
- + "Spark resources. If set to '*', operator would watch all namespaces.")
- .typeParameterClass(String.class)
- .defaultValue("default")
- .build();
-
- public static final ConfigOption<Boolean> TERMINATE_ON_INFORMER_FAILURE_ENABLED =
- ConfigOption.<Boolean>builder()
- .key("spark.kubernetes.operator.terminateOnInformerFailureEnabled")
- .enableDynamicOverride(false)
- .description(
- "Enable to indicate informer errors should stop operator startup. If "
- + "disabled, operator startup will ignore recoverable errors, "
- + "caused for example by RBAC issues and will retry "
- + "periodically.")
- .typeParameterClass(Boolean.class)
- .defaultValue(false)
- .build();
-
- public static final ConfigOption<Integer> RECONCILER_TERMINATION_TIMEOUT_SECONDS =
- ConfigOption.<Integer>builder()
- .key("spark.kubernetes.operator.reconciler.terminationTimeoutSeconds")
- .enableDynamicOverride(false)
- .description(
- "Grace period for operator shutdown before reconciliation threads are killed.")
- .typeParameterClass(Integer.class)
- .defaultValue(30)
- .build();
-
- public static final ConfigOption<Integer> RECONCILER_PARALLELISM =
- ConfigOption.<Integer>builder()
- .key("spark.kubernetes.operator.reconciler.parallelism")
- .enableDynamicOverride(false)
- .description(
- "Thread pool size for Spark Operator reconcilers. Unbounded pool would be used if "
- + "set to non-positive number.")
- .typeParameterClass(Integer.class)
- .defaultValue(50)
- .build();
-
- public static final ConfigOption<Long> RECONCILER_FOREGROUND_REQUEST_TIMEOUT_SECONDS =
- ConfigOption.<Long>builder()
- .key("spark.kubernetes.operator.reconciler.foregroundRequestTimeoutSeconds")
- .enableDynamicOverride(true)
- .description(
- "Timeout (in seconds) to for requests made to API server. This "
- + "applies only to foreground requests.")
- .typeParameterClass(Long.class)
- .defaultValue(30L)
- .build();
-
- public static final ConfigOption<Long> RECONCILER_INTERVAL_SECONDS =
- ConfigOption.<Long>builder()
- .key("spark.kubernetes.operator.reconciler.intervalSeconds")
- .enableDynamicOverride(true)
- .description(
- "Interval (in seconds, non-negative) to reconcile Spark applications. Note that "
- + "reconciliation is always expected to be triggered when app spec / status is "
- + "updated. This interval controls the reconcile behavior of operator "
- + "reconciliation even when there's no update on SparkApplication, e.g. to "
- + "determine whether a hanging app needs to be proactively terminated. Thus "
- + "this is recommended to set to above 2 minutes to avoid unnecessary no-op "
- + "reconciliation.")
- .typeParameterClass(Long.class)
... 11366 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org