You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2023/10/17 14:27:29 UTC
[flink-kubernetes-operator] branch main updated (305498a9 -> b1c40e35)
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
from 305498a9 [FLINK-33187] Don't record duplicate event if no change
new 5c76d01c [FLINK-33097][autoscaler] Initialize the generic autoscaler module and interfaces
new b4e958c4 [FLINK-33098][autoscaler] Move non-kubernetes related autoscaler classes to flink-autoscaler module
new 0877ac28 [FLINK-33098][autoscaler] Support kubernetes autoscaler using generic interface
new b1c40e35 [FLINK-33098][autoscaler] Removing the operator prefix for all autoscaler options
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Dockerfile | 6 +-
docker-entrypoint.sh | 2 +-
docs/content/docs/custom-resource/autoscaler.md | 28 +-
docs/content/docs/operations/configuration.md | 2 +
.../generated/auto_scaler_configuration.html | 48 +--
e2e-tests/data/autoscaler.yaml | 8 +-
examples/autoscaling/autoscaling-dynamic.yaml | 10 +-
examples/autoscaling/autoscaling.yaml | 8 +-
.../pom.xml | 64 ++--
.../org/apache/flink/autoscaler/JobAutoScaler.java | 19 +-
.../flink/autoscaler/JobAutoScalerContext.java | 62 ++++
.../apache/flink/autoscaler/JobAutoScalerImpl.java | 210 +++++++++++++
.../apache/flink}/autoscaler/JobVertexScaler.java | 85 +++---
.../apache/flink/autoscaler/NoopJobAutoscaler.java | 17 +-
.../flink}/autoscaler/RestApiMetricsCollector.java | 32 +-
.../apache/flink}/autoscaler/ScalingExecutor.java | 130 ++++----
.../flink}/autoscaler/ScalingMetricCollector.java | 140 ++++-----
.../flink}/autoscaler/ScalingMetricEvaluator.java | 53 ++--
.../apache/flink}/autoscaler/ScalingSummary.java | 6 +-
.../autoscaler/config/AutoScalerOptions.java | 48 ++-
.../autoscaler/event/AutoScalerEventHandler.java | 43 ++-
.../metrics}/AutoscalerFlinkMetrics.java | 58 +++-
.../autoscaler/metrics/CollectedMetricHistory.java | 4 +-
.../autoscaler/metrics/CollectedMetrics.java | 2 +-
.../org/apache/flink}/autoscaler/metrics/Edge.java | 2 +-
.../autoscaler/metrics/EvaluatedScalingMetric.java | 2 +-
.../flink}/autoscaler/metrics/FlinkMetric.java | 2 +-
.../autoscaler/metrics/MetricAggregator.java | 2 +-
.../autoscaler/metrics/ScalingHistoryUtils.java | 115 +++++++
.../flink}/autoscaler/metrics/ScalingMetric.java | 2 +-
.../flink}/autoscaler/metrics/ScalingMetrics.java | 8 +-
.../flink/autoscaler/realizer/ScalingRealizer.java | 17 +-
.../autoscaler/state/AutoScalerStateStore.java | 74 +++++
.../state/InMemoryAutoScalerStateStore.java | 114 +++++++
.../flink}/autoscaler/topology/JobTopology.java | 2 +-
.../flink}/autoscaler/topology/VertexInfo.java | 2 +-
.../autoscaler/utils/AutoScalerSerDeModule.java | 4 +-
.../flink}/autoscaler/utils/AutoScalerUtils.java | 17 +-
.../autoscaler/AutoScalerFlinkMetricsTest.java | 120 ++++----
.../flink}/autoscaler/BacklogBasedScalingTest.java | 206 ++++++-------
.../flink/autoscaler/JobAutoScalerImplTest.java | 263 ++++++++++++++++
.../flink}/autoscaler/JobVertexScalerTest.java | 224 +++++---------
.../MetricsCollectionAndEvaluationTest.java | 144 ++++-----
.../autoscaler/RecommendedParallelismTest.java | 176 +++++------
.../autoscaler/RestApiMetricsCollectorTest.java | 113 +++++++
.../flink}/autoscaler/ScalingExecutorTest.java | 160 +++++-----
.../autoscaler/ScalingMetricCollectorTest.java | 56 ++--
.../autoscaler/ScalingMetricEvaluatorTest.java | 44 +--
.../flink/autoscaler/TestingAutoscalerUtils.java | 82 +++++
.../flink}/autoscaler/TestingMetricsCollector.java | 21 +-
.../autoscaler/event/TestingEventCollector.java | 110 +++++++
.../autoscaler/metrics/ScalingMetricsTest.java | 8 +-
.../realizer/TestingScalingRealizer.java | 56 ++++
.../state/TestingAutoscalerStateStore.java | 23 +-
.../autoscaler/topology}/JobTopologyTest.java | 3 +-
.../autoscaler/utils/AutoScalerUtilsTest.java | 4 +-
.../src/test/resources/log4j2-test.properties | 0
flink-kubernetes-docs/pom.xml | 3 +-
.../configuration/ConfigOptionsDocGenerator.java | 4 +-
.../operator/autoscaler/AutoScalerInfo.java | 286 ------------------
.../operator/autoscaler/AutoscalerInfoManager.java | 135 ---------
.../operator/autoscaler/JobAutoScalerImpl.java | 248 ---------------
.../operator/autoscaler/AutoScalerInfoTest.java | 242 ---------------
.../autoscaler/AutoscalerInfoManagerTest.java | 132 --------
.../operator/autoscaler/AutoscalerTestUtils.java | 33 --
.../operator/autoscaler/JobAutoScalerImplTest.java | 301 -------------------
.../autoscaler/RestApiMetricsCollectorTest.java | 77 -----
flink-kubernetes-operator/pom.xml | 6 +
.../flink/kubernetes/operator/FlinkOperator.java | 7 +-
.../operator/autoscaler/AutoscalerFactory.java | 49 +++
.../operator/autoscaler/ConfigMapStore.java | 174 +++++++++++
.../KubernetesAutoScalerEventHandler.java | 68 +++++
.../autoscaler/KubernetesAutoScalerStateStore.java | 270 +++++++++++++++++
.../autoscaler/KubernetesJobAutoScalerContext.java | 67 +++++
.../autoscaler/KubernetesScalingRealizer.java | 34 ++-
.../operator/controller/FlinkResourceContext.java | 50 ++++
.../AbstractFlinkResourceReconciler.java | 34 ++-
.../deployment/AbstractJobReconciler.java | 7 +-
.../deployment/ApplicationReconciler.java | 7 +-
.../reconciler/deployment/JobAutoScaler.java | 30 --
.../deployment/NoopJobAutoscalerFactory.java | 36 ---
.../reconciler/deployment/ReconcilerFactory.java | 12 +-
.../reconciler/deployment/SessionReconciler.java | 3 +-
.../sessionjob/SessionJobReconciler.java | 4 +-
.../operator/utils/AutoscalerLoader.java | 57 ----
.../kubernetes/operator/utils/EventRecorder.java | 4 +-
.../operator/autoscaler/AutoscalerFactoryTest.java | 44 +++
.../operator/autoscaler/ConfigMapStoreTest.java | 123 ++++++++
.../KubernetesAutoScalerStateStoreTest.java | 333 +++++++++++++++++++++
.../autoscaler/TestingAutoscalerFactory.java | 23 --
.../TestingKubernetesAutoscalerUtils.java | 48 +++
.../TestingFlinkDeploymentController.java | 5 +-
.../deployment/ApplicationReconcilerTest.java | 25 +-
.../ApplicationReconcilerUpgradeModeTest.java | 3 +-
.../operator/utils/AutoscalerLoaderTest.java | 61 ----
pom.xml | 4 +-
96 files changed, 3514 insertions(+), 2796 deletions(-)
rename {flink-kubernetes-operator-autoscaler => flink-autoscaler}/pom.xml (61%)
rename flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScalerFactory.java => flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScaler.java (66%)
create mode 100644 flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerContext.java
create mode 100644 flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerImpl.java
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/JobVertexScaler.java (78%)
copy flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/health/InformerIdentifier.java => flink-autoscaler/src/main/java/org/apache/flink/autoscaler/NoopJobAutoscaler.java (69%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/RestApiMetricsCollector.java (74%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/ScalingExecutor.java (68%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/ScalingMetricCollector.java (80%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/ScalingMetricEvaluator.java (82%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/ScalingSummary.java (89%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/config/AutoScalerOptions.java (76%)
copy flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/spec/UpgradeMode.java => flink-autoscaler/src/main/java/org/apache/flink/autoscaler/event/AutoScalerEventHandler.java (51%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler => flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics}/AutoscalerFlinkMetrics.java (76%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/CollectedMetricHistory.java (88%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/CollectedMetrics.java (94%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/Edge.java (94%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/EvaluatedScalingMetric.java (95%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/FlinkMetric.java (97%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/MetricAggregator.java (95%)
create mode 100644 flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingHistoryUtils.java
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/ScalingMetric.java (97%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/metrics/ScalingMetrics.java (97%)
copy flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/FlinkSessionJobList.java => flink-autoscaler/src/main/java/org/apache/flink/autoscaler/realizer/ScalingRealizer.java (63%)
create mode 100644 flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/AutoScalerStateStore.java
create mode 100644 flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/InMemoryAutoScalerStateStore.java
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/topology/JobTopology.java (99%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/topology/VertexInfo.java (95%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/utils/AutoScalerSerDeModule.java (95%)
rename {flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/main/java/org/apache/flink}/autoscaler/utils/AutoScalerUtils.java (84%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/AutoScalerFlinkMetricsTest.java (56%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/BacklogBasedScalingTest.java (72%)
create mode 100644 flink-autoscaler/src/test/java/org/apache/flink/autoscaler/JobAutoScalerImplTest.java
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/JobVertexScalerTest.java (63%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/MetricsCollectionAndEvaluationTest.java (82%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/RecommendedParallelismTest.java (60%)
create mode 100644 flink-autoscaler/src/test/java/org/apache/flink/autoscaler/RestApiMetricsCollectorTest.java
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/ScalingExecutorTest.java (62%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/ScalingMetricCollectorTest.java (83%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/ScalingMetricEvaluatorTest.java (86%)
create mode 100644 flink-autoscaler/src/test/java/org/apache/flink/autoscaler/TestingAutoscalerUtils.java
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/TestingMetricsCollector.java (78%)
create mode 100644 flink-autoscaler/src/test/java/org/apache/flink/autoscaler/event/TestingEventCollector.java
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/metrics/ScalingMetricsTest.java (98%)
create mode 100644 flink-autoscaler/src/test/java/org/apache/flink/autoscaler/realizer/TestingScalingRealizer.java
copy flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/exception/ReconciliationException.java => flink-autoscaler/src/test/java/org/apache/flink/autoscaler/state/TestingAutoscalerStateStore.java (61%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler => flink-autoscaler/src/test/java/org/apache/flink/autoscaler/topology}/JobTopologyTest.java (96%)
rename {flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator => flink-autoscaler/src/test/java/org/apache/flink}/autoscaler/utils/AutoScalerUtilsTest.java (94%)
rename {flink-kubernetes-operator-autoscaler => flink-autoscaler}/src/test/resources/log4j2-test.properties (100%)
delete mode 100644 flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java
delete mode 100644 flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManager.java
delete mode 100644 flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java
delete mode 100644 flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java
delete mode 100644 flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManagerTest.java
delete mode 100644 flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerTestUtils.java
delete mode 100644 flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java
delete mode 100644 flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java
create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactory.java
create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStore.java
create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerEventHandler.java
create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStore.java
create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesJobAutoScalerContext.java
rename flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java => flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesScalingRealizer.java (50%)
delete mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScaler.java
delete mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/NoopJobAutoscalerFactory.java
delete mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoader.java
create mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactoryTest.java
create mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStoreTest.java
create mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStoreTest.java
delete mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingAutoscalerFactory.java
create mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingKubernetesAutoscalerUtils.java
delete mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoaderTest.java
[flink-kubernetes-operator] 03/04: [FLINK-33098][autoscaler] Support kubernetes autoscaler using generic interface
Posted by gy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
commit 0877ac28c6a698ea9fd85838a0bb1c05bd72a9bb
Author: fanrui <19...@gmail.com>
AuthorDate: Sat Sep 23 18:11:55 2023 +0800
[FLINK-33098][autoscaler] Support kubernetes autoscaler using generic interface
---
Dockerfile | 10 +-
docker-entrypoint.sh | 2 +-
.../generated/auto_scaler_configuration.html | 6 +
flink-autoscaler/pom.xml | 37 ++-
.../apache/flink/autoscaler/JobAutoScalerImpl.java | 210 +++++++++++++
.../apache/flink}/autoscaler/JobVertexScaler.java | 58 ++--
.../apache/flink/autoscaler/NoopJobAutoscaler.java | 14 +-
.../flink}/autoscaler/RestApiMetricsCollector.java | 30 +-
.../apache/flink}/autoscaler/ScalingExecutor.java | 99 +++---
.../flink}/autoscaler/ScalingMetricCollector.java | 126 ++++----
.../flink/autoscaler/ScalingMetricEvaluator.java | 2 +-
.../flink/autoscaler/config/AutoScalerOptions.java | 16 +-
.../metrics}/AutoscalerFlinkMetrics.java | 54 +++-
.../autoscaler/metrics/ScalingHistoryUtils.java | 115 +++++++
.../state/InMemoryAutoScalerStateStore.java | 114 +++++++
.../flink/autoscaler/utils/AutoScalerUtils.java | 5 +-
.../autoscaler/AutoScalerFlinkMetricsTest.java | 106 ++++---
.../flink}/autoscaler/BacklogBasedScalingTest.java | 199 +++++-------
.../flink/autoscaler/JobAutoScalerImplTest.java | 263 ++++++++++++++++
.../flink}/autoscaler/JobVertexScalerTest.java | 218 +++++---------
.../MetricsCollectionAndEvaluationTest.java | 131 ++++----
.../autoscaler/RecommendedParallelismTest.java | 165 +++++-----
.../autoscaler/RestApiMetricsCollectorTest.java | 113 +++++++
.../flink}/autoscaler/ScalingExecutorTest.java | 154 +++++-----
.../autoscaler/ScalingMetricCollectorTest.java | 47 ++-
.../flink/autoscaler/TestingAutoscalerUtils.java | 82 +++++
.../flink}/autoscaler/TestingMetricsCollector.java | 17 +-
.../autoscaler/event/TestingEventCollector.java | 110 +++++++
.../realizer/TestingScalingRealizer.java | 56 ++++
.../state/TestingAutoscalerStateStore.java | 24 +-
flink-kubernetes-docs/pom.xml | 1 -
flink-kubernetes-operator-autoscaler/pom.xml | 127 --------
.../operator/autoscaler/AutoScalerInfo.java | 287 ------------------
.../operator/autoscaler/AutoscalerInfoManager.java | 135 ---------
.../operator/autoscaler/JobAutoScalerImpl.java | 249 ---------------
.../operator/autoscaler/AutoScalerInfoTest.java | 243 ---------------
.../autoscaler/AutoscalerInfoManagerTest.java | 132 --------
.../operator/autoscaler/JobAutoScalerImplTest.java | 302 -------------------
.../autoscaler/RestApiMetricsCollectorTest.java | 77 -----
.../src/test/resources/log4j2-test.properties | 26 --
flink-kubernetes-operator/pom.xml | 6 +
.../flink/kubernetes/operator/FlinkOperator.java | 7 +-
.../operator/autoscaler/AutoscalerFactory.java | 49 +++
.../operator/autoscaler/ConfigMapStore.java | 174 +++++++++++
.../KubernetesAutoScalerEventHandler.java | 68 +++++
.../autoscaler/KubernetesAutoScalerStateStore.java | 270 +++++++++++++++++
.../autoscaler/KubernetesJobAutoScalerContext.java | 67 +++++
.../autoscaler/KubernetesScalingRealizer.java | 35 +--
.../operator/controller/FlinkResourceContext.java | 50 ++++
.../AbstractFlinkResourceReconciler.java | 34 ++-
.../deployment/AbstractJobReconciler.java | 7 +-
.../deployment/ApplicationReconciler.java | 7 +-
.../reconciler/deployment/JobAutoScaler.java | 30 --
.../deployment/NoopJobAutoscalerFactory.java | 36 ---
.../reconciler/deployment/ReconcilerFactory.java | 12 +-
.../reconciler/deployment/SessionReconciler.java | 3 +-
.../sessionjob/SessionJobReconciler.java | 4 +-
.../operator/utils/AutoscalerLoader.java | 57 ----
.../kubernetes/operator/utils/EventRecorder.java | 4 +-
.../operator/autoscaler/AutoscalerFactoryTest.java | 44 +++
.../operator/autoscaler/ConfigMapStoreTest.java | 123 ++++++++
.../KubernetesAutoScalerStateStoreTest.java | 333 +++++++++++++++++++++
.../autoscaler/TestingAutoscalerFactory.java | 23 --
.../TestingKubernetesAutoscalerUtils.java | 48 +++
.../TestingFlinkDeploymentController.java | 5 +-
.../deployment/ApplicationReconcilerTest.java | 25 +-
.../ApplicationReconcilerUpgradeModeTest.java | 3 +-
.../operator/utils/AutoscalerLoaderTest.java | 61 ----
pom.xml | 3 +-
69 files changed, 3094 insertions(+), 2656 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 4cfb8f66..0d98ec1f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,12 +23,10 @@ WORKDIR /app
COPY . .
-RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-autoscaler,flink-kubernetes-operator-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS
+RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS
RUN cd /app/tools/license; mkdir jars; cd jars; \
cp /app/flink-kubernetes-operator/target/flink-kubernetes-operator-*-shaded.jar . && \
- cp /app/flink-kubernetes-operator-autoscaler/target/flink-kubernetes-operator-autoscaler-*.jar . && \
- cp /app/flink-autoscaler/target/flink-autoscaler-*.jar . && \
cp /app/flink-kubernetes-webhook/target/flink-kubernetes-webhook-*-shaded.jar . && \
cp /app/flink-kubernetes-standalone/target/flink-kubernetes-standalone-*.jar . && \
cp -r /app/flink-kubernetes-operator/target/plugins ./plugins && \
@@ -40,8 +38,6 @@ ENV FLINK_HOME=/opt/flink
ENV FLINK_PLUGINS_DIR=$FLINK_HOME/plugins
ENV OPERATOR_VERSION=1.7-SNAPSHOT
ENV OPERATOR_JAR=flink-kubernetes-operator-$OPERATOR_VERSION-shaded.jar
-ENV FLINK_AUTOSCALER_JAR=flink-autoscaler-$OPERATOR_VERSION.jar
-ENV AUTOSCALER_JAR=flink-kubernetes-operator-autoscaler-$OPERATOR_VERSION.jar
ENV WEBHOOK_JAR=flink-kubernetes-webhook-$OPERATOR_VERSION-shaded.jar
ENV KUBERNETES_STANDALONE_JAR=flink-kubernetes-standalone-$OPERATOR_VERSION.jar
@@ -53,8 +49,6 @@ RUN groupadd --system --gid=9999 flink && \
useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink
COPY --from=build /app/flink-kubernetes-operator/target/$OPERATOR_JAR .
-COPY --from=build /app/flink-autoscaler/target/$FLINK_AUTOSCALER_JAR .
-COPY --from=build /app/flink-kubernetes-operator-autoscaler/target/$AUTOSCALER_JAR .
COPY --from=build /app/flink-kubernetes-webhook/target/$WEBHOOK_JAR .
COPY --from=build /app/flink-kubernetes-standalone/target/$KUBERNETES_STANDALONE_JAR .
COPY --from=build /app/flink-kubernetes-operator/target/plugins $FLINK_HOME/plugins
@@ -64,8 +58,6 @@ COPY docker-entrypoint.sh /
RUN chown -R flink:flink $FLINK_HOME && \
chown flink:flink $OPERATOR_JAR && \
- chown flink:flink $FLINK_AUTOSCALER_JAR && \
- chown flink:flink $AUTOSCALER_JAR && \
chown flink:flink $WEBHOOK_JAR && \
chown flink:flink $KUBERNETES_STANDALONE_JAR && \
chown flink:flink /docker-entrypoint.sh
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index c46d672b..3343f2d6 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -29,7 +29,7 @@ if [ "$1" = "help" ]; then
elif [ "$1" = "operator" ]; then
echo "Starting Operator"
- exec java -cp "./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:./$FLINK_AUTOSCALER_JAR:./$AUTOSCALER_JAR:$OPERATOR_LIB/*" $LOG_CONFIG $JVM_ARGS org.apache.flink.kubernetes.operator.FlinkOperator
+ exec java -cp "./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:$OPERATOR_LIB/*" $LOG_CONFIG $JVM_ARGS org.apache.flink.kubernetes.operator.FlinkOperator
elif [ "$1" = "webhook" ]; then
echo "Starting Webhook"
diff --git a/docs/layouts/shortcodes/generated/auto_scaler_configuration.html b/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
index 8b9d02b7..bc35f57d 100644
--- a/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
+++ b/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
@@ -26,6 +26,12 @@
<td>Boolean</td>
<td>Enable job autoscaler module.</td>
</tr>
+ <tr>
+ <td><h5>kubernetes.operator.job.autoscaler.flink.client.timeout</h5></td>
+ <td style="word-wrap: break-word;">10 s</td>
+ <td>Duration</td>
+ <td>The timeout for waiting the flink rest client to return.</td>
+ </tr>
<tr>
<td><h5>kubernetes.operator.job.autoscaler.history.max.age</h5></td>
<td style="word-wrap: break-word;">86400000 ms</td>
diff --git a/flink-autoscaler/pom.xml b/flink-autoscaler/pom.xml
index 8e216ee9..e6890d57 100644
--- a/flink-autoscaler/pom.xml
+++ b/flink-autoscaler/pom.xml
@@ -46,6 +46,19 @@ under the License.
<scope>provided</scope>
</dependency>
+ <!-- TODO FLINK-33098: These jackson dependencies can be replaced with flink shaded jackson. It can be done
+ after the flink-1.18 is released, because the shaded jackson version of flink-1.17 is 2.13.4,
+ it doesn't support loaderOptions. -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -53,23 +66,33 @@ under the License.
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>
- <!-- TODO FLINK-33098: These jackson dependencies can be replaced with flink shaded jackson. It can be done
- after the flink-1.18 is released, because the shaded jackson version of flink-1.17 is 2.13.4,
- it doesn't support loaderOptions. -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <version>${assertj.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime</artifactId>
+ <version>${flink.version}</version>
+ <type>test-jar</type>
+ <classifier>tests</classifier>
+ <scope>test</scope>
</dependency>
</dependencies>
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerImpl.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerImpl.java
new file mode 100644
index 00000000..3196a3de
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerImpl.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.AutoScalerEventHandler;
+import org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.realizer.ScalingRealizer;
+import org.apache.flink.autoscaler.state.AutoScalerStateStore;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
+import static org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics.initRecommendedParallelism;
+import static org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics.resetRecommendedParallelism;
+
+/** The default implementation of {@link JobAutoScaler}. */
+public class JobAutoScalerImpl<KEY, Context extends JobAutoScalerContext<KEY>>
+ implements JobAutoScaler<KEY, Context> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JobAutoScalerImpl.class);
+
+ @VisibleForTesting protected static final String AUTOSCALER_ERROR = "AutoscalerError";
+
+ private final ScalingMetricCollector<KEY, Context> metricsCollector;
+ private final ScalingMetricEvaluator evaluator;
+ private final ScalingExecutor<KEY, Context> scalingExecutor;
+ private final AutoScalerEventHandler<KEY, Context> eventHandler;
+ private final ScalingRealizer<KEY, Context> scalingRealizer;
+ private final AutoScalerStateStore<KEY, Context> stateStore;
+
+ @VisibleForTesting
+ final Map<KEY, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>
+ lastEvaluatedMetrics = new ConcurrentHashMap<>();
+
+ @VisibleForTesting
+ final Map<KEY, AutoscalerFlinkMetrics> flinkMetrics = new ConcurrentHashMap<>();
+
+ public JobAutoScalerImpl(
+ ScalingMetricCollector<KEY, Context> metricsCollector,
+ ScalingMetricEvaluator evaluator,
+ ScalingExecutor<KEY, Context> scalingExecutor,
+ AutoScalerEventHandler<KEY, Context> eventHandler,
+ ScalingRealizer<KEY, Context> scalingRealizer,
+ AutoScalerStateStore<KEY, Context> stateStore) {
+ this.metricsCollector = metricsCollector;
+ this.evaluator = evaluator;
+ this.scalingExecutor = scalingExecutor;
+ this.eventHandler = eventHandler;
+ this.scalingRealizer = scalingRealizer;
+ this.stateStore = stateStore;
+ }
+
+ @Override
+ public void scale(Context ctx) throws Exception {
+ var autoscalerMetrics = getOrInitAutoscalerFlinkMetrics(ctx);
+
+ try {
+ if (!ctx.getConfiguration().getBoolean(AUTOSCALER_ENABLED)) {
+ LOG.debug("Autoscaler is disabled");
+ clearParallelismOverrides(ctx);
+ return;
+ }
+
+ if (ctx.getJobStatus() != JobStatus.RUNNING) {
+ lastEvaluatedMetrics.remove(ctx.getJobKey());
+ return;
+ }
+
+ runScalingLogic(ctx, autoscalerMetrics);
+ stateStore.flush(ctx);
+ } catch (Throwable e) {
+ onError(ctx, autoscalerMetrics, e);
+ } finally {
+ applyParallelismOverrides(ctx);
+ }
+ }
+
+ @Override
+ public void cleanup(KEY jobKey) {
+ LOG.info("Cleaning up autoscaling meta data");
+ metricsCollector.cleanup(jobKey);
+ lastEvaluatedMetrics.remove(jobKey);
+ flinkMetrics.remove(jobKey);
+ stateStore.removeInfoFromCache(jobKey);
+ }
+
+ private void clearParallelismOverrides(Context ctx) throws Exception {
+ var parallelismOverrides = stateStore.getParallelismOverrides(ctx);
+ if (parallelismOverrides.isPresent()) {
+ stateStore.removeParallelismOverrides(ctx);
+ stateStore.flush(ctx);
+ }
+ }
+
+ @VisibleForTesting
+ protected Optional<Map<String, String>> getParallelismOverrides(Context ctx) throws Exception {
+ return stateStore.getParallelismOverrides(ctx);
+ }
+
+ /**
+ * If there are any parallelism overrides by the {@link JobAutoScaler} apply them to the
+ * scalingRealizer.
+ *
+ * @param ctx Job context
+ */
+ @VisibleForTesting
+ protected void applyParallelismOverrides(Context ctx) throws Exception {
+ var overridesOpt = getParallelismOverrides(ctx);
+ if (overridesOpt.isEmpty() || overridesOpt.get().isEmpty()) {
+ return;
+ }
+ Map<String, String> overrides = overridesOpt.get();
+ LOG.debug("Applying parallelism overrides: {}", overrides);
+
+ var conf = ctx.getConfiguration();
+ var userOverrides = new HashMap<>(conf.get(PipelineOptions.PARALLELISM_OVERRIDES));
+ var exclusions = conf.get(AutoScalerOptions.VERTEX_EXCLUDE_IDS);
+
+ overrides.forEach(
+ (k, v) -> {
+ // Respect user override for excluded vertices
+ if (exclusions.contains(k)) {
+ userOverrides.putIfAbsent(k, v);
+ } else {
+ userOverrides.put(k, v);
+ }
+ });
+ scalingRealizer.realize(ctx, userOverrides);
+ }
+
+ private void runScalingLogic(Context ctx, AutoscalerFlinkMetrics autoscalerMetrics)
+ throws Exception {
+
+ var collectedMetrics = metricsCollector.updateMetrics(ctx, stateStore);
+
+ if (collectedMetrics.getMetricHistory().isEmpty()) {
+ return;
+ }
+ LOG.debug("Collected metrics: {}", collectedMetrics);
+
+ var evaluatedMetrics = evaluator.evaluate(ctx.getConfiguration(), collectedMetrics);
+ LOG.debug("Evaluated metrics: {}", evaluatedMetrics);
+ lastEvaluatedMetrics.put(ctx.getJobKey(), evaluatedMetrics);
+
+ initRecommendedParallelism(evaluatedMetrics);
+ autoscalerMetrics.registerScalingMetrics(
+ collectedMetrics.getJobTopology().getVerticesInTopologicalOrder(),
+ () -> lastEvaluatedMetrics.get(ctx.getJobKey()));
+
+ if (!collectedMetrics.isFullyCollected()) {
+ // We have done an upfront evaluation, but we are not ready for scaling.
+ resetRecommendedParallelism(evaluatedMetrics);
+ return;
+ }
+
+ var parallelismChanged = scalingExecutor.scaleResource(ctx, evaluatedMetrics);
+
+ if (parallelismChanged) {
+ autoscalerMetrics.incrementScaling();
+ } else {
+ autoscalerMetrics.incrementBalanced();
+ }
+ }
+
+ private void onError(Context ctx, AutoscalerFlinkMetrics autoscalerMetrics, Throwable e) {
+ LOG.error("Error while scaling job", e);
+ autoscalerMetrics.incrementError();
+ eventHandler.handleEvent(
+ ctx,
+ AutoScalerEventHandler.Type.Warning,
+ AUTOSCALER_ERROR,
+ e.getMessage(),
+ null,
+ null);
+ }
+
+ private AutoscalerFlinkMetrics getOrInitAutoscalerFlinkMetrics(Context ctx) {
+ return this.flinkMetrics.computeIfAbsent(
+ ctx.getJobKey(),
+ id -> new AutoscalerFlinkMetrics(ctx.getMetricGroup().addGroup("AutoScaler")));
+ }
+}
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScaler.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobVertexScaler.java
similarity index 88%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScaler.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobVertexScaler.java
index 2fdc3e88..9bc46b7e 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScaler.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobVertexScaler.java
@@ -15,21 +15,18 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.autoscaler.ScalingSummary;
import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.AutoScalerEventHandler;
import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.util.Preconditions;
-import io.fabric8.kubernetes.client.KubernetesClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,30 +48,30 @@ import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
/** Component responsible for computing vertex parallelism based on the scaling metrics. */
-public class JobVertexScaler {
+public class JobVertexScaler<KEY, Context extends JobAutoScalerContext<KEY>> {
private static final Logger LOG = LoggerFactory.getLogger(JobVertexScaler.class);
+ @VisibleForTesting protected static final String INEFFECTIVE_SCALING = "IneffectiveScaling";
+
@VisibleForTesting
- public static final String INNEFFECTIVE_MESSAGE_FORMAT =
+ protected static final String INEFFECTIVE_MESSAGE_FORMAT =
"Skipping further scale up after ineffective previous scale up for %s";
private Clock clock = Clock.system(ZoneId.systemDefault());
- private EventRecorder eventRecorder;
+ private final AutoScalerEventHandler<KEY, Context> autoScalerEventHandler;
- public JobVertexScaler(EventRecorder eventRecorder) {
- this.eventRecorder = eventRecorder;
+ public JobVertexScaler(AutoScalerEventHandler<KEY, Context> autoScalerEventHandler) {
+ this.autoScalerEventHandler = autoScalerEventHandler;
}
public int computeScaleTargetParallelism(
- AbstractFlinkResource<?, ?> resource,
- Configuration conf,
+ Context context,
JobVertexID vertex,
Map<ScalingMetric, EvaluatedScalingMetric> evaluatedMetrics,
- SortedMap<Instant, ScalingSummary> history,
- KubernetesClient client) {
-
+ SortedMap<Instant, ScalingSummary> history) {
+ var conf = context.getConfiguration();
var currentParallelism = (int) evaluatedMetrics.get(PARALLELISM).getCurrent();
double averageTrueProcessingRate = evaluatedMetrics.get(TRUE_PROCESSING_RATE).getAverage();
if (Double.isNaN(averageTrueProcessingRate)) {
@@ -128,14 +125,13 @@ public class JobVertexScaler {
if (newParallelism == currentParallelism
|| blockScalingBasedOnPastActions(
- resource,
+ context,
vertex,
conf,
evaluatedMetrics,
history,
currentParallelism,
- newParallelism,
- client)) {
+ newParallelism)) {
return currentParallelism;
}
@@ -147,14 +143,13 @@ public class JobVertexScaler {
}
private boolean blockScalingBasedOnPastActions(
- AbstractFlinkResource<?, ?> resource,
+ Context context,
JobVertexID vertex,
Configuration conf,
Map<ScalingMetric, EvaluatedScalingMetric> evaluatedMetrics,
SortedMap<Instant, ScalingSummary> history,
int currentParallelism,
- int newParallelism,
- KubernetesClient client) {
+ int newParallelism) {
// If we don't have past scaling actions for this vertex, there is nothing to do
if (history.isEmpty()) {
@@ -168,7 +163,7 @@ public class JobVertexScaler {
if (currentParallelism == lastSummary.getNewParallelism() && lastSummary.isScaledUp()) {
if (scaledUp) {
return detectIneffectiveScaleUp(
- resource, vertex, conf, evaluatedMetrics, lastSummary, client);
+ context, vertex, conf, evaluatedMetrics, lastSummary);
} else {
return detectImmediateScaleDownAfterScaleUp(vertex, conf, lastScalingTs);
}
@@ -191,12 +186,11 @@ public class JobVertexScaler {
}
private boolean detectIneffectiveScaleUp(
- AbstractFlinkResource<?, ?> resource,
+ Context context,
JobVertexID vertex,
Configuration conf,
Map<ScalingMetric, EvaluatedScalingMetric> evaluatedMetrics,
- ScalingSummary lastSummary,
- KubernetesClient client) {
+ ScalingSummary lastSummary) {
double lastProcRate = lastSummary.getMetrics().get(TRUE_PROCESSING_RATE).getAverage();
double lastExpectedProcRate =
@@ -217,15 +211,15 @@ public class JobVertexScaler {
return false;
}
- var message = String.format(INNEFFECTIVE_MESSAGE_FORMAT, vertex);
+ var message = String.format(INEFFECTIVE_MESSAGE_FORMAT, vertex);
- eventRecorder.triggerEvent(
- resource,
- EventRecorder.Type.Normal,
- EventRecorder.Reason.IneffectiveScaling,
- EventRecorder.Component.Operator,
+ autoScalerEventHandler.handleEvent(
+ context,
+ AutoScalerEventHandler.Type.Normal,
+ INEFFECTIVE_SCALING,
message,
- client);
+ null,
+ null);
if (conf.get(AutoScalerOptions.SCALING_EFFECTIVENESS_DETECTION_ENABLED)) {
LOG.warn(
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScalerFactory.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/NoopJobAutoscaler.java
similarity index 69%
rename from flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScalerFactory.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/NoopJobAutoscaler.java
index c390a9d2..a7a66f14 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScalerFactory.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/NoopJobAutoscaler.java
@@ -15,12 +15,14 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.reconciler.deployment;
+package org.apache.flink.autoscaler;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+/** An autoscaler implementation which does nothing. */
+public class NoopJobAutoscaler<KEY, Context extends JobAutoScalerContext<KEY>>
+ implements JobAutoScaler<KEY, Context> {
+ @Override
+ public void scale(Context context) throws Exception {}
-/** Factory to construct a new autoscaler instance. */
-public interface JobAutoScalerFactory {
-
- JobAutoScaler create(EventRecorder eventRecorder);
+ @Override
+ public void cleanup(KEY jobKey) {}
}
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/RestApiMetricsCollector.java
similarity index 76%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/RestApiMetricsCollector.java
index dca52a0c..d6650044 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/RestApiMetricsCollector.java
@@ -15,13 +15,9 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
-import org.apache.flink.api.common.JobID;
import org.apache.flink.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.service.FlinkService;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.EmptyRequestBody;
import org.apache.flink.runtime.rest.messages.JobIDPathParameter;
@@ -39,41 +35,31 @@ import java.util.Map;
import java.util.stream.Collectors;
/** Metric collector using flink rest api. */
-public class RestApiMetricsCollector extends ScalingMetricCollector {
+public class RestApiMetricsCollector<KEY, Context extends JobAutoScalerContext<KEY>>
+ extends ScalingMetricCollector<KEY, Context> {
private static final Logger LOG = LoggerFactory.getLogger(RestApiMetricsCollector.class);
@Override
protected Map<JobVertexID, Map<FlinkMetric, AggregatedMetric>> queryAllAggregatedMetrics(
- AbstractFlinkResource<?, ?> cr,
- FlinkService flinkService,
- Configuration conf,
- Map<JobVertexID, Map<String, FlinkMetric>> filteredVertexMetricNames) {
+ Context ctx, Map<JobVertexID, Map<String, FlinkMetric>> filteredVertexMetricNames) {
return filteredVertexMetricNames.entrySet().stream()
.collect(
Collectors.toMap(
Map.Entry::getKey,
- e ->
- queryAggregatedVertexMetrics(
- flinkService, cr, conf, e.getKey(), e.getValue())));
+ e -> queryAggregatedVertexMetrics(ctx, e.getKey(), e.getValue())));
}
@SneakyThrows
protected Map<FlinkMetric, AggregatedMetric> queryAggregatedVertexMetrics(
- FlinkService flinkService,
- AbstractFlinkResource<?, ?> cr,
- Configuration conf,
- JobVertexID jobVertexID,
- Map<String, FlinkMetric> metrics) {
+ Context ctx, JobVertexID jobVertexID, Map<String, FlinkMetric> metrics) {
LOG.debug("Querying metrics {} for {}", metrics, jobVertexID);
- var jobId = JobID.fromHexString(cr.getStatus().getJobStatus().getJobId());
-
var parameters = new AggregatedSubtaskMetricsParameters();
var pathIt = parameters.getPathParameters().iterator();
- ((JobIDPathParameter) pathIt.next()).resolve(jobId);
+ ((JobIDPathParameter) pathIt.next()).resolve(ctx.getJobID());
((JobVertexIdPathParameter) pathIt.next()).resolve(jobVertexID);
parameters
@@ -82,7 +68,7 @@ public class RestApiMetricsCollector extends ScalingMetricCollector {
.next()
.resolveFromString(StringUtils.join(metrics.keySet(), ","));
- try (var restClient = flinkService.getClusterClient(conf)) {
+ try (var restClient = ctx.getRestClusterClient()) {
var responseBody =
restClient
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java
similarity index 77%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java
index 32859a43..cedb4874 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingExecutor.java
@@ -15,20 +15,17 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.autoscaler.ScalingSummary;
import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.AutoScalerEventHandler;
import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+import org.apache.flink.autoscaler.state.AutoScalerStateStore;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.util.Preconditions;
-import io.fabric8.kubernetes.client.KubernetesClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,6 +38,8 @@ import java.util.Map;
import java.util.SortedMap;
import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
+import static org.apache.flink.autoscaler.metrics.ScalingHistoryUtils.addToScalingHistoryAndStore;
+import static org.apache.flink.autoscaler.metrics.ScalingHistoryUtils.getTrimmedScalingHistory;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.EXPECTED_PROCESSING_RATE;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_DOWN_RATE_THRESHOLD;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_UP_RATE_THRESHOLD;
@@ -48,38 +47,47 @@ import static org.apache.flink.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE
import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
/** Class responsible for executing scaling decisions. */
-public class ScalingExecutor {
+public class ScalingExecutor<KEY, Context extends JobAutoScalerContext<KEY>> {
public static final String SCALING_SUMMARY_ENTRY =
" Vertex ID %s | Parallelism %d -> %d | Processing capacity %.2f -> %.2f | Target data rate %.2f";
public static final String SCALING_SUMMARY_HEADER_SCALING_DISABLED =
"Recommended parallelism change:";
public static final String SCALING_SUMMARY_HEADER_SCALING_ENABLED = "Scaling vertices:";
+ @VisibleForTesting static final String SCALING_REPORT_REASON = "ScalingReport";
+
private static final Logger LOG = LoggerFactory.getLogger(ScalingExecutor.class);
- private final JobVertexScaler jobVertexScaler;
- private final EventRecorder eventRecorder;
+ private final JobVertexScaler<KEY, Context> jobVertexScaler;
+ private final AutoScalerEventHandler<KEY, Context> autoScalerEventHandler;
+ private final AutoScalerStateStore<KEY, Context> autoScalerStateStore;
private Clock clock = Clock.system(ZoneId.systemDefault());
- public ScalingExecutor(EventRecorder eventRecorder) {
- this(new JobVertexScaler(eventRecorder), eventRecorder);
+ public ScalingExecutor(
+ AutoScalerEventHandler<KEY, Context> autoScalerEventHandler,
+ AutoScalerStateStore<KEY, Context> autoScalerStateStore) {
+ this(
+ new JobVertexScaler<>(autoScalerEventHandler),
+ autoScalerEventHandler,
+ autoScalerStateStore);
}
- public ScalingExecutor(JobVertexScaler jobVertexScaler, EventRecorder eventRecorder) {
+ public ScalingExecutor(
+ JobVertexScaler<KEY, Context> jobVertexScaler,
+ AutoScalerEventHandler<KEY, Context> autoScalerEventHandler,
+ AutoScalerStateStore<KEY, Context> autoScalerStateStore) {
this.jobVertexScaler = jobVertexScaler;
- this.eventRecorder = eventRecorder;
+ this.autoScalerEventHandler = autoScalerEventHandler;
+ this.autoScalerStateStore = autoScalerStateStore;
}
public boolean scaleResource(
- AbstractFlinkResource<?, ?> resource,
- AutoScalerInfo scalingInformation,
- Configuration conf,
- Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluatedMetrics,
- KubernetesClient client) {
+ Context context,
+ Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluatedMetrics)
+ throws Exception {
- var now = Instant.now();
- var scalingHistory = scalingInformation.getScalingHistory(now, conf);
- var scalingSummaries =
- computeScalingSummary(resource, conf, evaluatedMetrics, scalingHistory, client);
+ var conf = context.getConfiguration();
+ var scalingHistory = getTrimmedScalingHistory(autoScalerStateStore, context, Instant.now());
+ var scalingSummaries = computeScalingSummary(context, evaluatedMetrics, scalingHistory);
if (scalingSummaries.isEmpty()) {
LOG.info("All job vertices are currently running at their target parallelism.");
@@ -95,32 +103,22 @@ public class ScalingExecutor {
var scalingEnabled = conf.get(SCALING_ENABLED);
var scalingReport = scalingReport(scalingSummaries, scalingEnabled);
+ autoScalerEventHandler.handleEvent(
+ context,
+ AutoScalerEventHandler.Type.Normal,
+ SCALING_REPORT_REASON,
+ scalingReport,
+ "ScalingExecutor",
+ scalingEnabled ? null : conf.get(AutoScalerOptions.SCALING_REPORT_INTERVAL));
if (!scalingEnabled) {
- eventRecorder.triggerEventByInterval(
- resource,
- EventRecorder.Type.Normal,
- EventRecorder.Reason.ScalingReport,
- EventRecorder.Component.Operator,
- scalingReport,
- "ScalingExecutor",
- client,
- conf.get(AutoScalerOptions.SCALING_REPORT_INTERVAL));
return false;
- } else {
- eventRecorder.triggerEvent(
- resource,
- EventRecorder.Type.Normal,
- EventRecorder.Reason.ScalingReport,
- EventRecorder.Component.Operator,
- scalingReport,
- "ScalingExecutor",
- client);
}
- scalingInformation.addToScalingHistory(clock.instant(), scalingSummaries, conf);
- scalingInformation.setCurrentOverrides(
- getVertexParallelismOverrides(evaluatedMetrics, scalingSummaries));
+ addToScalingHistoryAndStore(
+ autoScalerStateStore, context, scalingHistory, clock.instant(), scalingSummaries);
+ autoScalerStateStore.storeParallelismOverrides(
+ context, getVertexParallelismOverrides(evaluatedMetrics, scalingSummaries));
return true;
}
@@ -194,14 +192,13 @@ public class ScalingExecutor {
}
private Map<JobVertexID, ScalingSummary> computeScalingSummary(
- AbstractFlinkResource<?, ?> resource,
- Configuration conf,
+ Context context,
Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluatedMetrics,
- Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory,
- KubernetesClient client) {
+ Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory) {
var out = new HashMap<JobVertexID, ScalingSummary>();
- var excludeVertexIdList = conf.get(AutoScalerOptions.VERTEX_EXCLUDE_IDS);
+ var excludeVertexIdList =
+ context.getConfiguration().get(AutoScalerOptions.VERTEX_EXCLUDE_IDS);
evaluatedMetrics.forEach(
(v, metrics) -> {
if (excludeVertexIdList.contains(v.toHexString())) {
@@ -213,13 +210,11 @@ public class ScalingExecutor {
(int) metrics.get(ScalingMetric.PARALLELISM).getCurrent();
var newParallelism =
jobVertexScaler.computeScaleTargetParallelism(
- resource,
- conf,
+ context,
v,
metrics,
scalingHistory.getOrDefault(
- v, Collections.emptySortedMap()),
- client);
+ v, Collections.emptySortedMap()));
if (currentParallelism != newParallelism) {
out.put(
v,
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricCollector.java
similarity index 82%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricCollector.java
index e620fb0e..013fe14d 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricCollector.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
@@ -25,12 +25,10 @@ import org.apache.flink.autoscaler.metrics.CollectedMetrics;
import org.apache.flink.autoscaler.metrics.FlinkMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetrics;
+import org.apache.flink.autoscaler.state.AutoScalerStateStore;
import org.apache.flink.autoscaler.topology.JobTopology;
-import org.apache.flink.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.service.FlinkService;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.EmptyRequestBody;
@@ -42,7 +40,6 @@ import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedSubtaskMetri
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedSubtaskMetricsParameters;
import org.apache.flink.util.Preconditions;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,51 +56,62 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import static org.apache.flink.autoscaler.metrics.ScalingHistoryUtils.updateVertexList;
+import static org.apache.flink.autoscaler.utils.AutoScalerUtils.excludeVerticesFromScaling;
+
/** Metric collector using flink rest api. */
-public abstract class ScalingMetricCollector {
+public abstract class ScalingMetricCollector<KEY, Context extends JobAutoScalerContext<KEY>> {
private static final Logger LOG = LoggerFactory.getLogger(ScalingMetricCollector.class);
- private final Map<ResourceID, Map<JobVertexID, Map<String, FlinkMetric>>>
- availableVertexMetricNames = new ConcurrentHashMap<>();
+ private final Map<KEY, Map<JobVertexID, Map<String, FlinkMetric>>> availableVertexMetricNames =
+ new ConcurrentHashMap<>();
- private final Map<ResourceID, SortedMap<Instant, CollectedMetrics>> histories =
+ private final Map<KEY, SortedMap<Instant, CollectedMetrics>> histories =
new ConcurrentHashMap<>();
private Clock clock = Clock.systemDefaultZone();
public CollectedMetricHistory updateMetrics(
- AbstractFlinkResource<?, ?> cr,
- AutoScalerInfo autoscalerInfo,
- FlinkService flinkService,
- Configuration conf)
- throws Exception {
+ Context ctx, AutoScalerStateStore<KEY, Context> stateStore) throws Exception {
- var resourceID = ResourceID.fromResource(cr);
- var jobId = JobID.fromHexString(cr.getStatus().getJobStatus().getJobId());
+ var jobKey = ctx.getJobKey();
+ var conf = ctx.getConfiguration();
var now = clock.instant();
var metricHistory =
- histories.computeIfAbsent(resourceID, (k) -> autoscalerInfo.getMetricHistory());
+ histories.computeIfAbsent(
+ jobKey,
+ (k) -> {
+ try {
+ return stateStore.getEvaluatedMetrics(ctx).orElse(new TreeMap<>());
+ } catch (Exception exception) {
+ throw new RuntimeException(
+ "Get evaluated metrics failed.", exception);
+ }
+ });
// The timestamp of the first metric observation marks the start
// If we haven't collected any metrics, we are starting now
var metricCollectionStartTs = metricHistory.isEmpty() ? now : metricHistory.firstKey();
- var jobDetailsInfo = flinkService.getJobDetailsInfo(jobId, conf);
+ var jobDetailsInfo =
+ getJobDetailsInfo(ctx, conf.get(AutoScalerOptions.FLINK_CLIENT_TIMEOUT));
var jobUpdateTs = getJobUpdateTs(jobDetailsInfo);
if (jobUpdateTs.isAfter(metricCollectionStartTs)) {
LOG.info("Job updated at {}. Clearing metrics.", jobUpdateTs);
- autoscalerInfo.clearMetricHistory();
- cleanup(cr);
+ stateStore.removeEvaluatedMetrics(ctx);
+ cleanup(ctx.getJobKey());
metricHistory.clear();
metricCollectionStartTs = now;
}
- var topology = getJobTopology(flinkService, conf, autoscalerInfo, jobDetailsInfo);
+ var topology = getJobTopology(ctx, stateStore, jobDetailsInfo);
// Trim metrics outside the metric window from metrics history
var metricWindowSize = getMetricWindowSize(conf);
@@ -117,19 +125,18 @@ public abstract class ScalingMetricCollector {
}
// The filtered list of metrics we want to query for each vertex
- var filteredVertexMetricNames = queryFilteredMetricNames(flinkService, cr, conf, topology);
+ var filteredVertexMetricNames = queryFilteredMetricNames(ctx, topology);
// Aggregated job vertex metrics collected from Flink based on the filtered metric names
- var collectedVertexMetrics =
- queryAllAggregatedMetrics(cr, flinkService, conf, filteredVertexMetricNames);
+ var collectedVertexMetrics = queryAllAggregatedMetrics(ctx, filteredVertexMetricNames);
// The computed scaling metrics based on the collected aggregated vertex metrics
var scalingMetrics =
- convertToScalingMetrics(resourceID, collectedVertexMetrics, topology, conf);
+ convertToScalingMetrics(jobKey, collectedVertexMetrics, topology, conf);
// Add scaling metrics to history if they were computed successfully
metricHistory.put(now, scalingMetrics);
- autoscalerInfo.updateMetricHistory(metricHistory);
+ stateStore.storeEvaluatedMetrics(ctx, metricHistory);
var collectedMetrics = new CollectedMetricHistory(topology, metricHistory);
@@ -154,16 +161,17 @@ public abstract class ScalingMetricCollector {
}
protected JobTopology getJobTopology(
- FlinkService flinkService,
- Configuration conf,
- AutoScalerInfo scalerInfo,
+ Context ctx,
+ AutoScalerStateStore<KEY, Context> stateStore,
JobDetailsInfo jobDetailsInfo)
throws Exception {
var t = getJobTopology(jobDetailsInfo);
- scalerInfo.updateVertexList(t.getVerticesInTopologicalOrder(), clock.instant(), conf);
- updateKafkaSourceMaxParallelisms(flinkService, conf, jobDetailsInfo.getJobId(), t);
- AutoScalerUtils.excludeVerticesFromScaling(conf, t.getFinishedVertices());
+
+ Set<JobVertexID> vertexSet = Set.copyOf(t.getVerticesInTopologicalOrder());
+ updateVertexList(stateStore, ctx, clock.instant(), vertexSet);
+ updateKafkaSourceMaxParallelisms(ctx, jobDetailsInfo.getJobId(), t);
+ excludeVerticesFromScaling(ctx.getConfiguration(), t.getFinishedVertices());
return t;
}
@@ -191,10 +199,9 @@ public abstract class ScalingMetricCollector {
.collect(Collectors.toSet()));
}
- private void updateKafkaSourceMaxParallelisms(
- FlinkService flinkService, Configuration conf, JobID jobId, JobTopology topology)
+ private void updateKafkaSourceMaxParallelisms(Context ctx, JobID jobId, JobTopology topology)
throws Exception {
- try (var restClient = flinkService.getClusterClient(conf)) {
+ try (var restClient = ctx.getRestClusterClient()) {
var partitionRegex = Pattern.compile("^.*\\.partition\\.\\d+\\.currentOffset$");
for (Map.Entry<JobVertexID, Set<JobVertexID>> entry : topology.getInputs().entrySet()) {
if (entry.getValue().isEmpty()) {
@@ -220,14 +227,14 @@ public abstract class ScalingMetricCollector {
* Given a map of collected Flink vertex metrics we compute the scaling metrics for each job
* vertex.
*
- * @param resourceID Resource ID of CR
+ * @param jobKey KEY
* @param collectedMetrics Collected Flink metrics
* @param jobTopology Topology
* @param conf Configuration
* @return Collected metrics.
*/
private CollectedMetrics convertToScalingMetrics(
- ResourceID resourceID,
+ KEY jobKey,
Map<JobVertexID, Map<FlinkMetric, AggregatedMetric>> collectedMetrics,
JobTopology jobTopology,
Configuration conf) {
@@ -260,7 +267,7 @@ public abstract class ScalingMetricCollector {
double lagGrowthRate =
computeLagGrowthRate(
- resourceID,
+ jobKey,
jobVertexID,
vertexScalingMetrics.get(ScalingMetric.LAG));
@@ -285,9 +292,8 @@ public abstract class ScalingMetricCollector {
return new CollectedMetrics(out, outputRatios);
}
- private double computeLagGrowthRate(
- ResourceID resourceID, JobVertexID jobVertexID, Double currentLag) {
- var metricHistory = histories.get(resourceID);
+ private double computeLagGrowthRate(KEY jobKey, JobVertexID jobVertexID, Double currentLag) {
+ var metricHistory = histories.get(jobKey);
if (metricHistory == null || metricHistory.isEmpty()) {
return Double.NaN;
@@ -314,18 +320,13 @@ public abstract class ScalingMetricCollector {
/** Query the available metric names for each job vertex. */
@SneakyThrows
protected Map<JobVertexID, Map<String, FlinkMetric>> queryFilteredMetricNames(
- FlinkService flinkService,
- AbstractFlinkResource<?, ?> cr,
- Configuration conf,
- JobTopology topology) {
+ Context ctx, JobTopology topology) {
- var jobId = JobID.fromHexString(cr.getStatus().getJobStatus().getJobId());
var vertices = topology.getVerticesInTopologicalOrder();
- var resourceId = ResourceID.fromResource(cr);
var names =
availableVertexMetricNames.compute(
- resourceId,
+ ctx.getJobKey(),
(k, previousMetricNames) -> {
if (previousMetricNames != null
&& previousMetricNames
@@ -335,7 +336,7 @@ public abstract class ScalingMetricCollector {
return previousMetricNames;
}
- try (var restClient = flinkService.getClusterClient(conf)) {
+ try (var restClient = ctx.getRestClusterClient()) {
return vertices.stream()
.filter(v -> !topology.getFinishedVertices().contains(v))
.collect(
@@ -344,7 +345,7 @@ public abstract class ScalingMetricCollector {
v ->
getFilteredVertexMetricNames(
restClient,
- jobId,
+ ctx.getJobID(),
v,
topology)));
} catch (Exception e) {
@@ -441,15 +442,21 @@ public abstract class ScalingMetricCollector {
protected abstract Map<JobVertexID, Map<FlinkMetric, AggregatedMetric>>
queryAllAggregatedMetrics(
- AbstractFlinkResource<?, ?> cr,
- FlinkService flinkService,
- Configuration conf,
+ Context ctx,
Map<JobVertexID, Map<String, FlinkMetric>> filteredVertexMetricNames);
- public void cleanup(AbstractFlinkResource<?, ?> cr) {
- var resourceId = ResourceID.fromResource(cr);
- histories.remove(resourceId);
- availableVertexMetricNames.remove(resourceId);
+ public JobDetailsInfo getJobDetailsInfo(
+ JobAutoScalerContext<KEY> context, Duration clientTimeout) throws Exception {
+ try (var restClient = context.getRestClusterClient()) {
+ return restClient
+ .getJobDetails(context.getJobID())
+ .get(clientTimeout.toSeconds(), TimeUnit.SECONDS);
+ }
+ }
+
+ public void cleanup(KEY jobKey) {
+ histories.remove(jobKey);
+ availableVertexMetricNames.remove(jobKey);
}
@VisibleForTesting
@@ -458,13 +465,12 @@ public abstract class ScalingMetricCollector {
}
@VisibleForTesting
- protected Map<ResourceID, Map<JobVertexID, Map<String, FlinkMetric>>>
- getAvailableVertexMetricNames() {
+ protected Map<KEY, Map<JobVertexID, Map<String, FlinkMetric>>> getAvailableVertexMetricNames() {
return availableVertexMetricNames;
}
@VisibleForTesting
- protected Map<ResourceID, SortedMap<Instant, CollectedMetrics>> getHistories() {
+ protected Map<KEY, SortedMap<Instant, CollectedMetrics>> getHistories() {
return histories;
}
}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricEvaluator.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricEvaluator.java
index e5f9e3cb..1aec2cf2 100644
--- a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricEvaluator.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricEvaluator.java
@@ -155,7 +155,7 @@ public class ScalingMetricEvaluator {
}
@VisibleForTesting
- public static void computeProcessingRateThresholds(
+ protected static void computeProcessingRateThresholds(
Map<ScalingMetric, EvaluatedScalingMetric> metrics,
Configuration conf,
boolean processingBacklog) {
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
index 7a9d1d90..f22ea2f0 100644
--- a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
@@ -30,12 +30,16 @@ public class AutoScalerOptions {
public static final String DEPRECATED_K8S_OP_CONF_PREFIX = "kubernetes.operator.";
public static final String AUTOSCALER_CONF_PREFIX = "job.autoscaler.";
- public static ConfigOptions.OptionBuilder operatorConfig(String key) {
- return ConfigOptions.key(DEPRECATED_K8S_OP_CONF_PREFIX + key);
+ private static String operatorConfigKey(String key) {
+ return DEPRECATED_K8S_OP_CONF_PREFIX + key;
+ }
+
+ private static String autoScalerConfigKey(String key) {
+ return operatorConfigKey(AUTOSCALER_CONF_PREFIX + key);
}
private static ConfigOptions.OptionBuilder autoScalerConfig(String key) {
- return operatorConfig(AUTOSCALER_CONF_PREFIX + key);
+ return ConfigOptions.key(autoScalerConfigKey(key));
}
public static final ConfigOption<Boolean> AUTOSCALER_ENABLED =
@@ -179,4 +183,10 @@ public class AutoScalerOptions {
.durationType()
.defaultValue(Duration.ofSeconds(1800))
.withDescription("Time interval to resend the identical event");
+
+ public static final ConfigOption<Duration> FLINK_CLIENT_TIMEOUT =
+ autoScalerConfig("flink.rest-client.timeout")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(10))
+ .withDescription("The timeout for waiting the flink rest client to return.");
}
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/AutoscalerFlinkMetrics.java
similarity index 79%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/AutoscalerFlinkMetrics.java
index eab0f850..7a328df0 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/AutoscalerFlinkMetrics.java
@@ -15,11 +15,9 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler.metrics;
import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -39,17 +37,17 @@ import static org.apache.flink.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARA
public class AutoscalerFlinkMetrics {
private static final Logger LOG = LoggerFactory.getLogger(AutoscalerFlinkMetrics.class);
- @VisibleForTesting static final String CURRENT = "Current";
- @VisibleForTesting static final String AVERAGE = "Average";
- @VisibleForTesting static final String JOB_VERTEX_ID = "jobVertexID";
+ @VisibleForTesting public static final String CURRENT = "Current";
+ @VisibleForTesting public static final String AVERAGE = "Average";
+ @VisibleForTesting public static final String JOB_VERTEX_ID = "jobVertexID";
- final Counter numScalings;
+ private final Counter numScalings;
- final Counter numErrors;
+ private final Counter numErrors;
- final Counter numBalanced;
+ private final Counter numBalanced;
- final MetricGroup metricGroup;
+ private final MetricGroup metricGroup;
private boolean scalingMetricsInitialized;
@@ -60,6 +58,18 @@ public class AutoscalerFlinkMetrics {
this.metricGroup = metricGroup;
}
+ public void incrementScaling() {
+ numScalings.inc();
+ }
+
+ public void incrementError() {
+ numErrors.inc();
+ }
+
+ public void incrementBalanced() {
+ numBalanced.inc();
+ }
+
public void registerScalingMetrics(
List<JobVertexID> jobVertices,
Supplier<Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>
@@ -104,7 +114,7 @@ public class AutoscalerFlinkMetrics {
}
}
- static void initRecommendedParallelism(
+ public static void initRecommendedParallelism(
Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluatedMetrics) {
evaluatedMetrics.forEach(
(jobVertexID, evaluatedScalingMetricMap) ->
@@ -114,10 +124,30 @@ public class AutoscalerFlinkMetrics {
}
@VisibleForTesting
- static void resetRecommendedParallelism(
+ public static void resetRecommendedParallelism(
Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluatedMetrics) {
evaluatedMetrics.forEach(
(jobVertexID, evaluatedScalingMetricMap) ->
evaluatedScalingMetricMap.put(RECOMMENDED_PARALLELISM, null));
}
+
+ @VisibleForTesting
+ public long getNumScalingsCount() {
+ return numScalings.getCount();
+ }
+
+ @VisibleForTesting
+ public long getNumErrorsCount() {
+ return numErrors.getCount();
+ }
+
+ @VisibleForTesting
+ public long getNumBalancedCount() {
+ return numBalanced.getCount();
+ }
+
+ @VisibleForTesting
+ public MetricGroup getMetricGroup() {
+ return metricGroup;
+ }
}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingHistoryUtils.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingHistoryUtils.java
new file mode 100644
index 00000000..1f786ee9
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingHistoryUtils.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.metrics;
+
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.state.AutoScalerStateStore;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+
+import javax.annotation.Nonnull;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/** The utils for scaling history. */
+public class ScalingHistoryUtils {
+
+ public static <KEY, Context extends JobAutoScalerContext<KEY>> void addToScalingHistoryAndStore(
+ AutoScalerStateStore<KEY, Context> stateStore,
+ Context context,
+ Instant now,
+ Map<JobVertexID, ScalingSummary> summaries)
+ throws Exception {
+ addToScalingHistoryAndStore(
+ stateStore,
+ context,
+ getTrimmedScalingHistory(stateStore, context, now),
+ now,
+ summaries);
+ }
+
+ public static <KEY, Context extends JobAutoScalerContext<KEY>> void addToScalingHistoryAndStore(
+ AutoScalerStateStore<KEY, Context> stateStore,
+ Context context,
+ Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory,
+ Instant now,
+ Map<JobVertexID, ScalingSummary> summaries)
+ throws Exception {
+
+ summaries.forEach(
+ (id, summary) ->
+ scalingHistory.computeIfAbsent(id, j -> new TreeMap<>()).put(now, summary));
+ stateStore.storeScalingHistory(context, scalingHistory);
+ }
+
+ public static <KEY, Context extends JobAutoScalerContext<KEY>> void updateVertexList(
+ AutoScalerStateStore<KEY, Context> stateStore,
+ Context ctx,
+ Instant now,
+ Set<JobVertexID> vertexSet)
+ throws Exception {
+ Map<JobVertexID, SortedMap<Instant, ScalingSummary>> trimmedScalingHistory =
+ getTrimmedScalingHistory(stateStore, ctx, now);
+
+ if (trimmedScalingHistory.keySet().removeIf(v -> !vertexSet.contains(v))) {
+ stateStore.storeScalingHistory(ctx, trimmedScalingHistory);
+ }
+ }
+
+ @Nonnull
+ public static <KEY, Context extends JobAutoScalerContext<KEY>>
+ Map<JobVertexID, SortedMap<Instant, ScalingSummary>> getTrimmedScalingHistory(
+ AutoScalerStateStore<KEY, Context> autoScalerStateStore,
+ Context context,
+ Instant now)
+ throws Exception {
+ var conf = context.getConfiguration();
+ return autoScalerStateStore
+ .getScalingHistory(context)
+ .map(
+ scalingHistory -> {
+ Instant expectedStartTime =
+ now.minus(
+ conf.get(AutoScalerOptions.VERTEX_SCALING_HISTORY_AGE));
+ var entryIt = scalingHistory.entrySet().iterator();
+ while (entryIt.hasNext()) {
+ var entry = entryIt.next();
+ // Limit how long past scaling decisions are remembered
+ entry.setValue(
+ new TreeMap<>(entry.getValue().tailMap(expectedStartTime)));
+ var vertexHistory = entry.getValue();
+ while (vertexHistory.size()
+ > conf.get(
+ AutoScalerOptions.VERTEX_SCALING_HISTORY_COUNT)) {
+ vertexHistory.remove(vertexHistory.firstKey());
+ }
+ if (vertexHistory.isEmpty()) {
+ entryIt.remove();
+ }
+ }
+ return scalingHistory;
+ })
+ .orElse(new HashMap<>());
+ }
+}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/InMemoryAutoScalerStateStore.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/InMemoryAutoScalerStateStore.java
new file mode 100644
index 00000000..2685db91
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/InMemoryAutoScalerStateStore.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.state;
+
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The state store based on the Java Heap, the state will be discarded after process restarts.
+ *
+ * @param <KEY> The job key.
+ * @param <Context> The job autoscaler context.
+ */
+public class InMemoryAutoScalerStateStore<KEY, Context extends JobAutoScalerContext<KEY>>
+ implements AutoScalerStateStore<KEY, Context> {
+
+ private final Map<KEY, Map<JobVertexID, SortedMap<Instant, ScalingSummary>>>
+ scalingHistoryStore;
+
+ private final Map<KEY, SortedMap<Instant, CollectedMetrics>> evaluatedMetricsStore;
+
+ private final Map<KEY, Map<String, String>> parallelismOverridesStore;
+
+ public InMemoryAutoScalerStateStore() {
+ scalingHistoryStore = new ConcurrentHashMap<>();
+ evaluatedMetricsStore = new ConcurrentHashMap<>();
+ parallelismOverridesStore = new ConcurrentHashMap<>();
+ }
+
+ @Override
+ public void storeScalingHistory(
+ Context jobContext,
+ Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory) {
+ scalingHistoryStore.put(jobContext.getJobKey(), scalingHistory);
+ }
+
+ @Override
+ public Optional<Map<JobVertexID, SortedMap<Instant, ScalingSummary>>> getScalingHistory(
+ Context jobContext) {
+ return Optional.ofNullable(scalingHistoryStore.get(jobContext.getJobKey()));
+ }
+
+ @Override
+ public void removeScalingHistory(Context jobContext) {
+ scalingHistoryStore.remove(jobContext.getJobKey());
+ }
+
+ @Override
+ public void storeEvaluatedMetrics(
+ Context jobContext, SortedMap<Instant, CollectedMetrics> evaluatedMetrics) {
+ evaluatedMetricsStore.put(jobContext.getJobKey(), evaluatedMetrics);
+ }
+
+ @Override
+ public Optional<SortedMap<Instant, CollectedMetrics>> getEvaluatedMetrics(Context jobContext) {
+ return Optional.ofNullable(evaluatedMetricsStore.get(jobContext.getJobKey()));
+ }
+
+ @Override
+ public void removeEvaluatedMetrics(Context jobContext) {
+ evaluatedMetricsStore.remove(jobContext.getJobKey());
+ }
+
+ @Override
+ public void storeParallelismOverrides(
+ Context jobContext, Map<String, String> parallelismOverrides) {
+ parallelismOverridesStore.put(jobContext.getJobKey(), parallelismOverrides);
+ }
+
+ @Override
+ public Optional<Map<String, String>> getParallelismOverrides(Context jobContext) {
+ return Optional.ofNullable(parallelismOverridesStore.get(jobContext.getJobKey()));
+ }
+
+ @Override
+ public void removeParallelismOverrides(Context jobContext) {
+ parallelismOverridesStore.remove(jobContext.getJobKey());
+ }
+
+ @Override
+ public void flush(Context jobContext) {
+ // The InMemory state store doesn't persist data.
+ }
+
+ @Override
+ public void removeInfoFromCache(KEY jobKey) {
+ scalingHistoryStore.remove(jobKey);
+ evaluatedMetricsStore.remove(jobKey);
+ parallelismOverridesStore.remove(jobKey);
+ }
+}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerUtils.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerUtils.java
index f011fcec..544f13c5 100644
--- a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerUtils.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerUtils.java
@@ -74,7 +74,10 @@ public class AutoScalerUtils {
return Math.round(lagCatchupTargetRate + restartCatchupRate + inputTargetAtUtilization);
}
- /** Temporarily exclude vertex from scaling for this run. This does not update the spec. */
+ /**
+ * Temporarily exclude vertex from scaling for this run. This does not update the
+ * scalingRealizer.
+ */
public static boolean excludeVertexFromScaling(Configuration conf, JobVertexID jobVertexId) {
return excludeVerticesFromScaling(conf, List.of(jobVertexId));
}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerFlinkMetricsTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/AutoScalerFlinkMetricsTest.java
similarity index 60%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerFlinkMetricsTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/AutoScalerFlinkMetricsTest.java
index 22aa1462..6821675e 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerFlinkMetricsTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/AutoScalerFlinkMetricsTest.java
@@ -15,62 +15,75 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics;
import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.metrics.TestingMetricListener;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.metrics.groups.GenericMetricGroup;
+import org.apache.flink.runtime.metrics.util.TestingMetricRegistry;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import static org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics.AVERAGE;
+import static org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics.CURRENT;
+import static org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics.JOB_VERTEX_ID;
+import static org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics.initRecommendedParallelism;
+import static org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics.resetRecommendedParallelism;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.AVERAGE;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.CURRENT;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.JOB_VERTEX_ID;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.initRecommendedParallelism;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.resetRecommendedParallelism;
import static org.junit.jupiter.api.Assertions.assertEquals;
/** {@link AutoscalerFlinkMetrics} tests. */
public class AutoScalerFlinkMetricsTest {
- private final Configuration configuration = new Configuration();
+ public static final String DELIMITER = ".";
private final JobVertexID jobVertexID = new JobVertexID();
- private ResourceID resourceID;
- private TestingMetricListener listener;
+ private JobID jobID;
+
+ private GenericMetricGroup metricGroup;
+
private AutoscalerFlinkMetrics metrics;
+ private final Map<String, Metric> collectedMetrics = new HashMap<>();
@BeforeEach
public void init() {
- listener = new TestingMetricListener(configuration);
- metrics = new AutoscalerFlinkMetrics(listener.getMetricGroup());
- resourceID = ResourceID.fromResource(TestUtils.buildApplicationCluster());
+ TestingMetricRegistry registry =
+ TestingMetricRegistry.builder()
+ .setDelimiter(DELIMITER.charAt(0))
+ .setRegisterConsumer(
+ (metric, name, group) -> {
+ this.collectedMetrics.put(
+ group.getMetricIdentifier(name), metric);
+ })
+ .build();
+ metricGroup = new GenericMetricGroup(registry, null, "test");
+
+ metrics = new AutoscalerFlinkMetrics(metricGroup);
+ jobID = new JobID();
}
@Test
public void testMetricsRegistration() {
var evaluatedMetrics = Map.of(jobVertexID, testMetrics());
var lastEvaluatedMetrics =
- new HashMap<
- ResourceID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
+ new HashMap<JobID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
initRecommendedParallelism(evaluatedMetrics);
- lastEvaluatedMetrics.put(resourceID, evaluatedMetrics);
+ lastEvaluatedMetrics.put(jobID, evaluatedMetrics);
- metrics.registerScalingMetrics(
- List.of(jobVertexID), () -> lastEvaluatedMetrics.get(resourceID));
- metrics.registerScalingMetrics(
- List.of(jobVertexID), () -> lastEvaluatedMetrics.get(resourceID));
+ metrics.registerScalingMetrics(List.of(jobVertexID), () -> lastEvaluatedMetrics.get(jobID));
+ metrics.registerScalingMetrics(List.of(jobVertexID), () -> lastEvaluatedMetrics.get(jobID));
assertEquals(1.0, getCurrentMetricValue(PARALLELISM));
assertEquals(1.0, getCurrentMetricValue(RECOMMENDED_PARALLELISM));
@@ -80,7 +93,7 @@ public class AutoScalerFlinkMetricsTest {
@Test
public void testAllScalingMetricsAreRegistered() {
- int numMetricsAlreadyRegistered = listener.size();
+ int numMetricsAlreadyRegistered = collectedMetrics.size();
metrics.registerScalingMetrics(List.of(jobVertexID), () -> null);
int numScalingMetrics = 0;
for (ScalingMetric scalingMetric : ScalingMetric.values()) {
@@ -90,26 +103,24 @@ public class AutoScalerFlinkMetricsTest {
numScalingMetrics += 1;
}
}
- assertEquals(numMetricsAlreadyRegistered + numScalingMetrics, listener.size());
+ assertEquals(numMetricsAlreadyRegistered + numScalingMetrics, collectedMetrics.size());
}
@Test
public void testMetricsCleanup() {
var evaluatedMetrics = Map.of(jobVertexID, testMetrics());
var lastEvaluatedMetrics =
- new HashMap<
- ResourceID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
+ new HashMap<JobID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
initRecommendedParallelism(evaluatedMetrics);
- lastEvaluatedMetrics.put(resourceID, evaluatedMetrics);
- metrics.registerScalingMetrics(
- List.of(jobVertexID), () -> lastEvaluatedMetrics.get(resourceID));
+ lastEvaluatedMetrics.put(jobID, evaluatedMetrics);
+ metrics.registerScalingMetrics(List.of(jobVertexID), () -> lastEvaluatedMetrics.get(jobID));
assertEquals(1.0, getCurrentMetricValue(PARALLELISM));
assertEquals(1.0, getCurrentMetricValue(RECOMMENDED_PARALLELISM));
assertEquals(1000., getCurrentMetricValue(TRUE_PROCESSING_RATE));
assertEquals(2000., getAverageMetricValue(TRUE_PROCESSING_RATE));
- lastEvaluatedMetrics.remove(resourceID);
+ lastEvaluatedMetrics.remove(jobID);
assertEquals(Double.NaN, getCurrentMetricValue(PARALLELISM));
assertEquals(Double.NaN, getCurrentMetricValue(RECOMMENDED_PARALLELISM));
assertEquals(Double.NaN, getCurrentMetricValue(TRUE_PROCESSING_RATE));
@@ -120,14 +131,12 @@ public class AutoScalerFlinkMetricsTest {
public void testRecommendedParallelismWithinMetricWindow() {
var evaluatedMetrics = Map.of(jobVertexID, testMetrics());
var lastEvaluatedMetrics =
- new HashMap<
- ResourceID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
+ new HashMap<JobID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
initRecommendedParallelism(evaluatedMetrics);
resetRecommendedParallelism(evaluatedMetrics);
- lastEvaluatedMetrics.put(resourceID, evaluatedMetrics);
+ lastEvaluatedMetrics.put(jobID, evaluatedMetrics);
- metrics.registerScalingMetrics(
- List.of(jobVertexID), () -> lastEvaluatedMetrics.get(resourceID));
+ metrics.registerScalingMetrics(List.of(jobVertexID), () -> lastEvaluatedMetrics.get(jobID));
assertEquals(1.0, getCurrentMetricValue(PARALLELISM));
assertEquals(Double.NaN, getCurrentMetricValue(RECOMMENDED_PARALLELISM));
assertEquals(1000., getCurrentMetricValue(TRUE_PROCESSING_RATE));
@@ -138,13 +147,11 @@ public class AutoScalerFlinkMetricsTest {
public void testRecommendedParallelismPastMetricWindow() {
var evaluatedMetrics = Map.of(jobVertexID, testMetrics());
var lastEvaluatedMetrics =
- new HashMap<
- ResourceID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
+ new HashMap<JobID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>();
initRecommendedParallelism(evaluatedMetrics);
- lastEvaluatedMetrics.put(resourceID, evaluatedMetrics);
+ lastEvaluatedMetrics.put(jobID, evaluatedMetrics);
- metrics.registerScalingMetrics(
- List.of(jobVertexID), () -> lastEvaluatedMetrics.get(resourceID));
+ metrics.registerScalingMetrics(List.of(jobVertexID), () -> lastEvaluatedMetrics.get(jobID));
assertEquals(1.0, getCurrentMetricValue(PARALLELISM));
assertEquals(1.0, getCurrentMetricValue(RECOMMENDED_PARALLELISM));
assertEquals(1000., getCurrentMetricValue(TRUE_PROCESSING_RATE));
@@ -160,11 +167,15 @@ public class AutoScalerFlinkMetricsTest {
}
private Object getCurrentMetricValue(ScalingMetric metric) {
- return listener.getGauge(getCurrentMetricId(metric)).orElse(() -> Double.NaN).getValue();
+ return Optional.ofNullable((Gauge) this.collectedMetrics.get(getCurrentMetricId(metric)))
+ .orElse(() -> Double.NaN)
+ .getValue();
}
private Object getAverageMetricValue(ScalingMetric metric) {
- return listener.getGauge(getAverageMetricId(metric)).get().getValue();
+ return Optional.ofNullable((Gauge) this.collectedMetrics.get(getAverageMetricId(metric)))
+ .orElse(() -> Double.NaN)
+ .getValue();
}
private String getCurrentMetricId(ScalingMetric metric) {
@@ -176,7 +187,12 @@ public class AutoScalerFlinkMetricsTest {
}
private String getMetricId(ScalingMetric metric, String classifier) {
- return listener.getMetricId(
- JOB_VERTEX_ID, jobVertexID.toHexString(), metric.name(), classifier);
+ return metricGroup.getMetricIdentifier(
+ String.join(
+ DELIMITER,
+ JOB_VERTEX_ID,
+ jobVertexID.toHexString(),
+ metric.name(),
+ classifier));
}
}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/BacklogBasedScalingTest.java
similarity index 73%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/BacklogBasedScalingTest.java
index 9bdf777c..0128f3ca 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/BacklogBasedScalingTest.java
@@ -15,32 +15,22 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.TestingEventCollector;
+import org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.realizer.TestingScalingRealizer;
+import org.apache.flink.autoscaler.state.TestingAutoscalerStateStore;
import org.apache.flink.autoscaler.topology.JobTopology;
import org.apache.flink.autoscaler.topology.VertexInfo;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.OperatorTestBase;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
-import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
-import org.apache.flink.kubernetes.operator.utils.EventCollector;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
-import io.fabric8.kubernetes.api.model.HasMetadata;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import lombok.Getter;
-import org.jetbrains.annotations.NotNull;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -49,51 +39,50 @@ import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.SortedMap;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerTestUtils.getOrCreateInfo;
+import static org.apache.flink.autoscaler.JobAutoScalerImpl.AUTOSCALER_ERROR;
+import static org.apache.flink.autoscaler.TestingAutoscalerUtils.createDefaultJobAutoScalerContext;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Test for scaling metrics collection logic. */
-@EnableKubernetesMockClient(crud = true)
-public class BacklogBasedScalingTest extends OperatorTestBase {
+public class BacklogBasedScalingTest {
- @Getter private KubernetesClient kubernetesClient;
+ private JobAutoScalerContext<JobID> context;
+ private TestingEventCollector<JobID, JobAutoScalerContext<JobID>> eventCollector;
+ private TestingAutoscalerStateStore<JobID, JobAutoScalerContext<JobID>> stateStore;
- private ScalingMetricEvaluator evaluator;
- private TestingMetricsCollector metricsCollector;
- private ScalingExecutor scalingExecutor;
+ private TestingMetricsCollector<JobID, JobAutoScalerContext<JobID>> metricsCollector;
+ private ScalingExecutor<JobID, JobAutoScalerContext<JobID>> scalingExecutor;
- private FlinkDeployment app;
private JobVertexID source1, sink;
- private JobAutoScalerImpl autoscaler;
-
- private EventCollector eventCollector = new EventCollector();
+ private JobAutoScalerImpl<JobID, JobAutoScalerContext<JobID>> autoscaler;
@BeforeEach
public void setup() {
- evaluator = new ScalingMetricEvaluator();
- scalingExecutor = new ScalingExecutor(new EventRecorder(new EventCollector()));
+ context = createDefaultJobAutoScalerContext();
+
+ eventCollector = new TestingEventCollector<>();
+ stateStore = new TestingAutoscalerStateStore<>();
- app = TestUtils.buildApplicationCluster();
- app.getMetadata().setGeneration(1L);
- app.getStatus().getJobStatus().setJobId(new JobID().toHexString());
- kubernetesClient.resource(app).createOrReplace();
+ scalingExecutor = new ScalingExecutor<>(eventCollector, stateStore);
source1 = new JobVertexID();
sink = new JobVertexID();
metricsCollector =
- new TestingMetricsCollector(
+ new TestingMetricsCollector<>(
new JobTopology(
new VertexInfo(source1, Set.of(), 1, 720),
new VertexInfo(sink, Set.of(source1), 1, 720)));
- var defaultConf = new Configuration();
+ var defaultConf = context.getConfiguration();
defaultConf.set(AutoScalerOptions.AUTOSCALER_ENABLED, true);
defaultConf.set(AutoScalerOptions.STABILIZATION_INTERVAL, Duration.ZERO);
defaultConf.set(AutoScalerOptions.RESTART_TIME, Duration.ofSeconds(1));
@@ -105,18 +94,14 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
defaultConf.set(AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY, 0.1);
defaultConf.set(AutoScalerOptions.SCALE_UP_GRACE_PERIOD, Duration.ZERO);
- configManager = new FlinkConfigManager(defaultConf);
- ReconciliationUtils.updateStatusForDeployedSpec(
- app, configManager.getDeployConfig(app.getMetadata(), app.getSpec()));
- app.getStatus().getJobStatus().setState(JobStatus.RUNNING.name());
- app.getStatus().getReconciliationStatus().markReconciledSpecAsStable();
-
autoscaler =
- new JobAutoScalerImpl(
+ new JobAutoScalerImpl<>(
metricsCollector,
- evaluator,
+ new ScalingMetricEvaluator(),
scalingExecutor,
- new EventRecorder(eventCollector));
+ eventCollector,
+ new TestingScalingRealizer<>(),
+ stateStore);
// Reset custom window size to default
metricsCollector.setTestMetricWindowSize(null);
@@ -124,8 +109,6 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
@Test
public void test() throws Exception {
- var ctx = createAutoscalerTestContext();
-
/* Test scaling up. */
var now = Instant.ofEpochMilli(0);
setClocksTo(now);
@@ -153,19 +136,19 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
new AggregatedMetric(
"", Double.NaN, Double.NaN, Double.NaN, 500.))));
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(1, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
- assertFlinkMetricsCount(0, 0, ctx);
+ autoscaler.scale(context);
+ assertEvaluatedMetricsSize(1);
+ assertFlinkMetricsCount(0, 0);
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(2, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
+ autoscaler.scale(context);
+ assertEvaluatedMetricsSize(2);
- var scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ var scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.get(source1));
assertEquals(4, scaledParallelism.get(sink));
- assertFlinkMetricsCount(1, 0, ctx);
+ assertFlinkMetricsCount(1, 0);
/* Test stability while processing pending records. */
@@ -201,10 +184,10 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
// Adjust metric window size, so we can fill the metric window with three metrics
metricsCollector.setTestMetricWindowSize(Duration.ofSeconds(2));
- autoscaler.scale(getResourceContext(app, ctx));
- assertFlinkMetricsCount(1, 0, ctx);
- assertEquals(1, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ autoscaler.scale(context);
+ assertFlinkMetricsCount(1, 0);
+ assertEvaluatedMetricsSize(1);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.get(source1));
assertEquals(4, scaledParallelism.get(sink));
@@ -231,10 +214,10 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
- autoscaler.scale(getResourceContext(app, ctx));
- assertFlinkMetricsCount(1, 0, ctx);
- assertEquals(2, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ autoscaler.scale(context);
+ assertFlinkMetricsCount(1, 0);
+ assertEvaluatedMetricsSize(2);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.get(source1));
assertEquals(4, scaledParallelism.get(sink));
@@ -262,11 +245,11 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
"", Double.NaN, Double.NaN, Double.NaN, 800.))));
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
- autoscaler.scale(getResourceContext(app, ctx));
- assertFlinkMetricsCount(2, 0, ctx);
- assertEquals(3, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
+ autoscaler.scale(context);
+ assertFlinkMetricsCount(2, 0);
+ assertEvaluatedMetricsSize(3);
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(2, scaledParallelism.get(source1));
assertEquals(2, scaledParallelism.get(sink));
@@ -299,9 +282,9 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
"", Double.NaN, Double.NaN, Double.NaN, 900.))));
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
- autoscaler.scale(getResourceContext(app, ctx));
- assertFlinkMetricsCount(2, 0, ctx);
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ autoscaler.scale(context);
+ assertFlinkMetricsCount(2, 0);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(2, scaledParallelism.get(source1));
assertEquals(2, scaledParallelism.get(sink));
@@ -326,10 +309,10 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
"", Double.NaN, Double.NaN, Double.NaN, 900.))));
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
- autoscaler.scale(getResourceContext(app, ctx));
- assertFlinkMetricsCount(2, 0, ctx);
+ autoscaler.scale(context);
+ assertFlinkMetricsCount(2, 0);
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(2, scaledParallelism.get(source1));
assertEquals(2, scaledParallelism.get(sink));
@@ -354,16 +337,15 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
"", Double.NaN, Double.NaN, Double.NaN, 500.))));
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
- autoscaler.scale(getResourceContext(app, ctx));
- assertFlinkMetricsCount(2, 1, ctx);
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ autoscaler.scale(context);
+ assertFlinkMetricsCount(2, 1);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(2, scaledParallelism.get(source1));
assertEquals(2, scaledParallelism.get(sink));
}
@Test
- public void testMetricsPersistedAfterRedeploy() {
- var ctx = createAutoscalerTestContext();
+ public void testMetricsPersistedAfterRedeploy() throws Exception {
var now = Instant.ofEpochMilli(0);
setClocksTo(now);
metricsCollector.setJobUpdateTs(now);
@@ -385,69 +367,52 @@ public class BacklogBasedScalingTest extends OperatorTestBase {
new AggregatedMetric(
"", Double.NaN, Double.NaN, Double.NaN, 500.))));
- autoscaler.scale(getResourceContext(app, ctx));
- assertFalse(getOrCreateInfo(app, kubernetesClient).getMetricHistory().isEmpty());
+ autoscaler.scale(context);
+ assertFalse(stateStore.getEvaluatedMetrics(context).get().isEmpty());
}
@Test
- public void testEventOnError() {
+ public void testEventOnError() throws Exception {
// Invalid config
- app.getSpec()
- .getFlinkConfiguration()
- .put("kubernetes.operator.job.autoscaler.enabled", "3");
- autoscaler.scale(getResourceContext(app, createAutoscalerTestContext()));
+ context.getConfiguration().setString(AutoScalerOptions.AUTOSCALER_ENABLED.key(), "3");
+ autoscaler.scale(context);
var event = eventCollector.events.poll();
assertTrue(eventCollector.events.isEmpty());
- assertEquals(EventRecorder.Reason.AutoscalerError.toString(), event.getReason());
+ Assertions.assertEquals(AUTOSCALER_ERROR, event.getReason());
assertTrue(event.getMessage().startsWith("Could not parse"));
}
@Test
- public void testNoEvaluationDuringStabilization() {
- var conf = configManager.getDefaultConfig();
- conf.set(AutoScalerOptions.STABILIZATION_INTERVAL, Duration.ofMinutes(1));
- configManager.updateDefaultConfig(conf);
+ public void testNoEvaluationDuringStabilization() throws Exception {
+ context.getConfiguration()
+ .set(AutoScalerOptions.STABILIZATION_INTERVAL, Duration.ofMinutes(1));
- var ctx = createAutoscalerTestContext();
var now = Instant.ofEpochMilli(0);
setClocksTo(now);
metricsCollector.setJobUpdateTs(now);
- autoscaler.scale(getResourceContext(app, ctx));
- assertTrue(getOrCreateInfo(app, kubernetesClient).getMetricHistory().isEmpty());
+ autoscaler.scale(context);
+ assertTrue(stateStore.getEvaluatedMetrics(context).isEmpty());
assertTrue(eventCollector.events.isEmpty());
}
+ private void assertEvaluatedMetricsSize(int expectedSize) {
+ Optional<SortedMap<Instant, CollectedMetrics>> evaluatedMetricsOpt =
+ stateStore.getEvaluatedMetrics(context);
+ assertThat(evaluatedMetricsOpt).isPresent();
+ assertThat(evaluatedMetricsOpt.get()).hasSize(expectedSize);
+ }
+
private void setClocksTo(Instant time) {
var clock = Clock.fixed(time, ZoneId.systemDefault());
metricsCollector.setClock(clock);
scalingExecutor.setClock(clock);
}
- @NotNull
- private TestUtils.TestingContext<HasMetadata> createAutoscalerTestContext() {
- return new TestUtils.TestingContext<>() {
-
- @Override
- public <T1> Set<T1> getSecondaryResources(Class<T1> aClass) {
- return (Set)
- kubernetesClient.configMaps().inAnyNamespace().list().getItems().stream()
- .collect(Collectors.toSet());
- }
-
- @Override
- public KubernetesClient getClient() {
- return kubernetesClient;
- }
- };
- }
-
- private void assertFlinkMetricsCount(
- int scalingCount, int balancedCount, TestUtils.TestingContext<HasMetadata> ctx) {
+ private void assertFlinkMetricsCount(int scalingCount, int balancedCount) {
AutoscalerFlinkMetrics autoscalerFlinkMetrics =
- autoscaler.flinkMetrics.get(
- ResourceID.fromResource(getResourceContext(app, ctx).getResource()));
- assertEquals(scalingCount, autoscalerFlinkMetrics.numScalings.getCount());
- assertEquals(balancedCount, autoscalerFlinkMetrics.numBalanced.getCount());
+ autoscaler.flinkMetrics.get(context.getJobKey());
+ assertEquals(scalingCount, autoscalerFlinkMetrics.getNumScalingsCount());
+ assertEquals(balancedCount, autoscalerFlinkMetrics.getNumBalancedCount());
}
}
diff --git a/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/JobAutoScalerImplTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/JobAutoScalerImplTest.java
new file mode 100644
index 00000000..d041c384
--- /dev/null
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/JobAutoScalerImplTest.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.TestingEventCollector;
+import org.apache.flink.autoscaler.metrics.AutoscalerFlinkMetrics;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.realizer.TestingScalingRealizer;
+import org.apache.flink.autoscaler.state.TestingAutoscalerStateStore;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
+import org.apache.flink.configuration.PipelineOptions;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.mock.Whitebox;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.metrics.groups.GenericMetricGroup;
+import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.flink.autoscaler.TestingAutoscalerUtils.createDefaultJobAutoScalerContext;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/** Tests for JobAutoScalerImpl. */
+public class JobAutoScalerImplTest {
+
+ private JobAutoScalerContext<JobID> context;
+ private TestingScalingRealizer<JobID, JobAutoScalerContext<JobID>> scalingRealizer;
+ private TestingEventCollector<JobID, JobAutoScalerContext<JobID>> eventCollector;
+ private TestingAutoscalerStateStore<JobID, JobAutoScalerContext<JobID>> stateStore;
+
+ @BeforeEach
+ public void setup() {
+ context = createDefaultJobAutoScalerContext();
+ context.getConfiguration().set(AUTOSCALER_ENABLED, true);
+
+ scalingRealizer = new TestingScalingRealizer<>();
+ eventCollector = new TestingEventCollector<>();
+ stateStore = new TestingAutoscalerStateStore<>();
+ }
+
+ @Test
+ void testMetricReporting() throws Exception {
+ JobVertexID jobVertexID = new JobVertexID();
+ JobTopology jobTopology = new JobTopology(new VertexInfo(jobVertexID, Set.of(), 1, 10));
+
+ TestingMetricsCollector<JobID, JobAutoScalerContext<JobID>> metricsCollector =
+ new TestingMetricsCollector<>(jobTopology);
+ metricsCollector.setCurrentMetrics(
+ Map.of(
+ jobVertexID,
+ Map.of(
+ FlinkMetric.BUSY_TIME_PER_SEC,
+ new AggregatedMetric("load", 0., 420., 0., 0.))));
+ metricsCollector.setJobUpdateTs(Instant.ofEpochMilli(0));
+
+ ScalingMetricEvaluator evaluator = new ScalingMetricEvaluator();
+ ScalingExecutor<JobID, JobAutoScalerContext<JobID>> scalingExecutor =
+ new ScalingExecutor<>(eventCollector, stateStore);
+
+ var autoscaler =
+ new JobAutoScalerImpl<>(
+ metricsCollector,
+ evaluator,
+ scalingExecutor,
+ eventCollector,
+ scalingRealizer,
+ stateStore);
+
+ autoscaler.scale(context);
+
+ MetricGroup metricGroup = autoscaler.flinkMetrics.get(context.getJobKey()).getMetricGroup();
+ assertEquals(
+ 0.42,
+ getGaugeValue(
+ metricGroup,
+ AutoscalerFlinkMetrics.CURRENT,
+ AutoscalerFlinkMetrics.JOB_VERTEX_ID,
+ jobVertexID.toHexString(),
+ ScalingMetric.LOAD.name()),
+ "Expected scaling metric LOAD was not reported. Reporting is broken");
+ }
+
+ @SuppressWarnings("unchecked")
+ private static double getGaugeValue(
+ MetricGroup metricGroup, String gaugeName, String... nestedMetricGroupNames) {
+ for (String nestedMetricGroupName : nestedMetricGroupNames) {
+ metricGroup =
+ ((Map<String, GenericMetricGroup>)
+ Whitebox.getInternalState(metricGroup, "groups"))
+ .get(nestedMetricGroupName);
+ }
+ var metrics = (Map<String, Metric>) Whitebox.getInternalState(metricGroup, "metrics");
+ return ((Gauge<Double>) metrics.get(gaugeName)).getValue();
+ }
+
+ @Test
+ void testErrorReporting() throws Exception {
+ var autoscaler =
+ new JobAutoScalerImpl<>(
+ null, null, null, eventCollector, scalingRealizer, stateStore);
+
+ autoscaler.scale(context);
+ Assertions.assertEquals(
+ 1, autoscaler.flinkMetrics.get(context.getJobKey()).getNumErrorsCount());
+
+ autoscaler.scale(context);
+ Assertions.assertEquals(
+ 2, autoscaler.flinkMetrics.get(context.getJobKey()).getNumErrorsCount());
+
+ assertEquals(0, autoscaler.flinkMetrics.get(context.getJobKey()).getNumScalingsCount());
+ }
+
+ @Test
+ void testParallelismOverrides() throws Exception {
+ var autoscaler =
+ new JobAutoScalerImpl<>(
+ null, null, null, eventCollector, scalingRealizer, stateStore);
+
+ // Initially we should return empty overrides, do not crate any state
+ assertThat(autoscaler.getParallelismOverrides(context)).isEmpty();
+ assertThat(stateStore.getParallelismOverrides(context)).isEmpty();
+
+ var v1 = new JobVertexID().toString();
+ var v2 = new JobVertexID().toString();
+ stateStore.storeParallelismOverrides(context, Map.of(v1, "1", v2, "2"));
+ stateStore.flush(context);
+
+ autoscaler.applyParallelismOverrides(context);
+ assertParallelismOverrides(Map.of(v1, "1", v2, "2"));
+
+ assertThat(stateStore.getParallelismOverrides(context)).hasValue(Map.of(v1, "1", v2, "2"));
+
+ // Disabling autoscaler should clear overrides
+ context.getConfiguration().setString(AUTOSCALER_ENABLED.key(), "false");
+
+ autoscaler.scale(context);
+ assertThat(autoscaler.getParallelismOverrides(context)).isEmpty();
+ assertParallelismOverrides(null);
+
+ int requestCount = stateStore.getFlushCount();
+ // Make sure we don't update in kubernetes once removed
+ autoscaler.scale(context);
+ assertEquals(requestCount, stateStore.getFlushCount());
+
+ context.getConfiguration().setString(AUTOSCALER_ENABLED.key(), "true");
+ autoscaler.applyParallelismOverrides(context);
+
+ assertThat(stateStore.getParallelismOverrides(context)).isEmpty();
+ assertParallelismOverrides(null);
+
+ stateStore.storeParallelismOverrides(context, Map.of(v1, "1", v2, "2"));
+ stateStore.flush(context);
+ autoscaler.applyParallelismOverrides(context);
+
+ assertThat(autoscaler.getParallelismOverrides(context)).hasValue(Map.of(v1, "1", v2, "2"));
+ assertParallelismOverrides(Map.of(v1, "1", v2, "2"));
+
+ context.getConfiguration().setString(SCALING_ENABLED.key(), "false");
+
+ autoscaler.applyParallelismOverrides(context);
+ assertThat(autoscaler.getParallelismOverrides(context)).hasValue(Map.of(v1, "1", v2, "2"));
+ assertParallelismOverrides(Map.of(v1, "1", v2, "2"));
+
+ // Test error handling
+ // Invalid config
+ context.getConfiguration().setString(AUTOSCALER_ENABLED.key(), "asd");
+ autoscaler.scale(context);
+ assertParallelismOverrides(Map.of(v1, "1", v2, "2"));
+ }
+
+ @Test
+ public void testApplyAutoscalerParallelism() throws Exception {
+ var overrides = new HashMap<String, String>();
+ var autoscaler =
+ new JobAutoScalerImpl<>(
+ null, null, null, eventCollector, scalingRealizer, stateStore) {
+ public Optional<Map<String, String>> getParallelismOverrides(
+ JobAutoScalerContext<JobID> ctx) {
+ return Optional.of(new HashMap<>(overrides));
+ }
+ };
+
+ // Verify no scalingRealizer if overrides are empty
+ autoscaler.applyParallelismOverrides(context);
+ assertParallelismOverrides(null);
+
+ // Make sure overrides are applied to the scalingRealizer
+ var v1 = new JobVertexID();
+ overrides.put(v1.toHexString(), "2");
+
+ // Verify no upgrades if overrides are empty
+ autoscaler.applyParallelismOverrides(context);
+
+ assertParallelismOverrides(Map.of(v1.toHexString(), "2"));
+
+ // We set a user override for v1, it should be ignored and the autoscaler override should
+ // take precedence
+ context.getConfiguration()
+ .setString(PipelineOptions.PARALLELISM_OVERRIDES.key(), v1 + ":1");
+ autoscaler.applyParallelismOverrides(context);
+ assertParallelismOverrides(Map.of(v1.toHexString(), "2"));
+
+ // Define partly overlapping overrides, user overrides for new vertices should be applied
+ var v2 = new JobVertexID();
+
+ context.getConfiguration()
+ .setString(PipelineOptions.PARALLELISM_OVERRIDES.key(), v1 + ":1," + v2 + ":4");
+ autoscaler.applyParallelismOverrides(context);
+ assertParallelismOverrides(Map.of(v1.toString(), "2", v2.toString(), "4"));
+
+ // Make sure user overrides apply to excluded vertices
+ context.getConfiguration()
+ .setString(AutoScalerOptions.VERTEX_EXCLUDE_IDS.key(), v1.toString());
+ context.getConfiguration()
+ .setString(PipelineOptions.PARALLELISM_OVERRIDES.key(), v1 + ":1," + v2 + ":4");
+
+ autoscaler.applyParallelismOverrides(context);
+ assertParallelismOverrides(Map.of(v1.toString(), "1", v2.toString(), "4"));
+ }
+
+ private void assertParallelismOverrides(Map<String, String> expectedOverrides) {
+ TestingScalingRealizer.Event<JobID, JobAutoScalerContext<JobID>> scalingEvent;
+ scalingEvent = scalingRealizer.events.poll();
+ if (expectedOverrides == null) {
+ assertThat(scalingEvent).isNull();
+ return;
+ }
+ assertThat(scalingEvent).isNotNull();
+ assertEquals(expectedOverrides, scalingEvent.getParallelismOverrides());
+ }
+}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScalerTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/JobVertexScalerTest.java
similarity index 63%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScalerTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/JobVertexScalerTest.java
index a700f89e..d70db971 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScalerTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/JobVertexScalerTest.java
@@ -15,23 +15,18 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
-import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.TestingEventCollector;
import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.utils.EventCollector;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.runtime.jobgraph.JobVertexID;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
-import org.junit.Assert;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -43,33 +38,39 @@ import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
-import static org.apache.flink.kubernetes.operator.autoscaler.JobVertexScaler.INNEFFECTIVE_MESSAGE_FORMAT;
+import static org.apache.flink.autoscaler.JobVertexScaler.INEFFECTIVE_MESSAGE_FORMAT;
+import static org.apache.flink.autoscaler.JobVertexScaler.INEFFECTIVE_SCALING;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Test for vertex parallelism scaler logic. */
-@EnableKubernetesMockClient(crud = true)
public class JobVertexScalerTest {
- private JobVertexScaler vertexScaler;
+ private TestingEventCollector<JobID, JobAutoScalerContext<JobID>> eventCollector;
+ private JobVertexScaler<JobID, JobAutoScalerContext<JobID>> vertexScaler;
+ private JobAutoScalerContext<JobID> context;
private Configuration conf;
- private KubernetesClient kubernetesClient;
- private EventCollector eventCollector;
-
- private FlinkDeployment flinkDep;
-
@BeforeEach
public void setup() {
- flinkDep = TestUtils.buildApplicationCluster();
- kubernetesClient.resource(flinkDep).createOrReplace();
- eventCollector = new EventCollector();
- vertexScaler = new JobVertexScaler(new EventRecorder(eventCollector));
+ eventCollector = new TestingEventCollector<>();
+ vertexScaler = new JobVertexScaler<>(eventCollector);
conf = new Configuration();
conf.set(AutoScalerOptions.MAX_SCALE_DOWN_FACTOR, 1.);
conf.set(AutoScalerOptions.MAX_SCALE_UP_FACTOR, (double) Integer.MAX_VALUE);
conf.set(AutoScalerOptions.CATCH_UP_DURATION, Duration.ZERO);
+ JobID jobID = new JobID();
+ context =
+ new JobAutoScalerContext<>(
+ jobID,
+ jobID,
+ JobStatus.RUNNING,
+ conf,
+ new UnregisteredMetricsGroup(),
+ null);
}
@Test
@@ -79,123 +80,68 @@ public class JobVertexScalerTest {
assertEquals(
5,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 50, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 50, 100), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.TARGET_UTILIZATION, .8);
assertEquals(
8,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 50, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 50, 100), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.TARGET_UTILIZATION, .8);
assertEquals(
10,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 80, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 80, 100), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.TARGET_UTILIZATION, .8);
assertEquals(
8,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 60, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 60, 100), Collections.emptySortedMap()));
assertEquals(
8,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 59, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 59, 100), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.TARGET_UTILIZATION, 0.5);
assertEquals(
10,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(2, 100, 40),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(2, 100, 40), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.TARGET_UTILIZATION, 0.6);
assertEquals(
4,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(2, 100, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(2, 100, 100), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.TARGET_UTILIZATION, 1.);
conf.set(AutoScalerOptions.MAX_SCALE_DOWN_FACTOR, 0.5);
assertEquals(
5,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 10, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 10, 100), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.MAX_SCALE_DOWN_FACTOR, 0.6);
assertEquals(
4,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 10, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 10, 100), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.TARGET_UTILIZATION, 1.);
conf.set(AutoScalerOptions.MAX_SCALE_UP_FACTOR, 0.5);
assertEquals(
15,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 200, 10),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 200, 10), Collections.emptySortedMap()));
conf.set(AutoScalerOptions.MAX_SCALE_UP_FACTOR, 0.6);
assertEquals(
16,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
- op,
- evaluated(10, 200, 10),
- Collections.emptySortedMap(),
- kubernetesClient));
+ context, op, evaluated(10, 200, 10), Collections.emptySortedMap()));
}
@Test
@@ -226,11 +172,13 @@ public class JobVertexScalerTest {
@Test
public void ensureMinParallelismDoesNotExceedMax() {
- Assert.assertThrows(
- IllegalArgumentException.class,
- () ->
- assertEquals(
- 600, JobVertexScaler.scale(200, 720, Integer.MAX_VALUE, 500, 499)));
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(
+ () ->
+ assertEquals(
+ 600,
+ JobVertexScaler.scale(
+ 200, 720, Integer.MAX_VALUE, 500, 499)));
}
@Test
@@ -239,23 +187,19 @@ public class JobVertexScalerTest {
assertEquals(
5,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
+ context,
new JobVertexID(),
evaluated(10, 100, 500),
- Collections.emptySortedMap(),
- kubernetesClient));
+ Collections.emptySortedMap()));
// Make sure we respect current parallelism in case it's lower
assertEquals(
4,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
+ context,
new JobVertexID(),
evaluated(4, 100, 500),
- Collections.emptySortedMap(),
- kubernetesClient));
+ Collections.emptySortedMap()));
}
@Test
@@ -265,23 +209,19 @@ public class JobVertexScalerTest {
assertEquals(
10,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
+ context,
new JobVertexID(),
evaluated(10, 500, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ Collections.emptySortedMap()));
// Make sure we respect current parallelism in case it's higher
assertEquals(
12,
vertexScaler.computeScaleTargetParallelism(
- flinkDep,
- conf,
+ context,
new JobVertexID(),
evaluated(12, 500, 100),
- Collections.emptySortedMap(),
- kubernetesClient));
+ Collections.emptySortedMap()));
}
@Test
@@ -295,35 +235,27 @@ public class JobVertexScalerTest {
var evaluated = evaluated(5, 100, 50);
var history = new TreeMap<Instant, ScalingSummary>();
assertEquals(
- 10,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 10, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
history.put(clock.instant(), new ScalingSummary(5, 10, evaluated));
// Should not allow scale back down immediately
evaluated = evaluated(10, 50, 100);
assertEquals(
- 10,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 10, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
// Pass some time...
clock = Clock.offset(Clock.systemDefaultZone(), Duration.ofSeconds(61));
vertexScaler.setClock(clock);
assertEquals(
- 5,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 5, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
history.put(clock.instant(), new ScalingSummary(10, 5, evaluated));
// Allow immediate scale up
evaluated = evaluated(5, 100, 50);
assertEquals(
- 10,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 10, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
history.put(clock.instant(), new ScalingSummary(5, 10, evaluated));
}
@@ -337,18 +269,14 @@ public class JobVertexScalerTest {
var evaluated = evaluated(5, 100, 50);
var history = new TreeMap<Instant, ScalingSummary>();
assertEquals(
- 10,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 10, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
assertEquals(100, evaluated.get(ScalingMetric.EXPECTED_PROCESSING_RATE).getCurrent());
history.put(Instant.now(), new ScalingSummary(5, 10, evaluated));
// Allow to scale higher if scaling was effective (80%)
evaluated = evaluated(10, 180, 90);
assertEquals(
- 20,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 20, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
assertEquals(180, evaluated.get(ScalingMetric.EXPECTED_PROCESSING_RATE).getCurrent());
history.put(Instant.now(), new ScalingSummary(10, 20, evaluated));
@@ -356,50 +284,38 @@ public class JobVertexScalerTest {
// 90 -> 94. Do not try to scale above 20
evaluated = evaluated(20, 180, 94);
assertEquals(
- 20,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 20, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
assertFalse(evaluated.containsKey(ScalingMetric.EXPECTED_PROCESSING_RATE));
// Still considered ineffective (less than <10%)
evaluated = evaluated(20, 180, 98);
assertEquals(
- 20,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 20, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
assertFalse(evaluated.containsKey(ScalingMetric.EXPECTED_PROCESSING_RATE));
// Allow scale up if current parallelism doesnt match last (user rescaled manually)
evaluated = evaluated(10, 180, 90);
assertEquals(
- 20,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 20, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
// Over 10%, effective
evaluated = evaluated(20, 180, 100);
assertEquals(
- 36,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 36, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
assertTrue(evaluated.containsKey(ScalingMetric.EXPECTED_PROCESSING_RATE));
// Ineffective but detection is turned off
conf.set(AutoScalerOptions.SCALING_EFFECTIVENESS_DETECTION_ENABLED, false);
evaluated = evaluated(20, 180, 90);
assertEquals(
- 40,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 40, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
assertTrue(evaluated.containsKey(ScalingMetric.EXPECTED_PROCESSING_RATE));
conf.set(AutoScalerOptions.SCALING_EFFECTIVENESS_DETECTION_ENABLED, true);
// Allow scale down even if ineffective
evaluated = evaluated(20, 45, 90);
assertEquals(
- 10,
- vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, op, evaluated, history, kubernetesClient));
+ 10, vertexScaler.computeScaleTargetParallelism(context, op, evaluated, history));
assertTrue(evaluated.containsKey(ScalingMetric.EXPECTED_PROCESSING_RATE));
}
@@ -415,7 +331,7 @@ public class JobVertexScalerTest {
assertEquals(
10,
vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, jobVertexID, evaluated, history, kubernetesClient));
+ context, jobVertexID, evaluated, history));
assertEquals(100, evaluated.get(ScalingMetric.EXPECTED_PROCESSING_RATE).getCurrent());
history.put(Instant.now(), new ScalingSummary(5, 10, evaluated));
@@ -424,7 +340,7 @@ public class JobVertexScalerTest {
assertEquals(
20,
vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, jobVertexID, evaluated, history, kubernetesClient));
+ context, jobVertexID, evaluated, history));
assertEquals(180, evaluated.get(ScalingMetric.EXPECTED_PROCESSING_RATE).getCurrent());
history.put(Instant.now(), new ScalingSummary(10, 20, evaluated));
assertEquals(0, eventCollector.events.size());
@@ -434,12 +350,14 @@ public class JobVertexScalerTest {
assertEquals(
20,
vertexScaler.computeScaleTargetParallelism(
- flinkDep, conf, jobVertexID, evaluated, history, kubernetesClient));
+ context, jobVertexID, evaluated, history));
assertFalse(evaluated.containsKey(ScalingMetric.EXPECTED_PROCESSING_RATE));
assertEquals(1, eventCollector.events.size());
var event = eventCollector.events.poll();
- assertEquals(String.format(INNEFFECTIVE_MESSAGE_FORMAT, jobVertexID), event.getMessage());
- assertEquals(EventRecorder.Reason.IneffectiveScaling.name(), event.getReason());
+ assertThat(event).isNotNull();
+ assertThat(event.getMessage())
+ .isEqualTo(String.format(INEFFECTIVE_MESSAGE_FORMAT, jobVertexID));
+ assertThat(event.getReason()).isEqualTo(INEFFECTIVE_SCALING);
}
private Map<ScalingMetric, EvaluatedScalingMetric> evaluated(
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/MetricsCollectionAndEvaluationTest.java
similarity index 84%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/MetricsCollectionAndEvaluationTest.java
index f8cec403..c36d6e10 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/MetricsCollectionAndEvaluationTest.java
@@ -15,26 +15,18 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.TestingEventCollector;
import org.apache.flink.autoscaler.metrics.CollectedMetricHistory;
import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
import org.apache.flink.autoscaler.metrics.FlinkMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.state.InMemoryAutoScalerStateStore;
import org.apache.flink.autoscaler.topology.JobTopology;
import org.apache.flink.autoscaler.topology.VertexInfo;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.TestingFlinkService;
-import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
-import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
-import org.apache.flink.kubernetes.operator.utils.EventCollector;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.JobDetailsInfo;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
@@ -42,9 +34,6 @@ import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -52,11 +41,11 @@ import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import static org.apache.flink.autoscaler.TestingAutoscalerUtils.createDefaultJobAutoScalerContext;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -64,37 +53,27 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Test for scaling metrics collection logic. */
-@EnableKubernetesMockClient(crud = true)
public class MetricsCollectionAndEvaluationTest {
- private final AutoScalerInfo scalingInfo = new AutoScalerInfo(new HashMap<>());
-
+ private JobAutoScalerContext<JobID> context;
private ScalingMetricEvaluator evaluator;
- private TestingFlinkService service;
- private TestingMetricsCollector metricsCollector;
- private ScalingExecutor scalingExecutor;
+ private TestingMetricsCollector<JobID, JobAutoScalerContext<JobID>> metricsCollector;
+ private ScalingExecutor<JobID, JobAutoScalerContext<JobID>> scalingExecutor;
+ private InMemoryAutoScalerStateStore<JobID, JobAutoScalerContext<JobID>> stateStore;
- private FlinkDeployment app;
- private Configuration conf;
private JobVertexID source1, source2, map, sink;
private JobTopology topology;
- private KubernetesClient kubernetesClient;
-
private Clock clock;
private Instant startTime;
@BeforeEach
public void setup() {
+ context = createDefaultJobAutoScalerContext();
evaluator = new ScalingMetricEvaluator();
- scalingExecutor = new ScalingExecutor(new EventRecorder(new EventCollector()));
- service = new TestingFlinkService();
-
- app = TestUtils.buildApplicationCluster();
- app.getMetadata().setGeneration(1L);
- app.getStatus().getJobStatus().setJobId(new JobID().toHexString());
- kubernetesClient.resource(app).createOrReplace();
+ stateStore = new InMemoryAutoScalerStateStore<>();
+ scalingExecutor = new ScalingExecutor<>(new TestingEventCollector<>(), stateStore);
source1 = new JobVertexID();
source2 = new JobVertexID();
@@ -108,26 +87,25 @@ public class MetricsCollectionAndEvaluationTest {
new VertexInfo(map, Set.of(source1, source2), 12, 720),
new VertexInfo(sink, Set.of(map), 8, 24));
- metricsCollector = new TestingMetricsCollector(topology);
+ metricsCollector = new TestingMetricsCollector<>(topology);
- var confManager = new FlinkConfigManager(new Configuration());
- conf = confManager.getDeployConfig(app.getMetadata(), app.getSpec());
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.STABILIZATION_INTERVAL, Duration.ofSeconds(10));
conf.set(AutoScalerOptions.METRICS_WINDOW, Duration.ofSeconds(100));
conf.set(AutoScalerOptions.RESTART_TIME, Duration.ZERO);
conf.set(AutoScalerOptions.SCALING_ENABLED, true);
conf.set(AutoScalerOptions.MAX_SCALE_DOWN_FACTOR, 1.);
conf.set(AutoScalerOptions.MAX_SCALE_UP_FACTOR, (double) Integer.MAX_VALUE);
- ReconciliationUtils.updateStatusForDeployedSpec(app, conf);
+
clock = Clock.fixed(Instant.ofEpochSecond(0), ZoneId.systemDefault());
metricsCollector.setClock(clock);
startTime = clock.instant();
metricsCollector.setJobUpdateTs(startTime);
- app.getStatus().getJobStatus().setState(JobStatus.RUNNING.name());
}
@Test
public void testEndToEnd() throws Exception {
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.TARGET_UTILIZATION, 1.);
conf.set(AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY, 0.);
@@ -135,14 +113,14 @@ public class MetricsCollectionAndEvaluationTest {
// We haven't left the stabilization period
// => no metrics reporting and collection should take place
- var collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ var collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertTrue(collectedMetrics.getMetricHistory().isEmpty());
// We haven't collected a full window yet, no metrics should be reported but metrics should
// still get collected.
clock = Clock.offset(clock, conf.get(AutoScalerOptions.STABILIZATION_INTERVAL));
metricsCollector.setClock(clock);
- collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(1, collectedMetrics.getMetricHistory().size());
assertFalse(collectedMetrics.isFullyCollected());
@@ -150,7 +128,7 @@ public class MetricsCollectionAndEvaluationTest {
// => no metrics should be reported but metrics should still get collected.
clock = Clock.offset(clock, Duration.ofSeconds(1));
metricsCollector.setClock(clock);
- collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(2, collectedMetrics.getMetricHistory().size());
assertFalse(collectedMetrics.isFullyCollected());
@@ -162,23 +140,23 @@ public class MetricsCollectionAndEvaluationTest {
.plus(conf.get(AutoScalerOptions.METRICS_WINDOW)),
ZoneId.systemDefault());
metricsCollector.setClock(clock);
- collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(3, collectedMetrics.getMetricHistory().size());
assertTrue(collectedMetrics.isFullyCollected());
// Test resetting the collector and make sure we can deserialize the scalingInfo correctly
- metricsCollector = new TestingMetricsCollector(topology);
+ metricsCollector = new TestingMetricsCollector<>(topology);
metricsCollector.setJobUpdateTs(startTime);
metricsCollector.setClock(clock);
setDefaultMetrics(metricsCollector);
- collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(3, collectedMetrics.getMetricHistory().size());
assertTrue(collectedMetrics.isFullyCollected());
var evaluation = evaluator.evaluate(conf, collectedMetrics);
- scalingExecutor.scaleResource(app, scalingInfo, conf, evaluation, kubernetesClient);
+ scalingExecutor.scaleResource(context, evaluation);
- var scaledParallelism = ScalingExecutorTest.getScaledParallelism(scalingInfo);
+ var scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.size());
assertEquals(2, scaledParallelism.get(source1));
@@ -190,23 +168,23 @@ public class MetricsCollectionAndEvaluationTest {
conf.set(AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY, 0.);
evaluation = evaluator.evaluate(conf, collectedMetrics);
- scalingExecutor.scaleResource(app, scalingInfo, conf, evaluation, kubernetesClient);
+ scalingExecutor.scaleResource(context, evaluation);
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(scalingInfo);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.get(source1));
assertEquals(4, scaledParallelism.get(source2));
assertEquals(12, scaledParallelism.get(map));
assertEquals(8, scaledParallelism.get(sink));
- var resourceID = ResourceID.fromResource(app);
- assertNotNull(metricsCollector.getHistories().get(resourceID));
+ assertNotNull(metricsCollector.getHistories().get(context.getJobKey()));
- metricsCollector.cleanup(app);
- assertNull(metricsCollector.getHistories().get(resourceID));
- assertNull(metricsCollector.getAvailableVertexMetricNames().get(resourceID));
+ metricsCollector.cleanup(context.getJobKey());
+ assertNull(metricsCollector.getHistories().get(context.getJobKey()));
+ assertNull(metricsCollector.getAvailableVertexMetricNames().get(context.getJobKey()));
}
- private void setDefaultMetrics(TestingMetricsCollector metricsCollector) {
+ private void setDefaultMetrics(
+ TestingMetricsCollector<JobID, JobAutoScalerContext<JobID>> metricsCollector) {
metricsCollector.setCurrentMetrics(
Map.of(
source1,
@@ -249,15 +227,13 @@ public class MetricsCollectionAndEvaluationTest {
@Test
public void testKafkaPartitionMaxParallelism() throws Exception {
- var scalingInfo = new AutoScalerInfo(new HashMap<>());
-
setDefaultMetrics(metricsCollector);
- metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsCollector.updateMetrics(context, stateStore);
var clock = Clock.fixed(Instant.now().plus(Duration.ofSeconds(3)), ZoneId.systemDefault());
metricsCollector.setClock(clock);
- var collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ var collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(720, collectedMetrics.getJobTopology().getMaxParallelisms().get(source1));
assertEquals(720, collectedMetrics.getJobTopology().getMaxParallelisms().get(source2));
@@ -282,7 +258,7 @@ public class MetricsCollectionAndEvaluationTest {
new AggregatedMetric(
"1.Source__Kafka_Source_(testTopic).KafkaSourceReader.topic.testTopic.partition.3.currentOffset"))));
- collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(5, collectedMetrics.getJobTopology().getMaxParallelisms().get(source1));
assertEquals(720, collectedMetrics.getJobTopology().getMaxParallelisms().get(source2));
}
@@ -302,56 +278,58 @@ public class MetricsCollectionAndEvaluationTest {
@Test
public void testMetricCollectorWindow() throws Exception {
setDefaultMetrics(metricsCollector);
- var metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ var metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(0, metricsHistory.getMetricHistory().size());
// Not stable, nothing should be collected
metricsCollector.setClock(Clock.offset(clock, Duration.ofSeconds(1)));
- metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(0, metricsHistory.getMetricHistory().size());
// Update clock to stable time
+ var conf = context.getConfiguration();
clock = Clock.offset(clock, conf.get(AutoScalerOptions.STABILIZATION_INTERVAL));
metricsCollector.setClock(clock);
// This call will lead to metric collection but we haven't reached the window size yet
// which will hold back metrics
- metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(1, metricsHistory.getMetricHistory().size());
assertFalse(metricsHistory.isFullyCollected());
// Collect more values in window
metricsCollector.setClock(Clock.offset(clock, Duration.ofSeconds(1)));
- metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(2, metricsHistory.getMetricHistory().size());
assertFalse(metricsHistory.isFullyCollected());
// Window size reached
metricsCollector.setClock(Clock.offset(clock, conf.get(AutoScalerOptions.METRICS_WINDOW)));
- metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(3, metricsHistory.getMetricHistory().size());
assertTrue(metricsHistory.isFullyCollected());
// Window size + 1 will invalidate the first metric
metricsCollector.setClock(
Clock.offset(clock, conf.get(AutoScalerOptions.METRICS_WINDOW).plusSeconds(1)));
- metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(3, metricsHistory.getMetricHistory().size());
// Complete new metric window with just the currently collected metric
metricsCollector.setClock(
Clock.offset(clock, conf.get(AutoScalerOptions.METRICS_WINDOW).plusDays(1)));
- metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(1, metricsHistory.getMetricHistory().size());
// Existing metrics should be cleared on job updates
metricsCollector.setJobUpdateTs(clock.instant().plus(Duration.ofDays(10)));
- metricsHistory = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ metricsHistory = metricsCollector.updateMetrics(context, stateStore);
assertEquals(0, metricsHistory.getMetricHistory().size());
}
@Test
public void testClearHistoryOnTopoChange() throws Exception {
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.TARGET_UTILIZATION, 1.);
conf.set(AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY, 0.);
@@ -359,7 +337,7 @@ public class MetricsCollectionAndEvaluationTest {
// We haven't left the stabilization period
// => no metrics reporting and collection should take place
- var collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ var collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertTrue(collectedMetrics.getMetricHistory().isEmpty());
}
@@ -385,7 +363,7 @@ public class MetricsCollectionAndEvaluationTest {
var collectedMetrics = collectMetrics();
Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluation =
- evaluator.evaluate(conf, collectedMetrics);
+ evaluator.evaluate(context.getConfiguration(), collectedMetrics);
assertEquals(
500., evaluation.get(source1).get(ScalingMetric.TARGET_DATA_RATE).getCurrent());
assertEquals(
@@ -398,8 +376,8 @@ public class MetricsCollectionAndEvaluationTest {
500.,
evaluation.get(source1).get(ScalingMetric.SCALE_UP_RATE_THRESHOLD).getCurrent());
- scalingExecutor.scaleResource(app, scalingInfo, conf, evaluation, kubernetesClient);
- var scaledParallelism = ScalingExecutorTest.getScaledParallelism(scalingInfo);
+ scalingExecutor.scaleResource(context, evaluation);
+ var scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(1, scaledParallelism.get(source1));
}
@@ -449,7 +427,7 @@ public class MetricsCollectionAndEvaluationTest {
public void testScaleDownWithZeroProcessingRate() throws Exception {
var topology = new JobTopology(new VertexInfo(source1, Set.of(), 10, 720));
- metricsCollector = new TestingMetricsCollector(topology);
+ metricsCollector = new TestingMetricsCollector<>(topology);
metricsCollector.setJobUpdateTs(startTime);
metricsCollector.setCurrentMetrics(
Map.of(
@@ -466,7 +444,7 @@ public class MetricsCollectionAndEvaluationTest {
var collectedMetrics = collectMetrics();
Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluation =
- evaluator.evaluate(conf, collectedMetrics);
+ evaluator.evaluate(context.getConfiguration(), collectedMetrics);
assertEquals(0, evaluation.get(source1).get(ScalingMetric.TARGET_DATA_RATE).getCurrent());
assertEquals(
Double.POSITIVE_INFINITY,
@@ -478,24 +456,25 @@ public class MetricsCollectionAndEvaluationTest {
0.,
evaluation.get(source1).get(ScalingMetric.SCALE_UP_RATE_THRESHOLD).getCurrent());
- scalingExecutor.scaleResource(app, scalingInfo, conf, evaluation, kubernetesClient);
- var scaledParallelism = ScalingExecutorTest.getScaledParallelism(scalingInfo);
+ scalingExecutor.scaleResource(context, evaluation);
+ var scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(1, scaledParallelism.get(source1));
}
private CollectedMetricHistory collectMetrics() throws Exception {
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.STABILIZATION_INTERVAL, Duration.ZERO);
conf.set(AutoScalerOptions.METRICS_WINDOW, Duration.ofSeconds(2));
metricsCollector.setClock(clock);
- var collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ var collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(1, collectedMetrics.getMetricHistory().size());
assertFalse(collectedMetrics.isFullyCollected());
metricsCollector.setClock(Clock.offset(clock, Duration.ofSeconds(2)));
- collectedMetrics = metricsCollector.updateMetrics(app, scalingInfo, service, conf);
+ collectedMetrics = metricsCollector.updateMetrics(context, stateStore);
assertEquals(2, collectedMetrics.getMetricHistory().size());
assertTrue(collectedMetrics.isFullyCollected());
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RecommendedParallelismTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/RecommendedParallelismTest.java
similarity index 63%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RecommendedParallelismTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/RecommendedParallelismTest.java
index e11d76e9..da41c015 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RecommendedParallelismTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/RecommendedParallelismTest.java
@@ -15,33 +15,22 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.TestingEventCollector;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
import org.apache.flink.autoscaler.metrics.FlinkMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.realizer.TestingScalingRealizer;
+import org.apache.flink.autoscaler.state.TestingAutoscalerStateStore;
import org.apache.flink.autoscaler.topology.JobTopology;
import org.apache.flink.autoscaler.topology.VertexInfo;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.OperatorTestBase;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
-import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
-import org.apache.flink.kubernetes.operator.utils.EventCollector;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
-import io.fabric8.kubernetes.api.model.HasMetadata;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import lombok.Getter;
-import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -50,53 +39,52 @@ import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.SortedMap;
+import static org.apache.flink.autoscaler.TestingAutoscalerUtils.createDefaultJobAutoScalerContext;
+import static org.apache.flink.autoscaler.TestingAutoscalerUtils.getRestClusterClientSupplier;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
import static org.apache.flink.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerTestUtils.getOrCreateInfo;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Test for recommended parallelism. */
-@EnableKubernetesMockClient(crud = true)
-public class RecommendedParallelismTest extends OperatorTestBase {
+public class RecommendedParallelismTest {
- @Getter private KubernetesClient kubernetesClient;
+ private JobAutoScalerContext<JobID> context;
+ private TestingAutoscalerStateStore<JobID, JobAutoScalerContext<JobID>> stateStore;
- private ScalingMetricEvaluator evaluator;
- private TestingMetricsCollector metricsCollector;
- private ScalingExecutor scalingExecutor;
+ private TestingMetricsCollector<JobID, JobAutoScalerContext<JobID>> metricsCollector;
+ private ScalingExecutor<JobID, JobAutoScalerContext<JobID>> scalingExecutor;
- private FlinkDeployment app;
private JobVertexID source, sink;
- private JobAutoScalerImpl autoscaler;
-
- private EventCollector eventCollector = new EventCollector();
+ private JobAutoScalerImpl<JobID, JobAutoScalerContext<JobID>> autoscaler;
@BeforeEach
public void setup() {
- evaluator = new ScalingMetricEvaluator();
- scalingExecutor = new ScalingExecutor(new EventRecorder(new EventCollector()));
+ context = createDefaultJobAutoScalerContext();
+
+ TestingEventCollector<JobID, JobAutoScalerContext<JobID>> eventCollector =
+ new TestingEventCollector<>();
+ stateStore = new TestingAutoscalerStateStore<>();
- app = TestUtils.buildApplicationCluster();
- app.getMetadata().setGeneration(1L);
- app.getStatus().getJobStatus().setJobId(new JobID().toHexString());
- kubernetesClient.resource(app).createOrReplace();
+ scalingExecutor = new ScalingExecutor<>(eventCollector, stateStore);
source = new JobVertexID();
sink = new JobVertexID();
metricsCollector =
- new TestingMetricsCollector(
+ new TestingMetricsCollector<>(
new JobTopology(
new VertexInfo(source, Set.of(), 1, 720),
new VertexInfo(sink, Set.of(source), 1, 720)));
- var defaultConf = new Configuration();
+ var defaultConf = context.getConfiguration();
defaultConf.set(AutoScalerOptions.AUTOSCALER_ENABLED, true);
defaultConf.set(AutoScalerOptions.STABILIZATION_INTERVAL, Duration.ZERO);
defaultConf.set(AutoScalerOptions.RESTART_TIME, Duration.ofSeconds(1));
@@ -108,32 +96,27 @@ public class RecommendedParallelismTest extends OperatorTestBase {
defaultConf.set(AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY, 0.1);
defaultConf.set(AutoScalerOptions.SCALE_UP_GRACE_PERIOD, Duration.ZERO);
- configManager = new FlinkConfigManager(defaultConf);
- ReconciliationUtils.updateStatusForDeployedSpec(
- app, configManager.getDeployConfig(app.getMetadata(), app.getSpec()));
- app.getStatus().getJobStatus().setState(JobStatus.RUNNING.name());
- app.getStatus().getReconciliationStatus().markReconciledSpecAsStable();
-
autoscaler =
- new JobAutoScalerImpl(
+ new JobAutoScalerImpl<>(
metricsCollector,
- evaluator,
+ new ScalingMetricEvaluator(),
scalingExecutor,
- new EventRecorder(eventCollector));
+ eventCollector,
+ new TestingScalingRealizer<>(),
+ stateStore);
// Reset custom window size to default
metricsCollector.setTestMetricWindowSize(null);
}
@Test
- public void endToEnd() {
+ public void endToEnd() throws Exception {
// we start the autoscaler in advisor mode
- app.getSpec().getFlinkConfiguration().put(AutoScalerOptions.SCALING_ENABLED.key(), "false");
- var ctx = createAutoscalerTestContext();
+ context.getConfiguration().setString(AutoScalerOptions.SCALING_ENABLED.key(), "false");
// initially the last evaluated metrics are empty
- assertNull(autoscaler.lastEvaluatedMetrics.get(ResourceID.fromResource(app)));
+ assertNull(autoscaler.lastEvaluatedMetrics.get(context.getJobKey()));
var now = Instant.ofEpochMilli(0);
setClocksTo(now);
@@ -162,51 +145,51 @@ public class RecommendedParallelismTest extends OperatorTestBase {
"", Double.NaN, Double.NaN, Double.NaN, 500.))));
// the recommended parallelism values are empty initially
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(1, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
+ autoscaler.scale(context);
+ assertEvaluatedMetricsSize(1);
assertNull(getCurrentMetricValue(source, RECOMMENDED_PARALLELISM));
assertNull(getCurrentMetricValue(sink, RECOMMENDED_PARALLELISM));
assertEquals(1., getCurrentMetricValue(source, PARALLELISM));
assertEquals(1., getCurrentMetricValue(sink, PARALLELISM));
// the auto scaler is running in recommendation mode
- assertTrue(ScalingExecutorTest.getScaledParallelism(kubernetesClient, app).isEmpty());
+ assertTrue(ScalingExecutorTest.getScaledParallelism(stateStore, context).isEmpty());
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
// it stays empty until the metric window is full
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(2, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
+ autoscaler.scale(context);
+ assertEvaluatedMetricsSize(2);
assertNull(getCurrentMetricValue(source, RECOMMENDED_PARALLELISM));
assertNull(getCurrentMetricValue(sink, RECOMMENDED_PARALLELISM));
assertEquals(1., getCurrentMetricValue(source, PARALLELISM));
assertEquals(1., getCurrentMetricValue(sink, PARALLELISM));
- assertTrue(ScalingExecutorTest.getScaledParallelism(kubernetesClient, app).isEmpty());
+ assertTrue(ScalingExecutorTest.getScaledParallelism(stateStore, context).isEmpty());
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
// then the recommended parallelism can change according to the evaluated metrics
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(3, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
+ autoscaler.scale(context);
+ assertEvaluatedMetricsSize(3);
assertEquals(1., getCurrentMetricValue(source, PARALLELISM));
assertEquals(1., getCurrentMetricValue(sink, PARALLELISM));
assertEquals(4., getCurrentMetricValue(source, RECOMMENDED_PARALLELISM));
assertEquals(4., getCurrentMetricValue(sink, RECOMMENDED_PARALLELISM));
- assertTrue(ScalingExecutorTest.getScaledParallelism(kubernetesClient, app).isEmpty());
+ assertTrue(ScalingExecutorTest.getScaledParallelism(stateStore, context).isEmpty());
// once scaling is enabled
- app.getSpec().getFlinkConfiguration().put(AutoScalerOptions.SCALING_ENABLED.key(), "true");
+ context.getConfiguration().setString(AutoScalerOptions.SCALING_ENABLED.key(), "true");
now = now.plus(Duration.ofSeconds(1));
setClocksTo(now);
// the scaled parallelism will pick up the recommended parallelism values
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(3, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
+ autoscaler.scale(context);
+ assertEvaluatedMetricsSize(3);
assertEquals(4., getCurrentMetricValue(source, RECOMMENDED_PARALLELISM));
assertEquals(4., getCurrentMetricValue(sink, RECOMMENDED_PARALLELISM));
- var scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ var scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.get(source));
assertEquals(4, scaledParallelism.get(sink));
@@ -221,10 +204,10 @@ public class RecommendedParallelismTest extends OperatorTestBase {
restart(now);
// after restart while the job is not running the evaluated metrics are gone
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(3, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
- assertNull(autoscaler.lastEvaluatedMetrics.get(ResourceID.fromResource(app)));
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ autoscaler.scale(context);
+ assertEquals(3, stateStore.getEvaluatedMetrics(context).get().size());
+ assertNull(autoscaler.lastEvaluatedMetrics.get(context.getJobKey()));
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.get(source));
assertEquals(4, scaledParallelism.get(sink));
@@ -234,22 +217,29 @@ public class RecommendedParallelismTest extends OperatorTestBase {
// once the job is running we got back the evaluated metric except the recommended
// parallelisms (until the metric window is full again)
- autoscaler.scale(getResourceContext(app, ctx));
- assertEquals(1, getOrCreateInfo(app, kubernetesClient).getMetricHistory().size());
+ autoscaler.scale(context);
+ assertEvaluatedMetricsSize(1);
assertEquals(4., getCurrentMetricValue(source, PARALLELISM));
assertEquals(4., getCurrentMetricValue(sink, PARALLELISM));
assertNull(getCurrentMetricValue(source, RECOMMENDED_PARALLELISM));
assertNull(getCurrentMetricValue(sink, RECOMMENDED_PARALLELISM));
- scaledParallelism = ScalingExecutorTest.getScaledParallelism(kubernetesClient, app);
+ scaledParallelism = ScalingExecutorTest.getScaledParallelism(stateStore, context);
assertEquals(4, scaledParallelism.get(source));
assertEquals(4, scaledParallelism.get(sink));
}
+ private void assertEvaluatedMetricsSize(int expectedSize) {
+ Optional<SortedMap<Instant, CollectedMetrics>> evaluatedMetricsOpt =
+ stateStore.getEvaluatedMetrics(context);
+ assertThat(evaluatedMetricsOpt).isPresent();
+ assertThat(evaluatedMetricsOpt.get()).hasSize(expectedSize);
+ }
+
private Double getCurrentMetricValue(JobVertexID jobVertexID, ScalingMetric scalingMetric) {
var metric =
autoscaler
.lastEvaluatedMetrics
- .get(ResourceID.fromResource(app))
+ .get(context.getJobKey())
.get(jobVertexID)
.get(scalingMetric);
return metric == null ? null : metric.getCurrent();
@@ -257,12 +247,26 @@ public class RecommendedParallelismTest extends OperatorTestBase {
private void restart(Instant now) {
metricsCollector.setJobUpdateTs(now);
- app.getStatus().getJobStatus().setState(JobStatus.CREATED.name());
+ context =
+ new JobAutoScalerContext<>(
+ context.getJobKey(),
+ context.getJobID(),
+ JobStatus.CREATED,
+ context.getConfiguration(),
+ context.getMetricGroup(),
+ getRestClusterClientSupplier());
}
private void running(Instant now) {
metricsCollector.setJobUpdateTs(now);
- app.getStatus().getJobStatus().setState(JobStatus.RUNNING.name());
+ context =
+ new JobAutoScalerContext<>(
+ context.getJobKey(),
+ context.getJobID(),
+ JobStatus.RUNNING,
+ context.getConfiguration(),
+ context.getMetricGroup(),
+ getRestClusterClientSupplier());
}
private void setClocksTo(Instant time) {
@@ -270,21 +274,4 @@ public class RecommendedParallelismTest extends OperatorTestBase {
metricsCollector.setClock(clock);
scalingExecutor.setClock(clock);
}
-
- @NotNull
- private TestUtils.TestingContext<HasMetadata> createAutoscalerTestContext() {
- return new TestUtils.TestingContext<>() {
- @Override
- public <T1> Set<T1> getSecondaryResources(Class<T1> aClass) {
- return (Set)
- kubernetesClient.configMaps().inAnyNamespace().list().getItems().stream()
- .collect(Collectors.toSet());
- }
-
- @Override
- public KubernetesClient getClient() {
- return kubernetesClient;
- }
- };
- }
}
diff --git a/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/RestApiMetricsCollectorTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/RestApiMetricsCollectorTest.java
new file mode 100644
index 00000000..128c4094
--- /dev/null
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/RestApiMetricsCollectorTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.client.program.rest.RestClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneClientHAServices;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.rest.messages.EmptyResponseBody;
+import org.apache.flink.runtime.rest.messages.MessageHeaders;
+import org.apache.flink.runtime.rest.messages.MessageParameters;
+import org.apache.flink.runtime.rest.messages.RequestBody;
+import org.apache.flink.runtime.rest.messages.ResponseBody;
+import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
+import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetricsResponseBody;
+import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedSubtaskMetricsHeaders;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+/** Tests for {@link RestApiMetricsCollector}. */
+public class RestApiMetricsCollectorTest {
+
+ @Test
+ public void testAggregateMultiplePendingRecordsMetricsPerSource() throws Exception {
+ RestApiMetricsCollector<JobID, JobAutoScalerContext<JobID>> collector =
+ new RestApiMetricsCollector<>();
+
+ JobVertexID jobVertexID = new JobVertexID();
+ Map<String, FlinkMetric> flinkMetrics =
+ Map.of(
+ "a.pendingRecords", FlinkMetric.PENDING_RECORDS,
+ "b.pendingRecords", FlinkMetric.PENDING_RECORDS);
+ Map<JobVertexID, Map<String, FlinkMetric>> metrics = Map.of(jobVertexID, flinkMetrics);
+
+ List<AggregatedMetric> aggregatedMetricsResponse =
+ List.of(
+ new AggregatedMetric(
+ "a.pendingRecords", Double.NaN, Double.NaN, Double.NaN, 100.),
+ new AggregatedMetric(
+ "b.pendingRecords", Double.NaN, Double.NaN, Double.NaN, 100.),
+ new AggregatedMetric(
+ "c.unrelated", Double.NaN, Double.NaN, Double.NaN, 100.));
+
+ Configuration conf = new Configuration();
+ RestClusterClient<String> restClusterClient =
+ new RestClusterClient<>(
+ conf,
+ "test-cluster",
+ (c, e) -> new StandaloneClientHAServices("localhost")) {
+ @Override
+ public <
+ M extends MessageHeaders<R, P, U>,
+ U extends MessageParameters,
+ R extends RequestBody,
+ P extends ResponseBody>
+ CompletableFuture<P> sendRequest(
+ M messageHeaders, U messageParameters, R request) {
+ if (messageHeaders instanceof AggregatedSubtaskMetricsHeaders) {
+ return (CompletableFuture<P>)
+ CompletableFuture.completedFuture(
+ new AggregatedMetricsResponseBody(
+ aggregatedMetricsResponse));
+ }
+ return (CompletableFuture<P>)
+ CompletableFuture.completedFuture(EmptyResponseBody.getInstance());
+ }
+ };
+
+ JobID jobID = new JobID();
+ JobAutoScalerContext<JobID> context =
+ new JobAutoScalerContext<>(
+ jobID,
+ jobID,
+ JobStatus.RUNNING,
+ conf,
+ new UnregisteredMetricsGroup(),
+ () -> restClusterClient);
+
+ Map<JobVertexID, Map<FlinkMetric, AggregatedMetric>> jobVertexIDMapMap =
+ collector.queryAllAggregatedMetrics(context, metrics);
+
+ Assertions.assertEquals(1, jobVertexIDMapMap.size());
+ Map<FlinkMetric, AggregatedMetric> vertexMetrics = jobVertexIDMapMap.get(jobVertexID);
+ Assertions.assertNotNull(vertexMetrics);
+ AggregatedMetric pendingRecordsMetric = vertexMetrics.get(FlinkMetric.PENDING_RECORDS);
+ Assertions.assertNotNull(pendingRecordsMetric);
+ Assertions.assertEquals(pendingRecordsMetric.getSum(), 200);
+ }
+}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingExecutorTest.java
similarity index 63%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingExecutorTest.java
index d66d10e8..36fb6e31 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingExecutorTest.java
@@ -15,24 +15,17 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
-import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
-import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.event.TestingEventCollector;
import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
import org.apache.flink.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.utils.EventCollector;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+import org.apache.flink.autoscaler.state.AutoScalerStateStore;
+import org.apache.flink.autoscaler.state.InMemoryAutoScalerStateStore;
import org.apache.flink.runtime.jobgraph.JobVertexID;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -44,53 +37,44 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerTestUtils.getOrCreateInfo;
-import static org.apache.flink.kubernetes.operator.autoscaler.ScalingExecutor.SCALING_SUMMARY_ENTRY;
-import static org.apache.flink.kubernetes.operator.autoscaler.ScalingExecutor.SCALING_SUMMARY_HEADER_SCALING_DISABLED;
-import static org.apache.flink.kubernetes.operator.autoscaler.ScalingExecutor.SCALING_SUMMARY_HEADER_SCALING_ENABLED;
+import static org.apache.flink.autoscaler.ScalingExecutor.SCALING_SUMMARY_ENTRY;
+import static org.apache.flink.autoscaler.TestingAutoscalerUtils.createDefaultJobAutoScalerContext;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
-/** Test for scaling execution logic. */
-@EnableKubernetesMockClient(crud = true)
+/** Test for {@link ScalingExecutor}. */
public class ScalingExecutorTest {
- private ScalingExecutor scalingDecisionExecutor;
+ private JobAutoScalerContext<JobID> context;
+ private TestingEventCollector<JobID, JobAutoScalerContext<JobID>> eventCollector;
+ private ScalingExecutor<JobID, JobAutoScalerContext<JobID>> scalingDecisionExecutor;
- private EventCollector eventCollector;
- private Configuration conf;
- private KubernetesClient kubernetesClient;
- private FlinkDeployment flinkDep;
+ private InMemoryAutoScalerStateStore<JobID, JobAutoScalerContext<JobID>> stateStore;
@BeforeEach
public void setup() {
- eventCollector = new EventCollector();
- scalingDecisionExecutor = new ScalingExecutor(new EventRecorder(eventCollector));
- conf = new Configuration();
+ eventCollector = new TestingEventCollector<>();
+ context = createDefaultJobAutoScalerContext();
+ stateStore = new InMemoryAutoScalerStateStore<>();
+
+ scalingDecisionExecutor = new ScalingExecutor<>(eventCollector, stateStore);
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.STABILIZATION_INTERVAL, Duration.ZERO);
conf.set(AutoScalerOptions.SCALING_ENABLED, true);
conf.set(AutoScalerOptions.MAX_SCALE_DOWN_FACTOR, 1.);
conf.set(AutoScalerOptions.MAX_SCALE_UP_FACTOR, (double) Integer.MAX_VALUE);
conf.set(AutoScalerOptions.CATCH_UP_DURATION, Duration.ZERO);
-
- flinkDep = TestUtils.buildApplicationCluster();
- kubernetesClient.resource(flinkDep).createOrReplace();
- var jobStatus = flinkDep.getStatus().getJobStatus();
- jobStatus.setStartTime(String.valueOf(System.currentTimeMillis()));
- jobStatus.setUpdateTime(String.valueOf(System.currentTimeMillis()));
- jobStatus.setState(JobStatus.RUNNING.name());
}
@Test
- public void testUtilizationBoundaries() {
+ public void testUtilizationBoundaries() throws Exception {
// Restart time should not affect utilization boundary
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.RESTART_TIME, Duration.ZERO);
conf.set(AutoScalerOptions.CATCH_UP_DURATION, Duration.ZERO);
- var flinkDep = TestUtils.buildApplicationCluster();
- kubernetesClient.resource(flinkDep).createOrReplace();
-
var op1 = new JobVertexID();
conf.set(AutoScalerOptions.TARGET_UTILIZATION, 0.6);
@@ -104,7 +88,7 @@ public class ScalingExecutorTest {
evaluated = Map.of(op1, evaluated(1, 70, 100));
scalingSummary = Map.of(op1, new ScalingSummary(2, 1, evaluated.get(op1)));
assertTrue(ScalingExecutor.allVerticesWithinUtilizationTarget(evaluated, scalingSummary));
- assertTrue(getScaledParallelism(kubernetesClient, flinkDep).isEmpty());
+ assertTrue(getScaledParallelism(stateStore, context).isEmpty());
var op2 = new JobVertexID();
evaluated =
@@ -139,7 +123,7 @@ public class ScalingExecutorTest {
}
@Test
- public void testVertexesExclusionForScaling() {
+ public void testVertexesExclusionForScaling() throws Exception {
var sourceHexString = "0bfd135746ac8efb3cce668b12e16d3a";
var source = JobVertexID.fromHexString(sourceHexString);
var filterOperatorHexString = "869fb403873411306404e9f2e4438c0e";
@@ -147,7 +131,7 @@ public class ScalingExecutorTest {
var sinkHexString = "a6b7102b8d3e3a9564998c1ffeb5e2b7";
var sink = JobVertexID.fromHexString(sinkHexString);
- var scalingInfo = new AutoScalerInfo(new HashMap<>());
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.TARGET_UTILIZATION, .8);
var metrics =
Map.of(
@@ -159,36 +143,33 @@ public class ScalingExecutorTest {
evaluated(10, 80, 100));
// filter operator should not scale
conf.set(AutoScalerOptions.VERTEX_EXCLUDE_IDS, List.of(filterOperatorHexString));
- assertFalse(
- scalingDecisionExecutor.scaleResource(
- flinkDep, scalingInfo, conf, metrics, kubernetesClient));
+ assertFalse(scalingDecisionExecutor.scaleResource(context, metrics));
// filter operator should scale
conf.set(AutoScalerOptions.VERTEX_EXCLUDE_IDS, List.of());
- assertTrue(
- scalingDecisionExecutor.scaleResource(
- flinkDep, scalingInfo, conf, metrics, kubernetesClient));
+ assertTrue(scalingDecisionExecutor.scaleResource(context, metrics));
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
- public void testScalingEventsWith0Interval(boolean scalingEnabled) {
+ public void testScalingEventsWith0Interval(boolean scalingEnabled) throws Exception {
testScalingEvents(scalingEnabled, Duration.ofSeconds(0));
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
- public void testScalingEventsWithInterval(boolean scalingEnabled) {
+ public void testScalingEventsWithInterval(boolean scalingEnabled) throws Exception {
testScalingEvents(scalingEnabled, Duration.ofSeconds(1800));
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
- public void testScalingEventsWithDefaultInterval(boolean scalingEnabled) {
+ public void testScalingEventsWithDefaultInterval(boolean scalingEnabled) throws Exception {
testScalingEvents(scalingEnabled, null);
}
- private void testScalingEvents(boolean scalingEnabled, Duration interval) {
+ private void testScalingEvents(boolean scalingEnabled, Duration interval) throws Exception {
var jobVertexID = new JobVertexID();
+ var conf = context.getConfiguration();
conf.set(AutoScalerOptions.SCALING_ENABLED, scalingEnabled);
var metrics = Map.of(jobVertexID, evaluated(1, 110, 100));
@@ -197,22 +178,21 @@ public class ScalingExecutorTest {
conf.set(AutoScalerOptions.SCALING_REPORT_INTERVAL, interval);
}
- var scalingInfo = new AutoScalerInfo(new HashMap<>());
- assertEquals(
- scalingEnabled,
- scalingDecisionExecutor.scaleResource(
- flinkDep, scalingInfo, conf, metrics, kubernetesClient));
- assertEquals(
- scalingEnabled,
- scalingDecisionExecutor.scaleResource(
- flinkDep, scalingInfo, conf, metrics, kubernetesClient));
- assertEquals(
+ assertEquals(scalingEnabled, scalingDecisionExecutor.scaleResource(context, metrics));
+ assertEquals(scalingEnabled, scalingDecisionExecutor.scaleResource(context, metrics));
+
+ int expectedSize =
(interval == null || (!interval.isNegative() && !interval.isZero()))
&& !scalingEnabled
? 1
- : 2,
- eventCollector.events.size());
- var event = eventCollector.events.poll();
+ : 2;
+ assertEquals(expectedSize, eventCollector.events.size());
+
+ TestingEventCollector.Event<JobID, JobAutoScalerContext<JobID>> event;
+ do {
+ event = eventCollector.events.poll();
+ } while (!eventCollector.events.isEmpty());
+
assertTrue(
event.getMessage()
.contains(
@@ -228,22 +208,20 @@ public class ScalingExecutorTest {
event.getMessage()
.contains(
scalingEnabled
- ? SCALING_SUMMARY_HEADER_SCALING_ENABLED
- : SCALING_SUMMARY_HEADER_SCALING_DISABLED));
- assertEquals(EventRecorder.Reason.ScalingReport.name(), event.getReason());
+ ? ScalingExecutor.SCALING_SUMMARY_HEADER_SCALING_ENABLED
+ : ScalingExecutor.SCALING_SUMMARY_HEADER_SCALING_DISABLED));
+ assertEquals(ScalingExecutor.SCALING_REPORT_REASON, event.getReason());
metrics = Map.of(jobVertexID, evaluated(1, 110, 101));
- assertEquals(1, event.getCount());
+ assertEquals(expectedSize, event.getCount());
- assertEquals(
- scalingEnabled,
- scalingDecisionExecutor.scaleResource(
- flinkDep, scalingInfo, conf, metrics, kubernetesClient));
+ assertEquals(scalingEnabled, scalingDecisionExecutor.scaleResource(context, metrics));
var event2 = eventCollector.events.poll();
- assertEquals(event.getMetadata().getUid(), event2.getMetadata().getUid());
- assertEquals(2, event2.getCount());
- assertEquals(!scalingEnabled, scalingInfo.getCurrentOverrides().isEmpty());
+ assertThat(event2).isNotNull();
+ assertThat(event2.getContext()).isSameAs(event.getContext());
+ assertEquals(expectedSize + 1, event2.getCount());
+ assertEquals(!scalingEnabled, stateStore.getParallelismOverrides(context).isEmpty());
}
private Map<ScalingMetric, EvaluatedScalingMetric> evaluated(
@@ -255,7 +233,8 @@ public class ScalingExecutorTest {
metrics.put(ScalingMetric.CATCH_UP_DATA_RATE, EvaluatedScalingMetric.of(catchupRate));
metrics.put(
ScalingMetric.TRUE_PROCESSING_RATE, new EvaluatedScalingMetric(procRate, procRate));
- ScalingMetricEvaluator.computeProcessingRateThresholds(metrics, conf, false);
+ ScalingMetricEvaluator.computeProcessingRateThresholds(
+ metrics, context.getConfiguration(), false);
return metrics;
}
@@ -264,16 +243,19 @@ public class ScalingExecutorTest {
return evaluated(parallelism, target, procRate, 0.);
}
- protected static Map<JobVertexID, Integer> getScaledParallelism(
- KubernetesClient client, AbstractFlinkResource<?, ?> cr) {
- return getScaledParallelism(getOrCreateInfo(cr, client));
- }
-
- protected static Map<JobVertexID, Integer> getScaledParallelism(AutoScalerInfo info) {
- return info.getCurrentOverrides().entrySet().stream()
- .collect(
- Collectors.toMap(
- e -> JobVertexID.fromHexString(e.getKey()),
- e -> Integer.valueOf(e.getValue())));
+ protected static <KEY, Context extends JobAutoScalerContext<KEY>>
+ Map<JobVertexID, Integer> getScaledParallelism(
+ AutoScalerStateStore<KEY, Context> stateStore, Context context)
+ throws Exception {
+ return stateStore
+ .getParallelismOverrides(context)
+ .map(
+ overrides ->
+ overrides.entrySet().stream()
+ .collect(
+ Collectors.toMap(
+ e -> JobVertexID.fromHexString(e.getKey()),
+ e -> Integer.valueOf(e.getValue()))))
+ .orElse(Map.of());
}
}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollectorTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingMetricCollectorTest.java
similarity index 85%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollectorTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingMetricCollectorTest.java
index d6c8f14f..57fe3de9 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollectorTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingMetricCollectorTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
@@ -23,10 +23,6 @@ import org.apache.flink.autoscaler.metrics.FlinkMetric;
import org.apache.flink.autoscaler.topology.JobTopology;
import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.client.program.rest.RestClusterClient;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.TestingFlinkService;
-import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.JobPlanInfo;
import org.apache.flink.runtime.rest.messages.job.JobDetailsInfo;
@@ -34,6 +30,7 @@ import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
@@ -45,6 +42,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.flink.autoscaler.TestingAutoscalerUtils.createDefaultJobAutoScalerContext;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -52,13 +50,20 @@ import static org.junit.jupiter.api.Assertions.fail;
/** Tests for {@link ScalingMetricCollector}. */
public class ScalingMetricCollectorTest {
+ private JobAutoScalerContext<JobID> context;
+
+ @BeforeEach
+ public void setup() {
+ context = createDefaultJobAutoScalerContext();
+ }
+
@Test
public void testJobTopologyParsingFromJobDetails() throws Exception {
String s =
"{\"jid\":\"8b6cdb9a1db8876d3dd803d5e6108ae3\",\"name\":\"State machine job\",\"isStoppable\":false,\"state\":\"RUNNING\",\"start-time\":1686216314565,\"end-time\":-1,\"duration\":25867,\"maxParallelism\":-1,\"now\":1686216340432,\"timestamps\":{\"INITIALIZING\":1686216314565,\"RECONCILING\":0,\"CANCELLING\":0,\"FAILING\":0,\"CREATED\":1686216314697,\"SUSPENDED\":0,\"RUNNING\":1686216314900,\"FAILED\":0,\"CANCELED\":0,\"FINISHED\":0,\"RESTARTING\":0},\"vertices\":[{\"id\" [...]
JobDetailsInfo jobDetailsInfo = new ObjectMapper().readValue(s, JobDetailsInfo.class);
- var metricsCollector = new RestApiMetricsCollector();
+ var metricsCollector = new RestApiMetricsCollector<>();
metricsCollector.getJobTopology(jobDetailsInfo);
}
@@ -79,15 +84,15 @@ public class ScalingMetricCollectorTest {
List.of(),
Map.of(),
new JobPlanInfo.RawJson(""));
- var metricsCollector = new RestApiMetricsCollector();
+ var metricsCollector = new RestApiMetricsCollector<>();
assertEquals(Instant.ofEpochMilli(3L), metricsCollector.getJobUpdateTs(details));
}
@Test
public void testQueryNamesOnTopologyChange() {
var metricNameQueryCounter = new AtomicInteger(0);
- var collector =
- new RestApiMetricsCollector() {
+ RestApiMetricsCollector<JobID, JobAutoScalerContext<JobID>> collector =
+ new RestApiMetricsCollector<>() {
@Override
protected Map<String, FlinkMetric> getFilteredVertexMetricNames(
RestClusterClient<?> rc, JobID id, JobVertexID jvi, JobTopology t) {
@@ -96,32 +101,26 @@ public class ScalingMetricCollectorTest {
}
};
- var app = TestUtils.buildApplicationCluster();
- app.getStatus().getJobStatus().setJobId(new JobID().toHexString());
- var conf =
- new FlinkConfigManager(new Configuration())
- .getDeployConfig(app.getMetadata(), app.getSpec());
-
var t1 = new JobTopology(new VertexInfo(new JobVertexID(), Collections.emptySet(), 1, 1));
var t2 = new JobTopology(new VertexInfo(new JobVertexID(), Collections.emptySet(), 1, 1));
- collector.queryFilteredMetricNames(new TestingFlinkService(), app, conf, t1);
+ collector.queryFilteredMetricNames(context, t1);
assertEquals(1, metricNameQueryCounter.get());
- collector.queryFilteredMetricNames(new TestingFlinkService(), app, conf, t1);
- collector.queryFilteredMetricNames(new TestingFlinkService(), app, conf, t1);
+ collector.queryFilteredMetricNames(context, t1);
+ collector.queryFilteredMetricNames(context, t1);
assertEquals(1, metricNameQueryCounter.get());
- collector.queryFilteredMetricNames(new TestingFlinkService(), app, conf, t2);
+ collector.queryFilteredMetricNames(context, t2);
assertEquals(2, metricNameQueryCounter.get());
- collector.queryFilteredMetricNames(new TestingFlinkService(), app, conf, t2);
- collector.queryFilteredMetricNames(new TestingFlinkService(), app, conf, t2);
+ collector.queryFilteredMetricNames(context, t2);
+ collector.queryFilteredMetricNames(context, t2);
assertEquals(2, metricNameQueryCounter.get());
}
@Test
public void testRequiredMetrics() {
List<AggregatedMetric> metricList = new ArrayList<>();
- var testCollector =
- new RestApiMetricsCollector() {
+ RestApiMetricsCollector<JobID, JobAutoScalerContext<JobID>> testCollector =
+ new RestApiMetricsCollector<>() {
@Override
protected Collection<AggregatedMetric> queryAggregatedMetricNames(
RestClusterClient<?> restClient, JobID jobID, JobVertexID jobVertexID) {
@@ -144,7 +143,7 @@ public class ScalingMetricCollectorTest {
private void testRequiredMetrics(
List<AggregatedMetric> metricList,
List<AggregatedMetric> requiredMetrics,
- RestApiMetricsCollector testCollector,
+ RestApiMetricsCollector<JobID, JobAutoScalerContext<JobID>> testCollector,
JobVertexID vertex,
JobTopology topology) {
for (var m : requiredMetrics) {
diff --git a/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/TestingAutoscalerUtils.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/TestingAutoscalerUtils.java
new file mode 100644
index 00000000..e5837d90
--- /dev/null
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/TestingAutoscalerUtils.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.client.program.rest.RestClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneClientHAServices;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.NoOpMetricRegistry;
+import org.apache.flink.runtime.metrics.groups.GenericMetricGroup;
+import org.apache.flink.runtime.rest.messages.JobPlanInfo;
+import org.apache.flink.runtime.rest.messages.job.JobDetailsInfo;
+import org.apache.flink.util.function.SupplierWithException;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+/** The utils for test autoscaler. */
+public class TestingAutoscalerUtils {
+
+ public static JobAutoScalerContext<JobID> createDefaultJobAutoScalerContext() {
+ MetricRegistry registry = NoOpMetricRegistry.INSTANCE;
+ GenericMetricGroup metricGroup = new GenericMetricGroup(registry, null, "test");
+ final JobID jobID = new JobID();
+ return new JobAutoScalerContext<JobID>(
+ jobID,
+ jobID,
+ JobStatus.RUNNING,
+ new Configuration(),
+ metricGroup,
+ getRestClusterClientSupplier());
+ }
+
+ public static SupplierWithException<RestClusterClient<String>, Exception>
+ getRestClusterClientSupplier() {
+ return () ->
+ new RestClusterClient<>(
+ new Configuration(),
+ "test-cluster",
+ (c, e) -> new StandaloneClientHAServices("localhost")) {
+
+ @Override
+ public CompletableFuture<JobDetailsInfo> getJobDetails(JobID jobId) {
+ CompletableFuture<JobDetailsInfo> result = new CompletableFuture<>();
+ result.complete(
+ new JobDetailsInfo(
+ jobId,
+ "",
+ false,
+ JobStatus.RUNNING,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ Map.of(),
+ List.of(),
+ Map.of(),
+ new JobPlanInfo.RawJson("")));
+ return result;
+ }
+ };
+ }
+}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/TestingMetricsCollector.java
similarity index 82%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/TestingMetricsCollector.java
index 277ee33b..a9a150a1 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/TestingMetricsCollector.java
@@ -15,15 +15,13 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.autoscaler.metrics.FlinkMetric;
import org.apache.flink.autoscaler.topology.JobTopology;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.service.FlinkService;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.JobDetailsInfo;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
@@ -38,7 +36,8 @@ import java.util.HashMap;
import java.util.Map;
/** Testing {@link ScalingMetricCollector} implementation. */
-public class TestingMetricsCollector extends ScalingMetricCollector {
+public class TestingMetricsCollector<KEY, Context extends JobAutoScalerContext<KEY>>
+ extends ScalingMetricCollector<KEY, Context> {
@Setter private JobTopology jobTopology;
@@ -62,19 +61,13 @@ public class TestingMetricsCollector extends ScalingMetricCollector {
@Override
protected Map<JobVertexID, Map<FlinkMetric, AggregatedMetric>> queryAllAggregatedMetrics(
- AbstractFlinkResource<?, ?> cr,
- FlinkService flinkService,
- Configuration conf,
- Map<JobVertexID, Map<String, FlinkMetric>> filteredVertexMetricNames) {
+ Context ctx, Map<JobVertexID, Map<String, FlinkMetric>> filteredVertexMetricNames) {
return currentMetrics;
}
@Override
protected Map<JobVertexID, Map<String, FlinkMetric>> queryFilteredMetricNames(
- FlinkService flinkService,
- AbstractFlinkResource<?, ?> cr,
- Configuration conf,
- JobTopology topology) {
+ Context ctx, JobTopology topology) {
return Collections.emptyMap();
}
diff --git a/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/event/TestingEventCollector.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/event/TestingEventCollector.java
new file mode 100644
index 00000000..bd742cde
--- /dev/null
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/event/TestingEventCollector.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.event;
+
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+
+import lombok.Getter;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** Testing {@link AutoScalerEventHandler} implementation. */
+public class TestingEventCollector<KEY, Context extends JobAutoScalerContext<KEY>>
+ implements AutoScalerEventHandler<KEY, Context> {
+
+ public final LinkedList<Event<KEY, Context>> events = new LinkedList<>();
+
+ public final Map<String, Event<KEY, Context>> eventMap = new ConcurrentHashMap<>();
+
+ @Override
+ public void handleEvent(
+ Context context,
+ Type type,
+ String reason,
+ String message,
+ @Nullable String messageKey,
+ @Nullable Duration interval) {
+ String eventKey =
+ generateEventKey(context, type, reason, messageKey != null ? messageKey : message);
+ Event<KEY, Context> event = eventMap.get(eventKey);
+ if (event == null) {
+ Event<KEY, Context> newEvent = new Event<>(context, type, reason, message, messageKey);
+ events.add(newEvent);
+ eventMap.put(eventKey, newEvent);
+ return;
+ }
+ if (Objects.equals(event.getMessage(), message)
+ && interval != null
+ && Instant.now()
+ .isBefore(event.getLastUpdateTimestamp().plusMillis(interval.toMillis()))) {
+ // The event should be ignored.
+ return;
+ }
+ event.incrementCount();
+ events.add(event);
+ }
+
+ private String generateEventKey(Context context, Type type, String reason, String message) {
+ return context.getJobID() + type.name() + reason + message;
+ }
+
+ /** The collected event. */
+ public static class Event<KEY, Context extends JobAutoScalerContext<KEY>> {
+
+ @Getter private Instant lastUpdateTimestamp;
+
+ @Getter private final Context context;
+
+ @Getter private final Type type;
+
+ @Getter private final String reason;
+
+ @Getter private final String message;
+
+ @Getter @Nullable private final String messageKey;
+
+ @Getter private int count;
+
+ public Event(
+ Context context,
+ Type type,
+ String reason,
+ String message,
+ @Nullable String messageKey) {
+ this.lastUpdateTimestamp = Instant.now();
+ this.context = context;
+ this.type = type;
+ this.reason = reason;
+ this.message = message;
+ this.messageKey = messageKey;
+ this.count = 1;
+ }
+
+ private void incrementCount() {
+ count++;
+ lastUpdateTimestamp = Instant.now();
+ }
+ }
+}
diff --git a/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/realizer/TestingScalingRealizer.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/realizer/TestingScalingRealizer.java
new file mode 100644
index 00000000..1e5031c1
--- /dev/null
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/realizer/TestingScalingRealizer.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.realizer;
+
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+
+import java.util.LinkedList;
+import java.util.Map;
+
+/** Testing {@link ScalingRealizer} implementation. */
+public class TestingScalingRealizer<KEY, Context extends JobAutoScalerContext<KEY>>
+ implements ScalingRealizer<KEY, Context> {
+
+ public final LinkedList<Event<KEY, Context>> events = new LinkedList<>();
+
+ @Override
+ public void realize(Context context, Map<String, String> parallelismOverrides) {
+ events.add(new Event<>(context, parallelismOverrides));
+ }
+
+ /** The collected event. */
+ public static class Event<KEY, Context extends JobAutoScalerContext<KEY>> {
+
+ private final Context context;
+
+ private final Map<String, String> parallelismOverrides;
+
+ public Event(Context context, Map<String, String> parallelismOverrides) {
+ this.context = context;
+ this.parallelismOverrides = parallelismOverrides;
+ }
+
+ public Context getContext() {
+ return context;
+ }
+
+ public Map<String, String> getParallelismOverrides() {
+ return parallelismOverrides;
+ }
+ }
+}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerTestUtils.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/state/TestingAutoscalerStateStore.java
similarity index 61%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerTestUtils.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/state/TestingAutoscalerStateStore.java
index 17265b90..3b3fafa1 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerTestUtils.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/state/TestingAutoscalerStateStore.java
@@ -15,19 +15,23 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler.state;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
+import org.apache.flink.autoscaler.JobAutoScalerContext;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import lombok.SneakyThrows;
+/** Testing {@link AutoScalerStateStore} implementation. */
+public class TestingAutoscalerStateStore<KEY, Context extends JobAutoScalerContext<KEY>>
+ extends InMemoryAutoScalerStateStore<KEY, Context> {
-/** Autoscaler test utilities. * */
-public class AutoscalerTestUtils {
+ private int flushCount;
- @SneakyThrows
- public static AutoScalerInfo getOrCreateInfo(
- AbstractFlinkResource<?, ?> cr, KubernetesClient client) {
- return new AutoscalerInfoManager().getOrCreateInfo(cr, client);
+ @Override
+ public void flush(Context jobContext) {
+ super.flush(jobContext);
+ flushCount++;
+ }
+
+ public int getFlushCount() {
+ return flushCount;
}
}
diff --git a/flink-kubernetes-docs/pom.xml b/flink-kubernetes-docs/pom.xml
index fd42c792..54957c1e 100644
--- a/flink-kubernetes-docs/pom.xml
+++ b/flink-kubernetes-docs/pom.xml
@@ -32,7 +32,6 @@ under the License.
<properties>
<generated.docs.dir>./docs/layouts/shortcodes/generated</generated.docs.dir>
- <assertj.version>3.21.0</assertj.version>
</properties>
<dependencies>
diff --git a/flink-kubernetes-operator-autoscaler/pom.xml b/flink-kubernetes-operator-autoscaler/pom.xml
deleted file mode 100644
index 2a6498ca..00000000
--- a/flink-kubernetes-operator-autoscaler/pom.xml
+++ /dev/null
@@ -1,127 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-kubernetes-operator-parent</artifactId>
- <version>1.7-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-kubernetes-operator-autoscaler</artifactId>
- <name>Flink Kubernetes Operator Autoscaler</name>
- <packaging>jar</packaging>
-
- <properties>
- <auto-service.version>1.0.1</auto-service.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-autoscaler</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>io.fabric8</groupId>
- <artifactId>kubernetes-client</artifactId>
- <version>${fabric8.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-kubernetes-operator</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-kubernetes</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.auto.service</groupId>
- <artifactId>auto-service</artifactId>
- <version>${auto-service.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- <version>${lombok.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils</artifactId>
- <version>${flink.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-params</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-kubernetes-operator</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-kubernetes-operator-api</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>io.fabric8</groupId>
- <artifactId>kubernetes-server-mock</artifactId>
- <version>${fabric8.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.squareup.okhttp3</groupId>
- <artifactId>mockwebserver</artifactId>
- <version>${okhttp.version}</version>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-</project>
\ No newline at end of file
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java
deleted file mode 100644
index c37b299f..00000000
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.autoscaler.ScalingSummary;
-import org.apache.flink.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.autoscaler.metrics.CollectedMetrics;
-import org.apache.flink.autoscaler.utils.AutoScalerSerDeModule;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.ConfigurationUtils;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.util.Preconditions;
-
-import com.fasterxml.jackson.core.JacksonException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import io.fabric8.kubernetes.api.model.ConfigMap;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import lombok.SneakyThrows;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.LoaderOptions;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.time.Instant;
-import java.util.Base64;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-
-/** Class for encapsulating information stored for each resource when using the autoscaler. */
-public class AutoScalerInfo {
-
- private static final Logger LOG = LoggerFactory.getLogger(AutoScalerInfo.class);
-
- protected static final String COLLECTED_METRICS_KEY = "collectedMetrics";
- protected static final String SCALING_HISTORY_KEY = "scalingHistory";
-
- protected static final String PARALLELISM_OVERRIDES_KEY = "parallelismOverrides";
-
- protected static final int MAX_CM_BYTES = 1000000;
-
- protected static final ObjectMapper YAML_MAPPER =
- new ObjectMapper(yamlFactory())
- .registerModule(new JavaTimeModule())
- .registerModule(new AutoScalerSerDeModule());
-
- private ConfigMap configMap;
- private Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory;
-
- public AutoScalerInfo(ConfigMap configMap) {
- this.configMap = configMap;
- }
-
- @VisibleForTesting
- public AutoScalerInfo(Map<String, String> data) {
- this(new ConfigMap());
- configMap.setData(Preconditions.checkNotNull(data));
- }
-
- public SortedMap<Instant, CollectedMetrics> getMetricHistory() {
- var historyYaml = configMap.getData().get(COLLECTED_METRICS_KEY);
- if (historyYaml == null) {
- return new TreeMap<>();
- }
-
- try {
- return YAML_MAPPER.readValue(decompress(historyYaml), new TypeReference<>() {});
- } catch (JacksonException e) {
- LOG.error(
- "Could not deserialize metric history, possibly the format changed. Discarding...");
- configMap.getData().remove(COLLECTED_METRICS_KEY);
- return new TreeMap<>();
- }
- }
-
- @SneakyThrows
- public void updateMetricHistory(SortedMap<Instant, CollectedMetrics> history) {
- configMap
- .getData()
- .put(COLLECTED_METRICS_KEY, compress(YAML_MAPPER.writeValueAsString(history)));
- }
-
- @SneakyThrows
- public void updateVertexList(List<JobVertexID> vertexList, Instant now, Configuration conf) {
- // Make sure to init history
- getScalingHistory(now, conf);
-
- if (scalingHistory.keySet().removeIf(v -> !vertexList.contains(v))) {
- storeScalingHistory();
- }
- }
-
- public void clearMetricHistory() {
- configMap.getData().remove(COLLECTED_METRICS_KEY);
- }
-
- private void trimScalingHistory(Instant now, Configuration conf) {
- var entryIt = scalingHistory.entrySet().iterator();
- while (entryIt.hasNext()) {
- var entry = entryIt.next();
- // Limit how long past scaling decisions are remembered
- entry.setValue(
- entry.getValue()
- .tailMap(
- now.minus(
- conf.get(
- AutoScalerOptions
- .VERTEX_SCALING_HISTORY_AGE))));
- var vertexHistory = entry.getValue();
- while (vertexHistory.size()
- > conf.get(AutoScalerOptions.VERTEX_SCALING_HISTORY_COUNT)) {
- vertexHistory.remove(vertexHistory.firstKey());
- }
- if (vertexHistory.isEmpty()) {
- entryIt.remove();
- }
- }
- }
-
- @SneakyThrows
- public Map<JobVertexID, SortedMap<Instant, ScalingSummary>> getScalingHistory(
- Instant now, Configuration conf) {
- if (scalingHistory != null) {
- trimScalingHistory(now, conf);
- return scalingHistory;
- }
- var yaml = decompress(configMap.getData().get(SCALING_HISTORY_KEY));
- if (yaml == null) {
- scalingHistory = new HashMap<>();
- return scalingHistory;
- }
- try {
- scalingHistory = YAML_MAPPER.readValue(yaml, new TypeReference<>() {});
- } catch (JacksonException e) {
- LOG.error(
- "Could not deserialize scaling history, possibly the format changed. Discarding...");
- configMap.getData().remove(SCALING_HISTORY_KEY);
- scalingHistory = new HashMap<>();
- }
- return scalingHistory;
- }
-
- @SneakyThrows
- public void addToScalingHistory(
- Instant now, Map<JobVertexID, ScalingSummary> summaries, Configuration conf) {
- // Make sure to init history
- getScalingHistory(now, conf);
-
- summaries.forEach(
- (id, summary) ->
- scalingHistory.computeIfAbsent(id, j -> new TreeMap<>()).put(now, summary));
-
- storeScalingHistory();
- }
-
- public void setCurrentOverrides(Map<String, String> overrides) {
- configMap
- .getData()
- .put(
- PARALLELISM_OVERRIDES_KEY,
- ConfigurationUtils.convertValue(overrides, String.class));
- }
-
- public Map<String, String> getCurrentOverrides() {
- var overridesStr = configMap.getData().get(PARALLELISM_OVERRIDES_KEY);
- if (overridesStr == null) {
- return Map.of();
- }
- return ConfigurationUtils.convertValue(overridesStr, Map.class);
- }
-
- public void removeCurrentOverrides() {
- configMap.getData().remove(PARALLELISM_OVERRIDES_KEY);
- }
-
- private void storeScalingHistory() throws Exception {
- configMap
- .getData()
- .put(SCALING_HISTORY_KEY, compress(YAML_MAPPER.writeValueAsString(scalingHistory)));
- }
-
- public void replaceInKubernetes(KubernetesClient client) throws Exception {
- trimHistoryToMaxCmSize();
- try {
- configMap = client.resource(configMap).update();
- } catch (Exception e) {
- LOG.error(
- "Error while updating autoscaler info configmap, invalidating to clear the cache",
- e);
- configMap = null;
- throw e;
- }
- }
-
- public boolean isValid() {
- return configMap != null;
- }
-
- @VisibleForTesting
- protected void trimHistoryToMaxCmSize() throws Exception {
- var data = configMap.getData();
-
- int scalingHistorySize = data.getOrDefault(SCALING_HISTORY_KEY, "").length();
- int metricHistorySize = data.getOrDefault(COLLECTED_METRICS_KEY, "").length();
-
- SortedMap<Instant, CollectedMetrics> metricHistory = null;
- while (scalingHistorySize + metricHistorySize > MAX_CM_BYTES) {
- if (metricHistory == null) {
- metricHistory = getMetricHistory();
- }
- if (metricHistory.isEmpty()) {
- return;
- }
- var firstKey = metricHistory.firstKey();
- LOG.info("Trimming metric history by removing {}", firstKey);
- metricHistory.remove(firstKey);
- String compressed = compress(YAML_MAPPER.writeValueAsString(metricHistory));
- data.put(COLLECTED_METRICS_KEY, compressed);
- metricHistorySize = compressed.length();
- }
- }
-
- @VisibleForTesting
- protected ConfigMap getConfigMap() {
- return configMap;
- }
-
- private static String compress(String original) throws IOException {
- ByteArrayOutputStream rstBao = new ByteArrayOutputStream();
- try (var zos = new GZIPOutputStream(rstBao)) {
- zos.write(original.getBytes(StandardCharsets.UTF_8));
- }
-
- return Base64.getEncoder().encodeToString(rstBao.toByteArray());
- }
-
- private static String decompress(String compressed) {
- if (compressed == null) {
- return null;
- }
-
- try {
- byte[] bytes = Base64.getDecoder().decode(compressed);
- try (var zi = new GZIPInputStream(new ByteArrayInputStream(bytes))) {
- return IOUtils.toString(zi, StandardCharsets.UTF_8);
- }
- } catch (Exception e) {
- LOG.warn("Error while decompressing scaling data, treating as uncompressed");
- // Fall back to non-compressed for migration
- return compressed;
- }
- }
-
- private static YAMLFactory yamlFactory() {
- // Set yaml size limit to 10mb
- var loaderOptions = new LoaderOptions();
- loaderOptions.setCodePointLimit(20 * 1024 * 1024);
- return YAMLFactory.builder().loaderOptions(loaderOptions).build();
- }
-}
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManager.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManager.java
deleted file mode 100644
index 8602743f..00000000
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManager.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.utils.Constants;
-
-import io.fabric8.kubernetes.api.model.ConfigMap;
-import io.fabric8.kubernetes.api.model.HasMetadata;
-import io.fabric8.kubernetes.api.model.ObjectMeta;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-
-/** Class responsible to managing the creation and retrieval of {@link AutoScalerInfo} objects. */
-public class AutoscalerInfoManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(AutoscalerInfoManager.class);
- private static final String LABEL_COMPONENT_AUTOSCALER = "autoscaler";
-
- private final ConcurrentHashMap<ResourceID, Optional<AutoScalerInfo>> cache =
- new ConcurrentHashMap<>();
-
- public AutoscalerInfoManager() {}
-
- public AutoScalerInfo getOrCreateInfo(
- AbstractFlinkResource<?, ?> cr, KubernetesClient kubernetesClient) {
- return cache.compute(
- ResourceID.fromResource(cr),
- (id, infOpt) -> {
- // If in the cache and valid simply return
- if (infOpt != null
- && infOpt.map(AutoScalerInfo::isValid).orElse(false)) {
- return infOpt;
- }
- // Otherwise get or create
- return Optional.of(getOrCreateInternal(cr, kubernetesClient));
- })
- .get();
- }
-
- public Optional<AutoScalerInfo> getInfo(
- AbstractFlinkResource<?, ?> cr, KubernetesClient kubernetesClient) {
- return cache.compute(
- ResourceID.fromResource(cr),
- (id, infOpt) -> {
- // If in the cache and empty or valid simply return
- if (infOpt != null && (infOpt.isEmpty() || infOpt.get().isValid())) {
- return infOpt;
- }
-
- // Otherwise get
- return getScalingInfoConfigMap(
- createCmObjectMeta(ResourceID.fromResource(cr)),
- kubernetesClient)
- .map(AutoScalerInfo::new);
- });
- }
-
- public void removeInfoFromCache(AbstractFlinkResource<?, ?> cr) {
- // We don't need to remove from Kubernetes, that is handled through the owner reference
- cache.remove(ResourceID.fromResource(cr));
- }
-
- @VisibleForTesting
- protected Optional<ConfigMap> getInfoFromKubernetes(
- AbstractFlinkResource<?, ?> cr, KubernetesClient kubernetesClient) {
- return getScalingInfoConfigMap(
- createCmObjectMeta(ResourceID.fromResource(cr)), kubernetesClient);
- }
-
- private AutoScalerInfo getOrCreateInternal(HasMetadata cr, KubernetesClient kubernetesClient) {
- var meta = createCmObjectMeta(ResourceID.fromResource(cr));
- var info =
- getScalingInfoConfigMap(meta, kubernetesClient)
- .orElseGet(() -> createConfigMap(cr, meta, kubernetesClient));
-
- return new AutoScalerInfo(info);
- }
-
- private ConfigMap createConfigMap(
- HasMetadata cr, ObjectMeta meta, KubernetesClient kubernetesClient) {
- LOG.info("Creating scaling info config map");
- var cm = new ConfigMap();
- cm.setMetadata(meta);
- cm.addOwnerReference(cr);
- cm.setData(new HashMap<>());
- return kubernetesClient.resource(cm).create();
- }
-
- private static ObjectMeta createCmObjectMeta(ResourceID uid) {
- var objectMeta = new ObjectMeta();
- objectMeta.setName("autoscaler-" + uid.getName());
- uid.getNamespace().ifPresent(objectMeta::setNamespace);
- objectMeta.setLabels(
- Map.of(
- Constants.LABEL_COMPONENT_KEY,
- LABEL_COMPONENT_AUTOSCALER,
- Constants.LABEL_APP_KEY,
- uid.getName()));
- return objectMeta;
- }
-
- private Optional<ConfigMap> getScalingInfoConfigMap(
- ObjectMeta objectMeta, KubernetesClient kubernetesClient) {
- return Optional.ofNullable(
- kubernetesClient
- .configMaps()
- .inNamespace(objectMeta.getNamespace())
- .withName(objectMeta.getName())
- .get());
- }
-}
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java
deleted file mode 100644
index 87603fce..00000000
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
-import org.apache.flink.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.ConfigurationUtils;
-import org.apache.flink.configuration.PipelineOptions;
-import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.api.lifecycle.ResourceLifecycleState;
-import org.apache.flink.kubernetes.operator.api.status.CommonStatus;
-import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScaler;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.apache.flink.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.initRecommendedParallelism;
-import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.resetRecommendedParallelism;
-
-/** Application and SessionJob autoscaler. */
-public class JobAutoScalerImpl implements JobAutoScaler {
-
- private static final Logger LOG = LoggerFactory.getLogger(JobAutoScalerImpl.class);
-
- private final ScalingMetricCollector metricsCollector;
- private final ScalingMetricEvaluator evaluator;
- private final ScalingExecutor scalingExecutor;
- private final EventRecorder eventRecorder;
-
- @VisibleForTesting
- final Map<ResourceID, Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>>>
- lastEvaluatedMetrics = new ConcurrentHashMap<>();
-
- @VisibleForTesting
- final Map<ResourceID, AutoscalerFlinkMetrics> flinkMetrics = new ConcurrentHashMap<>();
-
- @VisibleForTesting final AutoscalerInfoManager infoManager;
-
- public JobAutoScalerImpl(
- ScalingMetricCollector metricsCollector,
- ScalingMetricEvaluator evaluator,
- ScalingExecutor scalingExecutor,
- EventRecorder eventRecorder) {
- this.metricsCollector = metricsCollector;
- this.evaluator = evaluator;
- this.scalingExecutor = scalingExecutor;
- this.eventRecorder = eventRecorder;
- this.infoManager = new AutoscalerInfoManager();
- }
-
- @Override
- public void scale(FlinkResourceContext<?> ctx) {
- var conf = ctx.getObserveConfig();
- var resource = ctx.getResource();
- var resourceId = ResourceID.fromResource(resource);
- var autoscalerMetrics = getOrInitAutoscalerFlinkMetrics(ctx, resourceId);
-
- try {
- runScalingLogic(ctx, conf, resource, resourceId, autoscalerMetrics);
- } catch (Throwable e) {
- onError(ctx, resource, autoscalerMetrics, e);
- } finally {
- applyParallelismOverrides(ctx);
- }
- }
-
- @Override
- public void cleanup(FlinkResourceContext<?> ctx) {
- LOG.info("Cleaning up autoscaling meta data");
- var cr = ctx.getResource();
- metricsCollector.cleanup(cr);
- var resourceId = ResourceID.fromResource(cr);
- lastEvaluatedMetrics.remove(resourceId);
- flinkMetrics.remove(resourceId);
- infoManager.removeInfoFromCache(cr);
- }
-
- private void clearParallelismOverrides(FlinkResourceContext<?> ctx) throws Exception {
- var infoOpt = infoManager.getInfo(ctx.getResource(), ctx.getKubernetesClient());
- if (infoOpt.isPresent()) {
- var info = infoOpt.get();
- info.removeCurrentOverrides();
- info.replaceInKubernetes(ctx.getKubernetesClient());
- }
- }
-
- @VisibleForTesting
- protected Map<String, String> getParallelismOverrides(FlinkResourceContext<?> ctx) {
- return infoManager
- .getInfo(ctx.getResource(), ctx.getKubernetesClient())
- .map(AutoScalerInfo::getCurrentOverrides)
- .orElse(Map.of());
- }
-
- /**
- * If there are any parallelism overrides by the {@link JobAutoScaler} apply them to the spec.
- *
- * @param ctx Resource context
- */
- @VisibleForTesting
- protected void applyParallelismOverrides(FlinkResourceContext<?> ctx) {
- var overrides = getParallelismOverrides(ctx);
- if (overrides.isEmpty()) {
- return;
- }
-
- LOG.debug("Applying parallelism overrides: {}", overrides);
-
- var spec = ctx.getResource().getSpec();
- var conf = ctx.getDeployConfig(spec);
- var userOverrides = new HashMap<>(conf.get(PipelineOptions.PARALLELISM_OVERRIDES));
- var exclusions = conf.get(AutoScalerOptions.VERTEX_EXCLUDE_IDS);
-
- overrides.forEach(
- (k, v) -> {
- // Respect user override for excluded vertices
- if (exclusions.contains(k)) {
- userOverrides.putIfAbsent(k, v);
- } else {
- userOverrides.put(k, v);
- }
- });
- spec.getFlinkConfiguration()
- .put(
- PipelineOptions.PARALLELISM_OVERRIDES.key(),
- ConfigurationUtils.convertValue(userOverrides, String.class));
- }
-
- private void runScalingLogic(
- FlinkResourceContext<?> ctx,
- Configuration conf,
- AbstractFlinkResource<?, ?> resource,
- ResourceID resourceId,
- AutoscalerFlinkMetrics autoscalerMetrics)
- throws Exception {
-
- if (resource.getSpec().getJob() == null || !conf.getBoolean(AUTOSCALER_ENABLED)) {
- LOG.debug("Autoscaler is disabled");
- clearParallelismOverrides(ctx);
- return;
- }
-
- var status = resource.getStatus();
- if (status.getLifecycleState() != ResourceLifecycleState.STABLE
- || !status.getJobStatus().getState().equals(JobStatus.RUNNING.name())) {
- LOG.info("Autoscaler is waiting for stable, running state");
- lastEvaluatedMetrics.remove(resourceId);
- return;
- }
-
- var autoScalerInfo = infoManager.getOrCreateInfo(resource, ctx.getKubernetesClient());
-
- var collectedMetrics =
- metricsCollector.updateMetrics(
- resource, autoScalerInfo, ctx.getFlinkService(), conf);
-
- if (collectedMetrics.getMetricHistory().isEmpty()) {
- autoScalerInfo.replaceInKubernetes(ctx.getKubernetesClient());
- return;
- }
- LOG.debug("Collected metrics: {}", collectedMetrics);
-
- var evaluatedMetrics = evaluator.evaluate(conf, collectedMetrics);
- LOG.debug("Evaluated metrics: {}", evaluatedMetrics);
- lastEvaluatedMetrics.put(resourceId, evaluatedMetrics);
-
- initRecommendedParallelism(evaluatedMetrics);
- autoscalerMetrics.registerScalingMetrics(
- collectedMetrics.getJobTopology().getVerticesInTopologicalOrder(),
- () -> lastEvaluatedMetrics.get(resourceId));
-
- if (!collectedMetrics.isFullyCollected()) {
- // We have done an upfront evaluation, but we are not ready for scaling.
- resetRecommendedParallelism(evaluatedMetrics);
- autoScalerInfo.replaceInKubernetes(ctx.getKubernetesClient());
- return;
- }
-
- var parallelismChanged =
- scalingExecutor.scaleResource(
- resource,
- autoScalerInfo,
- conf,
- evaluatedMetrics,
- ctx.getKubernetesClient());
-
- if (parallelismChanged) {
- autoscalerMetrics.numScalings.inc();
- } else {
- autoscalerMetrics.numBalanced.inc();
- }
-
- autoScalerInfo.replaceInKubernetes(ctx.getKubernetesClient());
- }
-
- private void onError(
- FlinkResourceContext<?> ctx,
- AbstractFlinkResource<?, ? extends CommonStatus<?>> resource,
- AutoscalerFlinkMetrics autoscalerMetrics,
- Throwable e) {
- LOG.error("Error while scaling resource", e);
- autoscalerMetrics.numErrors.inc();
- eventRecorder.triggerEvent(
- resource,
- EventRecorder.Type.Warning,
- EventRecorder.Reason.AutoscalerError,
- EventRecorder.Component.Operator,
- e.getMessage(),
- ctx.getKubernetesClient());
- }
-
- private AutoscalerFlinkMetrics getOrInitAutoscalerFlinkMetrics(
- FlinkResourceContext<? extends AbstractFlinkResource<?, ?>> ctx, ResourceID resouceId) {
- return this.flinkMetrics.computeIfAbsent(
- resouceId,
- id ->
- new AutoscalerFlinkMetrics(
- ctx.getResourceMetricGroup().addGroup("AutoScaler")));
- }
-}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java
deleted file mode 100644
index 94ada9fb..00000000
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.autoscaler.ScalingSummary;
-import org.apache.flink.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.autoscaler.metrics.CollectedMetrics;
-import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import io.fabric8.kubernetes.api.model.ConfigMap;
-import org.junit.jupiter.api.Test;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/** Test for AutoScalerInfo. */
-public class AutoScalerInfoTest {
-
- @Test
- public void testTopologyUpdate() {
- var data = new HashMap<String, String>();
- var info = new AutoScalerInfo(data);
-
- var v1 = new JobVertexID();
- var v2 = new JobVertexID();
- var v3 = new JobVertexID();
-
- var history = new HashMap<JobVertexID, ScalingSummary>();
- history.put(v1, new ScalingSummary(1, 2, null));
- history.put(v2, new ScalingSummary(1, 2, null));
-
- var conf = new Configuration();
- var now = Instant.now();
-
- info.addToScalingHistory(now, history, conf);
-
- assertEquals(history.keySet(), info.getScalingHistory(now, conf).keySet());
- assertEquals(
- history.keySet(), new AutoScalerInfo(data).getScalingHistory(now, conf).keySet());
-
- info.updateVertexList(List.of(v2, v3), now, conf);
-
- // Expect v1 to be removed
- assertEquals(Set.of(v2), info.getScalingHistory(now, conf).keySet());
- assertEquals(Set.of(v2), new AutoScalerInfo(data).getScalingHistory(now, conf).keySet());
- }
-
- @Test
- public void testHistorySizeConfigs() {
- var data = new HashMap<String, String>();
- var info = new AutoScalerInfo(data);
-
- var v1 = new JobVertexID();
-
- var history = new HashMap<JobVertexID, ScalingSummary>();
- history.put(v1, new ScalingSummary(1, 2, null));
-
- var conf = new Configuration();
- conf.set(AutoScalerOptions.VERTEX_SCALING_HISTORY_COUNT, 2);
- conf.set(AutoScalerOptions.VERTEX_SCALING_HISTORY_AGE, Duration.ofSeconds(10));
-
- var now = Instant.now();
-
- // Verify count based expiration
- info.addToScalingHistory(now, history, conf);
- assertEquals(1, info.getScalingHistory(now, conf).get(v1).size());
-
- info.addToScalingHistory(now.plus(Duration.ofSeconds(1)), history, conf);
- info.addToScalingHistory(now.plus(Duration.ofSeconds(2)), history, conf);
-
- assertEquals(
- 2, info.getScalingHistory(now.plus(Duration.ofSeconds(2)), conf).get(v1).size());
- assertEquals(
- Set.of(now.plus(Duration.ofSeconds(1)), now.plus(Duration.ofSeconds(2))),
- info.getScalingHistory(now.plus(Duration.ofSeconds(2)), conf).get(v1).keySet());
-
- // Verify time based expiration
- info.addToScalingHistory(now.plus(Duration.ofSeconds(15)), history, conf);
- assertEquals(
- 1, info.getScalingHistory(now.plus(Duration.ofSeconds(15)), conf).get(v1).size());
- assertEquals(
- Set.of(now.plus(Duration.ofSeconds(15))),
- info.getScalingHistory(now.plus(Duration.ofSeconds(15)), conf).get(v1).keySet());
- assertEquals(
- Set.of(now.plus(Duration.ofSeconds(15))),
- new AutoScalerInfo(data)
- .getScalingHistory(now.plus(Duration.ofSeconds(15)), conf)
- .get(v1)
- .keySet());
- }
-
- @Test
- public void testCompressionMigration() throws JsonProcessingException {
- var jobUpdateTs = Instant.now();
- var v1 = new JobVertexID();
-
- var metricHistory = new TreeMap<Instant, CollectedMetrics>();
- metricHistory.put(
- jobUpdateTs,
- new CollectedMetrics(
- Map.of(v1, Map.of(ScalingMetric.TRUE_PROCESSING_RATE, 1.)), Map.of()));
-
- var scalingHistory = new HashMap<JobVertexID, SortedMap<Instant, ScalingSummary>>();
- scalingHistory.put(v1, new TreeMap<>());
- scalingHistory
- .get(v1)
- .put(
- jobUpdateTs,
- new ScalingSummary(
- 1, 2, Map.of(ScalingMetric.LAG, EvaluatedScalingMetric.of(2.))));
-
- // Store uncompressed data in map to simulate migration
- var data = new HashMap<String, String>();
- data.put(
- AutoScalerInfo.COLLECTED_METRICS_KEY,
- AutoScalerInfo.YAML_MAPPER.writeValueAsString(metricHistory));
- data.put(
- AutoScalerInfo.SCALING_HISTORY_KEY,
- AutoScalerInfo.YAML_MAPPER.writeValueAsString(scalingHistory));
-
- var info = new AutoScalerInfo(data);
- var conf = new Configuration();
- var now = Instant.now();
-
- assertEquals(scalingHistory, info.getScalingHistory(now, conf));
- assertEquals(metricHistory, info.getMetricHistory());
-
- // Override with compressed data
- var newTs = Instant.now();
- info.updateMetricHistory(metricHistory);
- info.addToScalingHistory(newTs, Map.of(), conf);
-
- // Make sure we can still access everything
- assertEquals(scalingHistory, info.getScalingHistory(newTs, conf));
- assertEquals(metricHistory, info.getMetricHistory());
- }
-
- @Test
- public void testMetricsTrimming() throws Exception {
- var v1 = new JobVertexID();
- Random rnd = new Random();
-
- var metricHistory = new TreeMap<Instant, CollectedMetrics>();
- for (int i = 0; i < 50; i++) {
- var m = new HashMap<JobVertexID, Map<ScalingMetric, Double>>();
- for (int j = 0; j < 500; j++) {
- m.put(
- new JobVertexID(),
- Map.of(ScalingMetric.TRUE_PROCESSING_RATE, rnd.nextDouble()));
- }
- metricHistory.put(Instant.now(), new CollectedMetrics(m, Collections.emptyMap()));
- }
-
- var scalingHistory = new HashMap<JobVertexID, SortedMap<Instant, ScalingSummary>>();
- scalingHistory.put(v1, new TreeMap<>());
- scalingHistory
- .get(v1)
- .put(
- Instant.now(),
- new ScalingSummary(
- 1, 2, Map.of(ScalingMetric.LAG, EvaluatedScalingMetric.of(2.))));
-
- var data = new HashMap<String, String>();
- var info = new AutoScalerInfo(data);
-
- info.addToScalingHistory(
- Instant.now(),
- Map.of(
- v1,
- new ScalingSummary(
- 1, 2, Map.of(ScalingMetric.LAG, EvaluatedScalingMetric.of(2.)))),
- new Configuration());
-
- info.updateMetricHistory(metricHistory);
-
- assertFalse(
- data.get(AutoScalerInfo.COLLECTED_METRICS_KEY).length()
- + data.get(AutoScalerInfo.SCALING_HISTORY_KEY).length()
- < AutoScalerInfo.MAX_CM_BYTES);
-
- info.trimHistoryToMaxCmSize();
- assertTrue(
- data.get(AutoScalerInfo.COLLECTED_METRICS_KEY).length()
- + data.get(AutoScalerInfo.SCALING_HISTORY_KEY).length()
- < AutoScalerInfo.MAX_CM_BYTES);
- }
-
- @Test
- public void testDiscardInvalidHistory() {
- ConfigMap configMap = new ConfigMap();
- configMap.setData(
- new HashMap<>(
- Map.of(
- AutoScalerInfo.COLLECTED_METRICS_KEY,
- "invalid",
- AutoScalerInfo.SCALING_HISTORY_KEY,
- "invalid2")));
-
- var info = new AutoScalerInfo(configMap);
- var now = Instant.now();
- var conf = new Configuration();
-
- assertEquals(2, configMap.getData().size());
-
- assertEquals(new TreeMap<>(), info.getMetricHistory());
- assertNull(configMap.getData().get(AutoScalerInfo.COLLECTED_METRICS_KEY));
-
- assertEquals(new TreeMap<>(), info.getScalingHistory(now, conf));
- assertNull(configMap.getData().get(AutoScalerInfo.SCALING_HISTORY_KEY));
- }
-}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManagerTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManagerTest.java
deleted file mode 100644
index 4c51d05a..00000000
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerInfoManagerTest.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
-
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.KubernetesClientException;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
-import io.fabric8.kubernetes.client.server.mock.KubernetesMockServer;
-import org.junit.jupiter.api.Test;
-
-import java.util.Map;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/** Test for {@link AutoscalerInfoManager}. */
-@EnableKubernetesMockClient(crud = true)
-public class AutoscalerInfoManagerTest {
-
- KubernetesClient kubernetesClient;
- KubernetesMockServer mockWebServer;
-
- @Test
- void testCaching() throws Exception {
- var cr1 = TestUtils.buildApplicationCluster();
- cr1.getMetadata().setName("cr1");
- var cr2 = TestUtils.buildApplicationCluster();
- cr2.getMetadata().setName("cr2");
-
- var manager = new AutoscalerInfoManager();
- assertEquals(0, mockWebServer.getRequestCount());
-
- assertTrue(manager.getInfo(cr1, kubernetesClient).isEmpty());
- assertEquals(1, mockWebServer.getRequestCount());
-
- // Further gets should not go to K8s
- assertTrue(manager.getInfo(cr1, kubernetesClient).isEmpty());
- assertTrue(manager.getInfo(cr1, kubernetesClient).isEmpty());
- assertEquals(1, mockWebServer.getRequestCount());
-
- assertTrue(manager.getInfoFromKubernetes(cr1, kubernetesClient).isEmpty());
- assertEquals(2, mockWebServer.getRequestCount());
-
- var info1 = manager.getOrCreateInfo(cr1, kubernetesClient);
- assertEquals(4, mockWebServer.getRequestCount());
-
- assertTrue(info1 == manager.getOrCreateInfo(cr1, kubernetesClient));
- assertEquals(4, mockWebServer.getRequestCount());
-
- assertFalse(manager.getInfoFromKubernetes(cr1, kubernetesClient).isEmpty());
- assertTrue(manager.getInfoFromKubernetes(cr2, kubernetesClient).isEmpty());
-
- assertFalse(manager.getInfo(cr1, kubernetesClient).isEmpty());
- assertTrue(manager.getInfo(cr2, kubernetesClient).isEmpty());
-
- var info2 = manager.getOrCreateInfo(cr2, kubernetesClient);
- info1.setCurrentOverrides(Map.of("a", "1"));
- info1.replaceInKubernetes(kubernetesClient);
- info2.setCurrentOverrides(Map.of("b", "1"));
- info2.replaceInKubernetes(kubernetesClient);
-
- manager = new AutoscalerInfoManager();
- assertEquals(
- Map.of("a", "1"),
- manager.getInfo(cr1, kubernetesClient).get().getCurrentOverrides());
- assertEquals(
- Map.of("b", "1"),
- manager.getOrCreateInfo(cr2, kubernetesClient).getCurrentOverrides());
-
- // Removing from the cache should not affect the results
- manager.removeInfoFromCache(cr1);
- assertEquals(
- Map.of("a", "1"),
- manager.getInfo(cr1, kubernetesClient).get().getCurrentOverrides());
- assertEquals(
- Map.of("b", "1"),
- manager.getOrCreateInfo(cr2, kubernetesClient).getCurrentOverrides());
- }
-
- @Test
- void testErrorHandling() {
- var cr1 = TestUtils.buildApplicationCluster();
- cr1.getMetadata().setName("cr1");
-
- var manager = new AutoscalerInfoManager();
- assertEquals(0, mockWebServer.getRequestCount());
-
- var info1 = manager.getOrCreateInfo(cr1, kubernetesClient);
- assertTrue(info1.isValid());
- assertEquals(2, mockWebServer.getRequestCount());
-
- // Modify the autoscaler info in the background
- var cm = ReconciliationUtils.clone(info1.getConfigMap());
- cm.getData().put(AutoScalerInfo.PARALLELISM_OVERRIDES_KEY, "a:3");
- kubernetesClient.resource(cm).update();
-
- info1.setCurrentOverrides(Map.of("a", "1"));
-
- // Replace should throw an error due to the modification
- assertThrows(
- KubernetesClientException.class, () -> info1.replaceInKubernetes(kubernetesClient));
- assertFalse(info1.isValid());
-
- // Make sure we can get the new version
- assertEquals(
- Map.of("a", "3"),
- manager.getInfo(cr1, kubernetesClient).get().getCurrentOverrides());
- assertEquals(
- Map.of("a", "3"),
- manager.getOrCreateInfo(cr1, kubernetesClient).getCurrentOverrides());
- }
-}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java
deleted file mode 100644
index 7316668b..00000000
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobStatus;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
-import org.apache.flink.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.autoscaler.topology.JobTopology;
-import org.apache.flink.autoscaler.topology.VertexInfo;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.PipelineOptions;
-import org.apache.flink.kubernetes.operator.OperatorTestBase;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
-import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
-import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
-import org.apache.flink.metrics.Gauge;
-import org.apache.flink.metrics.Metric;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.mock.Whitebox;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.metrics.groups.GenericMetricGroup;
-import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
-
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
-import io.fabric8.kubernetes.client.server.mock.KubernetesMockServer;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import lombok.Getter;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.flink.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
-import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/** Tests for JobAutoScalerImpl. */
-@EnableKubernetesMockClient(crud = true)
-public class JobAutoScalerImplTest extends OperatorTestBase {
-
- @Getter private KubernetesClient kubernetesClient;
-
- KubernetesMockServer mockWebServer;
-
- private FlinkDeployment app;
-
- @BeforeEach
- public void setup() {
- app = TestUtils.buildApplicationCluster();
- app.getMetadata().setGeneration(1L);
- app.getStatus().getJobStatus().setJobId(new JobID().toHexString());
- kubernetesClient.resource(app).createOrReplace();
-
- var defaultConf = new Configuration();
- defaultConf.set(AUTOSCALER_ENABLED, true);
- configManager = new FlinkConfigManager(defaultConf);
- ReconciliationUtils.updateStatusForDeployedSpec(
- app, configManager.getDeployConfig(app.getMetadata(), app.getSpec()));
- app.getStatus().getJobStatus().setState(JobStatus.RUNNING.name());
- app.getStatus().getReconciliationStatus().markReconciledSpecAsStable();
- }
-
- @Test
- void testMetricReporting() {
- JobVertexID jobVertexID = new JobVertexID();
- JobTopology jobTopology = new JobTopology(new VertexInfo(jobVertexID, Set.of(), 1, 10));
-
- TestingMetricsCollector metricsCollector = new TestingMetricsCollector(jobTopology);
- metricsCollector.setCurrentMetrics(
- Map.of(
- jobVertexID,
- Map.of(
- FlinkMetric.BUSY_TIME_PER_SEC,
- new AggregatedMetric("load", 0., 420., 0., 0.))));
- metricsCollector.setJobUpdateTs(Instant.ofEpochMilli(0));
-
- ScalingMetricEvaluator evaluator = new ScalingMetricEvaluator();
- ScalingExecutor scalingExecutor = new ScalingExecutor(eventRecorder);
-
- var autoscaler =
- new JobAutoScalerImpl(metricsCollector, evaluator, scalingExecutor, eventRecorder);
- FlinkResourceContext<FlinkDeployment> resourceContext = getResourceContext(app);
- ResourceID resourceId = ResourceID.fromResource(app);
-
- autoscaler.scale(resourceContext);
-
- MetricGroup metricGroup = autoscaler.flinkMetrics.get(resourceId).metricGroup;
- assertEquals(
- 0.42,
- getGaugeValue(
- metricGroup,
- AutoscalerFlinkMetrics.CURRENT,
- AutoscalerFlinkMetrics.JOB_VERTEX_ID,
- jobVertexID.toHexString(),
- ScalingMetric.LOAD.name()),
- "Expected scaling metric LOAD was not reported. Reporting is broken");
- }
-
- @SuppressWarnings("unchecked")
- private static double getGaugeValue(
- MetricGroup metricGroup, String gaugeName, String... nestedMetricGroupNames) {
- for (String nestedMetricGroupName : nestedMetricGroupNames) {
- metricGroup =
- ((Map<String, GenericMetricGroup>)
- Whitebox.getInternalState(metricGroup, "groups"))
- .get(nestedMetricGroupName);
- }
- var metrics = (Map<String, Metric>) Whitebox.getInternalState(metricGroup, "metrics");
- return ((Gauge<Double>) metrics.get(gaugeName)).getValue();
- }
-
- @Test
- void testErrorReporting() {
- var autoscaler = new JobAutoScalerImpl(null, null, null, eventRecorder);
- FlinkResourceContext<FlinkDeployment> resourceContext = getResourceContext(app);
- ResourceID resourceId = ResourceID.fromResource(app);
-
- autoscaler.scale(resourceContext);
- Assertions.assertEquals(1, autoscaler.flinkMetrics.get(resourceId).numErrors.getCount());
-
- autoscaler.scale(resourceContext);
- Assertions.assertEquals(2, autoscaler.flinkMetrics.get(resourceId).numErrors.getCount());
-
- assertEquals(0, autoscaler.flinkMetrics.get(resourceId).numScalings.getCount());
- }
-
- @Test
- void testParallelismOverrides() throws Exception {
- var autoscaler = new JobAutoScalerImpl(null, null, null, eventRecorder);
- app.getStatus().getJobStatus().setState(JobStatus.CREATED.name());
- var appClone = ReconciliationUtils.clone(app);
- var ctx = getResourceContext(app);
-
- // Initially we should return empty overrides, do not crate any CM
- assertEquals(Map.of(), autoscaler.getParallelismOverrides(ctx));
- assertFalse(
- autoscaler.infoManager.getInfoFromKubernetes(app, kubernetesClient).isPresent());
-
- var autoscalerInfo = autoscaler.infoManager.getOrCreateInfo(app, kubernetesClient);
-
- var v1 = new JobVertexID().toString();
- var v2 = new JobVertexID().toString();
- autoscalerInfo.setCurrentOverrides(Map.of(v1, "1", v2, "2"));
- autoscalerInfo.replaceInKubernetes(kubernetesClient);
-
- autoscaler.scale(ctx);
- assertEquals(
- Map.of(v1, "1", v2, "2"),
- ctx.getDeployConfig(app.getSpec()).get(PipelineOptions.PARALLELISM_OVERRIDES));
-
- // Disabling autoscaler should clear overrides
- ctx = getResourceContext(app = ReconciliationUtils.clone(appClone));
- app.getSpec().getFlinkConfiguration().put(AUTOSCALER_ENABLED.key(), "false");
-
- autoscaler.scale(ctx);
- assertEquals(Map.of(), autoscaler.getParallelismOverrides(ctx));
- // But not clear the autoscaler info
- assertTrue(autoscaler.infoManager.getInfoFromKubernetes(app, kubernetesClient).isPresent());
- assertEquals(
- Map.of(),
- ctx.getDeployConfig(app.getSpec()).get(PipelineOptions.PARALLELISM_OVERRIDES));
-
- int requestCount = mockWebServer.getRequestCount();
- // Make sure we don't update in kubernetes once removed
- ctx = getResourceContext(app = ReconciliationUtils.clone(appClone));
- autoscaler.scale(ctx);
- assertEquals(requestCount, mockWebServer.getRequestCount());
-
- ctx = getResourceContext(app = ReconciliationUtils.clone(appClone));
- app.getSpec().getFlinkConfiguration().put(AUTOSCALER_ENABLED.key(), "true");
- autoscaler.scale(ctx);
-
- assertEquals(Map.of(), autoscaler.getParallelismOverrides(ctx));
- assertEquals(
- Map.of(),
- ctx.getDeployConfig(app.getSpec()).get(PipelineOptions.PARALLELISM_OVERRIDES));
-
- autoscalerInfo.setCurrentOverrides(Map.of(v1, "1", v2, "2"));
- autoscalerInfo.replaceInKubernetes(kubernetesClient);
- autoscaler.scale(ctx);
-
- assertEquals(Map.of(v1, "1", v2, "2"), autoscaler.getParallelismOverrides(ctx));
- assertEquals(
- Map.of(v1, "1", v2, "2"),
- ctx.getDeployConfig(app.getSpec()).get(PipelineOptions.PARALLELISM_OVERRIDES));
-
- ctx = getResourceContext(app = ReconciliationUtils.clone(appClone));
- app.getSpec().getFlinkConfiguration().put(SCALING_ENABLED.key(), "false");
-
- autoscaler.scale(ctx);
- assertEquals(Map.of(v1, "1", v2, "2"), autoscaler.getParallelismOverrides(ctx));
- assertEquals(
- Map.of(v1, "1", v2, "2"),
- ctx.getDeployConfig(app.getSpec()).get(PipelineOptions.PARALLELISM_OVERRIDES));
-
- // Test error handling
- // Invalid config
- ctx = getResourceContext(app = ReconciliationUtils.clone(appClone));
- app.getSpec().getFlinkConfiguration().put(AUTOSCALER_ENABLED.key(), "asd");
- autoscaler.scale(ctx);
- assertEquals(
- Map.of(v1, "1", v2, "2"),
- ctx.getDeployConfig(app.getSpec()).get(PipelineOptions.PARALLELISM_OVERRIDES));
- }
-
- @Test
- public void testApplyAutoscalerParallelism() {
- var overrides = new HashMap<String, String>();
- var autoscaler =
- new JobAutoScalerImpl(null, null, null, eventRecorder) {
- public Map<String, String> getParallelismOverrides(
- FlinkResourceContext<?> ctx) {
- return new HashMap<>(overrides);
- }
- };
-
- var deployment = TestUtils.buildApplicationCluster();
- var specClone = ReconciliationUtils.clone(deployment.getSpec());
-
- // Verify no spec change if overrides are empty
- autoscaler.applyParallelismOverrides(getResourceContext(deployment));
- assertEquals(specClone, deployment.getSpec());
-
- // Make sure overrides are applied to the spec
- var v1 = new JobVertexID();
- overrides.put(v1.toHexString(), "2");
-
- // Verify no upgrades if overrides are empty
- autoscaler.applyParallelismOverrides(getResourceContext(deployment));
- assertEquals(
- Map.of(v1.toHexString(), "2"),
- getResourceContext(deployment)
- .getDeployConfig(deployment.getSpec())
- .get(PipelineOptions.PARALLELISM_OVERRIDES));
-
- // We set a user override for v1, it should be ignored and the autoscaler override should
- // take precedence
- specClone = ReconciliationUtils.clone(deployment.getSpec());
- deployment
- .getSpec()
- .getFlinkConfiguration()
- .put(PipelineOptions.PARALLELISM_OVERRIDES.key(), v1 + ":1");
- autoscaler.applyParallelismOverrides(getResourceContext(deployment));
- assertEquals(specClone, deployment.getSpec());
-
- // Define partly overlapping overrides, user overrides for new vertices should be applied
- var v2 = new JobVertexID();
- deployment
- .getSpec()
- .getFlinkConfiguration()
- .put(PipelineOptions.PARALLELISM_OVERRIDES.key(), v1 + ":1," + v2 + ":4");
- autoscaler.applyParallelismOverrides(getResourceContext(deployment));
- assertEquals(
- Map.of(v1.toString(), "2", v2.toString(), "4"),
- getResourceContext(deployment)
- .getDeployConfig(deployment.getSpec())
- .get(PipelineOptions.PARALLELISM_OVERRIDES));
-
- // Make sure user overrides apply to excluded vertices
- deployment
- .getSpec()
- .getFlinkConfiguration()
- .put(AutoScalerOptions.VERTEX_EXCLUDE_IDS.key(), v1.toString());
- deployment
- .getSpec()
- .getFlinkConfiguration()
- .put(PipelineOptions.PARALLELISM_OVERRIDES.key(), v1 + ":1," + v2 + ":4");
- autoscaler.applyParallelismOverrides(getResourceContext(deployment));
- assertEquals(
- Map.of(v1.toString(), "1", v2.toString(), "4"),
- getResourceContext(deployment)
- .getDeployConfig(deployment.getSpec())
- .get(PipelineOptions.PARALLELISM_OVERRIDES));
- }
-}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java
deleted file mode 100644
index 9f3cb451..00000000
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
-import org.apache.flink.kubernetes.operator.TestingFlinkService;
-import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.List;
-import java.util.Map;
-
-/** Tests for RestApiMetrisCollector. */
-public class RestApiMetricsCollectorTest {
-
- @Test
- public void testAggregateMultiplePendingRecordsMetricsPerSource() {
- var collector = new RestApiMetricsCollector();
-
- JobVertexID jobVertexID = new JobVertexID();
- Map<String, FlinkMetric> flinkMetrics =
- Map.of(
- "a.pendingRecords", FlinkMetric.PENDING_RECORDS,
- "b.pendingRecords", FlinkMetric.PENDING_RECORDS);
- Map<JobVertexID, Map<String, FlinkMetric>> metrics = Map.of(jobVertexID, flinkMetrics);
-
- FlinkDeployment cr = new FlinkDeployment();
- cr.getStatus().getJobStatus().setJobId(new JobID().toHexString());
-
- TestingFlinkService flinkService = new TestingFlinkService();
- flinkService.setAggregatedMetricsResponse(
- List.of(
- new AggregatedMetric(
- "a.pendingRecords", Double.NaN, Double.NaN, Double.NaN, 100.),
- new AggregatedMetric(
- "b.pendingRecords", Double.NaN, Double.NaN, Double.NaN, 100.),
- new AggregatedMetric(
- "c.unrelated", Double.NaN, Double.NaN, Double.NaN, 100.)));
-
- Configuration conf = new Configuration();
- conf.set(KubernetesConfigOptions.CLUSTER_ID, "id");
-
- Map<JobVertexID, Map<FlinkMetric, AggregatedMetric>> jobVertexIDMapMap =
- collector.queryAllAggregatedMetrics(cr, flinkService, conf, metrics);
-
- System.out.println(jobVertexIDMapMap);
-
- Assertions.assertEquals(1, jobVertexIDMapMap.size());
- Map<FlinkMetric, AggregatedMetric> vertexMetrics = jobVertexIDMapMap.get(jobVertexID);
- Assertions.assertNotNull(vertexMetrics);
- AggregatedMetric pendingRecordsMetric = vertexMetrics.get(FlinkMetric.PENDING_RECORDS);
- Assertions.assertNotNull(pendingRecordsMetric);
- Assertions.assertEquals(pendingRecordsMetric.getSum(), 200);
- }
-}
diff --git a/flink-kubernetes-operator-autoscaler/src/test/resources/log4j2-test.properties b/flink-kubernetes-operator-autoscaler/src/test/resources/log4j2-test.properties
deleted file mode 100644
index 47b66644..00000000
--- a/flink-kubernetes-operator-autoscaler/src/test/resources/log4j2-test.properties
+++ /dev/null
@@ -1,26 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-rootLogger.level = INFO
-rootLogger.appenderRef.console.ref = ConsoleAppender
-
-# Log all infos to the console
-appender.console.name = ConsoleAppender
-appender.console.type = CONSOLE
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %style{%d}{yellow} %style{%-30c{1.}}{cyan} %highlight{[%-5level]%notEmpty{[%X{resource.namespace}/}%notEmpty{%X{resource.name}]} %msg%n%throwable}
diff --git a/flink-kubernetes-operator/pom.xml b/flink-kubernetes-operator/pom.xml
index c4722bfc..716cd2d2 100644
--- a/flink-kubernetes-operator/pom.xml
+++ b/flink-kubernetes-operator/pom.xml
@@ -49,6 +49,12 @@ under the License.
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-autoscaler</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-httpclient-okhttp</artifactId>
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java
index f6333153..3dbf4fee 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java
@@ -25,6 +25,7 @@ import org.apache.flink.core.plugin.PluginUtils;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
import org.apache.flink.kubernetes.operator.api.FlinkSessionJob;
import org.apache.flink.kubernetes.operator.api.listener.FlinkResourceListener;
+import org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFactory;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.config.FlinkOperatorConfiguration;
import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
@@ -43,7 +44,6 @@ import org.apache.flink.kubernetes.operator.observer.sessionjob.FlinkSessionJobO
import org.apache.flink.kubernetes.operator.reconciler.deployment.ReconcilerFactory;
import org.apache.flink.kubernetes.operator.reconciler.sessionjob.SessionJobReconciler;
import org.apache.flink.kubernetes.operator.service.FlinkResourceContextFactory;
-import org.apache.flink.kubernetes.operator.utils.AutoscalerLoader;
import org.apache.flink.kubernetes.operator.utils.EnvUtils;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.kubernetes.operator.utils.KubernetesClientUtils;
@@ -165,10 +165,9 @@ public class FlinkOperator {
var metricManager =
MetricManager.createFlinkDeploymentMetricManager(baseConfig, metricGroup);
var statusRecorder = StatusRecorder.create(client, metricManager, listeners);
- var autoscalerFactory = AutoscalerLoader.loadJobAutoscalerFactory();
+ var autoscaler = AutoscalerFactory.create(client, eventRecorder);
var reconcilerFactory =
- new ReconcilerFactory(
- configManager, eventRecorder, statusRecorder, autoscalerFactory);
+ new ReconcilerFactory(configManager, eventRecorder, statusRecorder, autoscaler);
var observerFactory = new FlinkDeploymentObserverFactory(eventRecorder);
var canaryResourceManager = new CanaryResourceManager<FlinkDeployment>(configManager);
HealthProbe.INSTANCE.registerCanaryResourceManager(canaryResourceManager);
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactory.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactory.java
new file mode 100644
index 00000000..f9816533
--- /dev/null
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactory.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.autoscaler.JobAutoScaler;
+import org.apache.flink.autoscaler.JobAutoScalerImpl;
+import org.apache.flink.autoscaler.RestApiMetricsCollector;
+import org.apache.flink.autoscaler.ScalingExecutor;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+
+/** The factory of {@link JobAutoScaler}. */
+public class AutoscalerFactory {
+
+ public static JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> create(
+ KubernetesClient client, EventRecorder eventRecorder) {
+
+ KubernetesAutoScalerStateStore stateStore =
+ new KubernetesAutoScalerStateStore(new ConfigMapStore(client));
+ KubernetesAutoScalerEventHandler eventHandler =
+ new KubernetesAutoScalerEventHandler(eventRecorder);
+
+ return new JobAutoScalerImpl<>(
+ new RestApiMetricsCollector<>(),
+ new ScalingMetricEvaluator(),
+ new ScalingExecutor<>(eventHandler, stateStore),
+ eventHandler,
+ new KubernetesScalingRealizer(),
+ stateStore);
+ }
+}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStore.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStore.java
new file mode 100644
index 00000000..2a60d4af
--- /dev/null
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStore.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.kubernetes.utils.Constants;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.api.model.HasMetadata;
+import io.fabric8.kubernetes.api.model.ObjectMeta;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** The ConfigMapStore persists state in Kubernetes ConfigMaps. */
+public class ConfigMapStore {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KubernetesAutoScalerStateStore.class);
+
+ private static final String LABEL_COMPONENT_AUTOSCALER = "autoscaler";
+
+ private final KubernetesClient kubernetesClient;
+
+ // The cache for each resourceId may be in three situations:
+ // 1. The resourceId isn't exist : ConfigMap isn't loaded from kubernetes, or it's removed.
+ // 2. The resourceId is exist, and value is the Optional.empty() : We have loaded the ConfigMap
+ // from kubernetes, but the ConfigMap isn't created at kubernetes side.
+ // 3. The resourceId is exist, and the Optional isn't empty : We have loaded the ConfigMap from
+ // kubernetes, it may be not same with kubernetes side due to it's not flushed after updating.
+ private final ConcurrentHashMap<ResourceID, Optional<ConfigMap>> cache =
+ new ConcurrentHashMap<>();
+
+ public ConfigMapStore(KubernetesClient kubernetesClient) {
+ this.kubernetesClient = kubernetesClient;
+ }
+
+ protected void putSerializedState(
+ KubernetesJobAutoScalerContext jobContext, String key, String value) {
+ getOrCreateState(jobContext).put(key, value);
+ }
+
+ protected Optional<String> getSerializedState(
+ KubernetesJobAutoScalerContext jobContext, String key) {
+ return getConfigMap(jobContext).map(configMap -> configMap.getData().get(key));
+ }
+
+ protected void removeSerializedState(KubernetesJobAutoScalerContext jobContext, String key) {
+ getConfigMap(jobContext)
+ .ifPresentOrElse(
+ configMap -> configMap.getData().remove(key),
+ () -> {
+ throw new IllegalStateException(
+ "The configMap isn't created, so the remove is unavailable.");
+ });
+ }
+
+ public void flush(KubernetesJobAutoScalerContext jobContext) {
+ Optional<ConfigMap> configMapOpt = cache.get(jobContext.getJobKey());
+ if (configMapOpt == null || configMapOpt.isEmpty()) {
+ LOG.debug("The configMap isn't updated, so skip the flush.");
+ return;
+ }
+ try {
+ cache.put(
+ jobContext.getJobKey(),
+ Optional.of(kubernetesClient.resource(configMapOpt.get()).update()));
+ } catch (Exception e) {
+ LOG.error(
+ "Error while updating autoscaler info configmap, invalidating to clear the cache",
+ e);
+ removeInfoFromCache(jobContext.getJobKey());
+ throw e;
+ }
+ }
+
+ public void removeInfoFromCache(ResourceID resourceID) {
+ cache.remove(resourceID);
+ }
+
+ private Optional<ConfigMap> getConfigMap(KubernetesJobAutoScalerContext jobContext) {
+ return cache.computeIfAbsent(
+ jobContext.getJobKey(), (id) -> getConfigMapFromKubernetes(jobContext));
+ }
+
+ private Map<String, String> getOrCreateState(KubernetesJobAutoScalerContext jobContext) {
+ return cache.compute(
+ jobContext.getJobKey(),
+ (id, configMapOpt) -> {
+ // If in the cache and valid simply return
+ if (configMapOpt != null && configMapOpt.isPresent()) {
+ return configMapOpt;
+ }
+ // Otherwise get or create
+ return Optional.of(getOrCreateConfigMapFromKubernetes(jobContext));
+ })
+ .get()
+ .getData();
+ }
+
+ @VisibleForTesting
+ protected Optional<ConfigMap> getConfigMapFromKubernetes(
+ KubernetesJobAutoScalerContext jobContext) {
+ HasMetadata cr = jobContext.getResource();
+ var meta = createCmObjectMeta(ResourceID.fromResource(cr));
+ return getScalingInfoConfigMap(meta);
+ }
+
+ @Nonnull
+ private ConfigMap getOrCreateConfigMapFromKubernetes(
+ KubernetesJobAutoScalerContext jobContext) {
+ HasMetadata cr = jobContext.getResource();
+ var meta = createCmObjectMeta(ResourceID.fromResource(cr));
+ return getScalingInfoConfigMap(meta).orElseGet(() -> createConfigMap(cr, meta));
+ }
+
+ private ConfigMap createConfigMap(HasMetadata cr, ObjectMeta meta) {
+ LOG.info("Creating scaling info config map");
+ var cm = new ConfigMap();
+ cm.setMetadata(meta);
+ cm.addOwnerReference(cr);
+ cm.setData(new HashMap<>());
+ return kubernetesClient.resource(cm).create();
+ }
+
+ private ObjectMeta createCmObjectMeta(ResourceID uid) {
+ var objectMeta = new ObjectMeta();
+ objectMeta.setName("autoscaler-" + uid.getName());
+ uid.getNamespace().ifPresent(objectMeta::setNamespace);
+ objectMeta.setLabels(
+ Map.of(
+ Constants.LABEL_COMPONENT_KEY,
+ LABEL_COMPONENT_AUTOSCALER,
+ Constants.LABEL_APP_KEY,
+ uid.getName()));
+ return objectMeta;
+ }
+
+ private Optional<ConfigMap> getScalingInfoConfigMap(ObjectMeta objectMeta) {
+ return Optional.ofNullable(
+ kubernetesClient
+ .configMaps()
+ .inNamespace(objectMeta.getNamespace())
+ .withName(objectMeta.getName())
+ .get());
+ }
+
+ @VisibleForTesting
+ protected ConcurrentHashMap<ResourceID, Optional<ConfigMap>> getCache() {
+ return cache;
+ }
+}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerEventHandler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerEventHandler.java
new file mode 100644
index 00000000..54fe84ae
--- /dev/null
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerEventHandler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.autoscaler.event.AutoScalerEventHandler;
+import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+
+/** An event handler which posts events to the Kubernetes events API. */
+public class KubernetesAutoScalerEventHandler
+ implements AutoScalerEventHandler<ResourceID, KubernetesJobAutoScalerContext> {
+
+ private final EventRecorder eventRecorder;
+
+ public KubernetesAutoScalerEventHandler(EventRecorder eventRecorder) {
+ this.eventRecorder = eventRecorder;
+ }
+
+ @Override
+ public void handleEvent(
+ KubernetesJobAutoScalerContext context,
+ Type type,
+ String reason,
+ String message,
+ @Nullable String messageKey,
+ @Nullable Duration interval) {
+ if (interval == null) {
+ eventRecorder.triggerEvent(
+ context.getResource(),
+ EventRecorder.Type.valueOf(type.name()),
+ reason,
+ message,
+ EventRecorder.Component.Operator,
+ messageKey,
+ context.getKubernetesClient());
+ } else {
+ eventRecorder.triggerEventByInterval(
+ context.getResource(),
+ EventRecorder.Type.valueOf(type.name()),
+ reason,
+ EventRecorder.Component.Operator,
+ message,
+ messageKey,
+ context.getKubernetesClient(),
+ interval);
+ }
+ }
+}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStore.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStore.java
new file mode 100644
index 00000000..2b446cbe
--- /dev/null
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStore.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.autoscaler.state.AutoScalerStateStore;
+import org.apache.flink.autoscaler.utils.AutoScalerSerDeModule;
+import org.apache.flink.configuration.ConfigurationUtils;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.LoaderOptions;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.Map;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+/** An AutoscalerStateStore which persists its state in Kubernetes ConfigMaps. */
+public class KubernetesAutoScalerStateStore
+ implements AutoScalerStateStore<ResourceID, KubernetesJobAutoScalerContext> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KubernetesAutoScalerStateStore.class);
+
+ @VisibleForTesting protected static final String SCALING_HISTORY_KEY = "scalingHistory";
+
+ @VisibleForTesting protected static final String COLLECTED_METRICS_KEY = "collectedMetrics";
+
+ @VisibleForTesting
+ protected static final String PARALLELISM_OVERRIDES_KEY = "parallelismOverrides";
+
+ @VisibleForTesting protected static final int MAX_CM_BYTES = 1000000;
+
+ protected static final ObjectMapper YAML_MAPPER =
+ new ObjectMapper(yamlFactory())
+ .registerModule(new JavaTimeModule())
+ .registerModule(new AutoScalerSerDeModule());
+
+ private final ConfigMapStore configMapStore;
+
+ public KubernetesAutoScalerStateStore(ConfigMapStore configMapStore) {
+ this.configMapStore = configMapStore;
+ }
+
+ @Override
+ public void storeScalingHistory(
+ KubernetesJobAutoScalerContext jobContext,
+ Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory) {
+ configMapStore.putSerializedState(
+ jobContext, SCALING_HISTORY_KEY, serializeScalingHistory(scalingHistory));
+ }
+
+ @Override
+ public Optional<Map<JobVertexID, SortedMap<Instant, ScalingSummary>>> getScalingHistory(
+ KubernetesJobAutoScalerContext jobContext) {
+ Optional<String> serializedScalingHistory =
+ configMapStore.getSerializedState(jobContext, SCALING_HISTORY_KEY);
+ if (serializedScalingHistory.isEmpty()) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.of(deserializeScalingHistory(serializedScalingHistory.get()));
+ } catch (JacksonException e) {
+ LOG.error(
+ "Could not deserialize scaling history, possibly the format changed. Discarding...",
+ e);
+ configMapStore.removeSerializedState(jobContext, SCALING_HISTORY_KEY);
+ return Optional.empty();
+ }
+ }
+
+ @Override
+ public void removeScalingHistory(KubernetesJobAutoScalerContext jobContext) {
+ configMapStore.removeSerializedState(jobContext, SCALING_HISTORY_KEY);
+ }
+
+ @Override
+ public void storeEvaluatedMetrics(
+ KubernetesJobAutoScalerContext jobContext,
+ SortedMap<Instant, CollectedMetrics> evaluatedMetrics) {
+ configMapStore.putSerializedState(
+ jobContext, COLLECTED_METRICS_KEY, serializeEvaluatedMetrics(evaluatedMetrics));
+ }
+
+ @Override
+ public Optional<SortedMap<Instant, CollectedMetrics>> getEvaluatedMetrics(
+ KubernetesJobAutoScalerContext jobContext) {
+ Optional<String> serializedEvaluatedMetricsOpt =
+ configMapStore.getSerializedState(jobContext, COLLECTED_METRICS_KEY);
+ if (serializedEvaluatedMetricsOpt.isEmpty()) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.of(deserializeEvaluatedMetrics(serializedEvaluatedMetricsOpt.get()));
+ } catch (JacksonException e) {
+ LOG.error(
+ "Could not deserialize metric history, possibly the format changed. Discarding...",
+ e);
+ configMapStore.removeSerializedState(jobContext, COLLECTED_METRICS_KEY);
+ return Optional.empty();
+ }
+ }
+
+ @Override
+ public void removeEvaluatedMetrics(KubernetesJobAutoScalerContext jobContext) {
+ configMapStore.removeSerializedState(jobContext, COLLECTED_METRICS_KEY);
+ }
+
+ @Override
+ public void storeParallelismOverrides(
+ KubernetesJobAutoScalerContext jobContext, Map<String, String> parallelismOverrides) {
+ configMapStore.putSerializedState(
+ jobContext,
+ PARALLELISM_OVERRIDES_KEY,
+ serializeParallelismOverrides(parallelismOverrides));
+ }
+
+ @Override
+ public Optional<Map<String, String>> getParallelismOverrides(
+ KubernetesJobAutoScalerContext jobContext) {
+ return configMapStore
+ .getSerializedState(jobContext, PARALLELISM_OVERRIDES_KEY)
+ .map(KubernetesAutoScalerStateStore::deserializeParallelismOverrides);
+ }
+
+ @Override
+ public void removeParallelismOverrides(KubernetesJobAutoScalerContext jobContext) {
+ configMapStore.removeSerializedState(jobContext, PARALLELISM_OVERRIDES_KEY);
+ }
+
+ @Override
+ public void flush(KubernetesJobAutoScalerContext jobContext) {
+ trimHistoryToMaxCmSize(jobContext);
+ configMapStore.flush(jobContext);
+ }
+
+ @Override
+ public void removeInfoFromCache(ResourceID resourceID) {
+ configMapStore.removeInfoFromCache(resourceID);
+ }
+
+ @SneakyThrows
+ protected static String serializeScalingHistory(
+ Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory) {
+ return compress(YAML_MAPPER.writeValueAsString(scalingHistory));
+ }
+
+ private static Map<JobVertexID, SortedMap<Instant, ScalingSummary>> deserializeScalingHistory(
+ String scalingHistory) throws JacksonException {
+ return YAML_MAPPER.readValue(decompress(scalingHistory), new TypeReference<>() {});
+ }
+
+ @VisibleForTesting
+ @SneakyThrows
+ protected static String serializeEvaluatedMetrics(
+ SortedMap<Instant, CollectedMetrics> evaluatedMetrics) {
+ return compress(YAML_MAPPER.writeValueAsString(evaluatedMetrics));
+ }
+
+ private static SortedMap<Instant, CollectedMetrics> deserializeEvaluatedMetrics(
+ String evaluatedMetrics) throws JacksonException {
+ return YAML_MAPPER.readValue(decompress(evaluatedMetrics), new TypeReference<>() {});
+ }
+
+ private static String serializeParallelismOverrides(Map<String, String> overrides) {
+ return ConfigurationUtils.convertValue(overrides, String.class);
+ }
+
+ private static Map<String, String> deserializeParallelismOverrides(String overrides) {
+ return ConfigurationUtils.convertValue(overrides, Map.class);
+ }
+
+ @VisibleForTesting
+ protected void trimHistoryToMaxCmSize(KubernetesJobAutoScalerContext context) {
+ int scalingHistorySize =
+ configMapStore
+ .getSerializedState(context, SCALING_HISTORY_KEY)
+ .map(String::length)
+ .orElse(0);
+ int metricHistorySize =
+ configMapStore
+ .getSerializedState(context, COLLECTED_METRICS_KEY)
+ .map(String::length)
+ .orElse(0);
+
+ Optional<SortedMap<Instant, CollectedMetrics>> evaluatedMetricsOpt =
+ getEvaluatedMetrics(context);
+ if (evaluatedMetricsOpt.isEmpty()) {
+ return;
+ }
+ SortedMap<Instant, CollectedMetrics> metricHistory = evaluatedMetricsOpt.get();
+ while (scalingHistorySize + metricHistorySize > MAX_CM_BYTES) {
+ if (metricHistory.isEmpty()) {
+ return;
+ }
+ var firstKey = metricHistory.firstKey();
+ LOG.info("Trimming metric history by removing {}", firstKey);
+ metricHistory.remove(firstKey);
+ String compressed = serializeEvaluatedMetrics(metricHistory);
+ configMapStore.putSerializedState(context, COLLECTED_METRICS_KEY, compressed);
+ metricHistorySize = compressed.length();
+ }
+ }
+
+ private static String compress(String original) throws IOException {
+ ByteArrayOutputStream rstBao = new ByteArrayOutputStream();
+ try (var zos = new GZIPOutputStream(rstBao)) {
+ zos.write(original.getBytes(StandardCharsets.UTF_8));
+ }
+
+ return Base64.getEncoder().encodeToString(rstBao.toByteArray());
+ }
+
+ private static String decompress(String compressed) {
+ if (compressed == null) {
+ return null;
+ }
+
+ try {
+ byte[] bytes = Base64.getDecoder().decode(compressed);
+ try (var zi = new GZIPInputStream(new ByteArrayInputStream(bytes))) {
+ return IOUtils.toString(zi, StandardCharsets.UTF_8);
+ }
+ } catch (Exception e) {
+ LOG.warn("Error while decompressing scaling data, treating as uncompressed");
+ // Fall back to non-compressed for migration
+ return compressed;
+ }
+ }
+
+ private static YAMLFactory yamlFactory() {
+ // Set yaml size limit to 10mb
+ var loaderOptions = new LoaderOptions();
+ loaderOptions.setCodePointLimit(20 * 1024 * 1024);
+ return YAMLFactory.builder().loaderOptions(loaderOptions).build();
+ }
+}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesJobAutoScalerContext.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesJobAutoScalerContext.java
new file mode 100644
index 00000000..b50f8a7a
--- /dev/null
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesJobAutoScalerContext.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+import org.apache.flink.client.program.rest.RestClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.util.function.SupplierWithException;
+
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+
+import javax.annotation.Nullable;
+
+/** An implementation of JobAutoscalerContext for Kubernetes. */
+public class KubernetesJobAutoScalerContext extends JobAutoScalerContext<ResourceID> {
+
+ private final AbstractFlinkResource<?, ?> resource;
+
+ private final KubernetesClient kubernetesClient;
+
+ public KubernetesJobAutoScalerContext(
+ @Nullable JobID jobID,
+ @Nullable JobStatus jobStatus,
+ Configuration configuration,
+ MetricGroup metricGroup,
+ SupplierWithException<RestClusterClient<String>, Exception> restClientSupplier,
+ AbstractFlinkResource<?, ?> resource,
+ KubernetesClient kubernetesClient) {
+ super(
+ ResourceID.fromResource(resource),
+ jobID,
+ jobStatus,
+ configuration,
+ metricGroup,
+ restClientSupplier);
+ this.resource = resource;
+ this.kubernetesClient = kubernetesClient;
+ }
+
+ public AbstractFlinkResource<?, ?> getResource() {
+ return resource;
+ }
+
+ public KubernetesClient getKubernetesClient() {
+ return kubernetesClient;
+ }
+}
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesScalingRealizer.java
similarity index 50%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java
rename to flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesScalingRealizer.java
index e0934a30..ca3f4ef2 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesScalingRealizer.java
@@ -17,25 +17,26 @@
package org.apache.flink.kubernetes.operator.autoscaler;
-import org.apache.flink.autoscaler.ScalingMetricEvaluator;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScaler;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScalerFactory;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+import org.apache.flink.autoscaler.realizer.ScalingRealizer;
+import org.apache.flink.configuration.ConfigurationUtils;
+import org.apache.flink.configuration.PipelineOptions;
-import com.google.auto.service.AutoService;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+
+import java.util.Map;
+
+/** The Kubernetes implementation for applying parallelism overrides. */
+public class KubernetesScalingRealizer
+ implements ScalingRealizer<ResourceID, KubernetesJobAutoScalerContext> {
-/**
- * Factory for loading JobAutoScalerImpl included in this module. This class will be dynamically
- * instantiated by the main operator module.
- */
-@AutoService(JobAutoScalerFactory.class)
-public class JobAutoscalerFactoryImpl implements JobAutoScalerFactory {
@Override
- public JobAutoScaler create(EventRecorder eventRecorder) {
- return new JobAutoScalerImpl(
- new RestApiMetricsCollector(),
- new ScalingMetricEvaluator(),
- new ScalingExecutor(eventRecorder),
- eventRecorder);
+ public void realize(
+ KubernetesJobAutoScalerContext context, Map<String, String> parallelismOverrides) {
+ context.getResource()
+ .getSpec()
+ .getFlinkConfiguration()
+ .put(
+ PipelineOptions.PARALLELISM_OVERRIDES.key(),
+ ConfigurationUtils.convertValue(parallelismOverrides, String.class));
}
}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkResourceContext.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkResourceContext.java
index 08286a6c..ab967e7c 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkResourceContext.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkResourceContext.java
@@ -17,11 +17,17 @@
package org.apache.flink.kubernetes.operator.controller;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
+import org.apache.flink.kubernetes.operator.api.lifecycle.ResourceLifecycleState;
import org.apache.flink.kubernetes.operator.api.spec.AbstractFlinkSpec;
import org.apache.flink.kubernetes.operator.api.spec.FlinkVersion;
import org.apache.flink.kubernetes.operator.api.spec.KubernetesDeploymentMode;
+import org.apache.flink.kubernetes.operator.api.status.CommonStatus;
+import org.apache.flink.kubernetes.operator.autoscaler.KubernetesJobAutoScalerContext;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.config.FlinkOperatorConfiguration;
import org.apache.flink.kubernetes.operator.metrics.KubernetesResourceMetricGroup;
@@ -32,6 +38,8 @@ import io.javaoperatorsdk.operator.api.reconciler.Context;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
+import javax.annotation.Nullable;
+
import java.util.function.Function;
/** Context for reconciling a Flink resource. */
@@ -47,6 +55,48 @@ public abstract class FlinkResourceContext<CR extends AbstractFlinkResource<?, ?
private FlinkOperatorConfiguration operatorConfig;
private Configuration observeConfig;
private FlinkService flinkService;
+ private KubernetesJobAutoScalerContext autoScalerContext;
+
+ public KubernetesJobAutoScalerContext getJobAutoScalerContext() {
+ if (autoScalerContext != null) {
+ return autoScalerContext;
+ }
+ return autoScalerContext = createJobAutoScalerContext();
+ }
+
+ private KubernetesJobAutoScalerContext createJobAutoScalerContext() {
+ Configuration conf = new Configuration(getObserveConfig());
+ conf.set(
+ AutoScalerOptions.FLINK_CLIENT_TIMEOUT,
+ getOperatorConfig().getFlinkClientTimeout());
+
+ CommonStatus<?> status = getResource().getStatus();
+ String jobId = status.getJobStatus().getJobId();
+
+ JobStatus jobStatus = generateJobStatusEnum(status);
+
+ return new KubernetesJobAutoScalerContext(
+ jobId == null ? null : JobID.fromHexString(jobId),
+ jobStatus,
+ conf,
+ getResourceMetricGroup(),
+ () -> getFlinkService().getClusterClient(conf),
+ resource,
+ getKubernetesClient());
+ }
+
+ @Nullable
+ private JobStatus generateJobStatusEnum(CommonStatus<?> status) {
+ if (status.getLifecycleState() != ResourceLifecycleState.STABLE) {
+ return null;
+ }
+
+ String state = status.getJobStatus().getState();
+ if (state == null) {
+ return null;
+ }
+ return JobStatus.valueOf(state);
+ }
/**
* Get the config that is currently deployed for the resource spec. The returned config may be
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
index ffe636dc..6f39582a 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractFlinkResourceReconciler.java
@@ -18,6 +18,8 @@
package org.apache.flink.kubernetes.operator.reconciler.deployment;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.JobAutoScaler;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
@@ -33,6 +35,7 @@ import org.apache.flink.kubernetes.operator.api.status.JobManagerDeploymentStatu
import org.apache.flink.kubernetes.operator.api.status.ReconciliationState;
import org.apache.flink.kubernetes.operator.api.status.Savepoint;
import org.apache.flink.kubernetes.operator.api.status.SnapshotTriggerType;
+import org.apache.flink.kubernetes.operator.autoscaler.KubernetesJobAutoScalerContext;
import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
import org.apache.flink.kubernetes.operator.reconciler.Reconciler;
@@ -47,6 +50,7 @@ import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.javaoperatorsdk.operator.api.reconciler.DeleteControl;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +61,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
+
/**
* Base class for all Flink resource reconcilers. It contains the general flow of reconciling Flink
* related resources including initial deployments, upgrades, rollbacks etc.
@@ -72,7 +78,7 @@ public abstract class AbstractFlinkResourceReconciler<
protected final EventRecorder eventRecorder;
protected final StatusRecorder<CR, STATUS> statusRecorder;
- protected final JobAutoScaler resourceScaler;
+ protected final JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoscaler;
public static final String MSG_SUSPENDED = "Suspending existing deployment.";
public static final String MSG_SPEC_CHANGED =
@@ -85,10 +91,10 @@ public abstract class AbstractFlinkResourceReconciler<
public AbstractFlinkResourceReconciler(
EventRecorder eventRecorder,
StatusRecorder<CR, STATUS> statusRecorder,
- JobAutoScalerFactory autoscalerFactory) {
+ JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoscaler) {
this.eventRecorder = eventRecorder;
this.statusRecorder = statusRecorder;
- this.resourceScaler = autoscalerFactory.create(eventRecorder);
+ this.autoscaler = autoscaler;
}
@Override
@@ -131,7 +137,8 @@ public abstract class AbstractFlinkResourceReconciler<
SPEC lastReconciledSpec =
cr.getStatus().getReconciliationStatus().deserializeLastReconciledSpec();
SPEC currentDeploySpec = cr.getSpec();
- resourceScaler.scale(ctx);
+
+ scaling(ctx);
var reconciliationState = reconciliationStatus.getState();
var specDiff =
@@ -174,6 +181,23 @@ public abstract class AbstractFlinkResourceReconciler<
}
}
+ private void scaling(FlinkResourceContext<CR> ctx) throws Exception {
+ KubernetesJobAutoScalerContext autoScalerContext = ctx.getJobAutoScalerContext();
+
+ if (autoscalerDisabled(ctx)) {
+ autoScalerContext.getConfiguration().set(AUTOSCALER_ENABLED, false);
+ } else if (autoScalerContext.getJobStatus() != JobStatus.RUNNING) {
+ LOG.info("Autoscaler is waiting for stable, running state");
+ }
+
+ autoscaler.scale(autoScalerContext);
+ }
+
+ private boolean autoscalerDisabled(FlinkResourceContext<CR> ctx) {
+ return ctx.getResource().getSpec().getJob() == null
+ || !ctx.getObserveConfig().getBoolean(AUTOSCALER_ENABLED);
+ }
+
private void triggerSpecChangeEvent(CR cr, DiffResult<SPEC> specDiff, KubernetesClient client) {
eventRecorder.triggerEventOnce(
cr,
@@ -250,7 +274,7 @@ public abstract class AbstractFlinkResourceReconciler<
@Override
public DeleteControl cleanup(FlinkResourceContext<CR> ctx) {
- resourceScaler.cleanup(ctx);
+ autoscaler.cleanup(ResourceID.fromResource(ctx.getResource()));
return cleanupInternal(ctx);
}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractJobReconciler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractJobReconciler.java
index c12367ff..e572c587 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractJobReconciler.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/AbstractJobReconciler.java
@@ -19,6 +19,7 @@ package org.apache.flink.kubernetes.operator.reconciler.deployment;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.autoscaler.JobAutoScaler;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
import org.apache.flink.kubernetes.operator.api.spec.AbstractFlinkSpec;
@@ -26,6 +27,7 @@ import org.apache.flink.kubernetes.operator.api.spec.JobState;
import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode;
import org.apache.flink.kubernetes.operator.api.status.CommonStatus;
import org.apache.flink.kubernetes.operator.api.status.JobStatus;
+import org.apache.flink.kubernetes.operator.autoscaler.KubernetesJobAutoScalerContext;
import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
@@ -35,6 +37,7 @@ import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.kubernetes.operator.utils.SnapshotUtils;
import org.apache.flink.kubernetes.operator.utils.StatusRecorder;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
import lombok.Value;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,8 +64,8 @@ public abstract class AbstractJobReconciler<
public AbstractJobReconciler(
EventRecorder eventRecorder,
StatusRecorder<CR, STATUS> statusRecorder,
- JobAutoScalerFactory autoscalerFactory) {
- super(eventRecorder, statusRecorder, autoscalerFactory);
+ JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoscaler) {
+ super(eventRecorder, statusRecorder, autoscaler);
}
@Override
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconciler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconciler.java
index f3719fd6..5d174a3f 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconciler.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconciler.java
@@ -19,6 +19,7 @@ package org.apache.flink.kubernetes.operator.reconciler.deployment;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.JobAutoScaler;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.HighAvailabilityOptions;
import org.apache.flink.configuration.PipelineOptionsInternal;
@@ -28,6 +29,7 @@ import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentSpec;
import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode;
import org.apache.flink.kubernetes.operator.api.status.FlinkDeploymentStatus;
import org.apache.flink.kubernetes.operator.api.status.JobManagerDeploymentStatus;
+import org.apache.flink.kubernetes.operator.autoscaler.KubernetesJobAutoScalerContext;
import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
import org.apache.flink.kubernetes.operator.exception.RecoveryFailureException;
@@ -46,6 +48,7 @@ import org.apache.flink.util.Preconditions;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.javaoperatorsdk.operator.api.reconciler.DeleteControl;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,8 +70,8 @@ public class ApplicationReconciler
public ApplicationReconciler(
EventRecorder eventRecorder,
StatusRecorder<FlinkDeployment, FlinkDeploymentStatus> statusRecorder,
- JobAutoScalerFactory autoscalerFactory) {
- super(eventRecorder, statusRecorder, autoscalerFactory);
+ JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoscaler) {
+ super(eventRecorder, statusRecorder, autoscaler);
}
@Override
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScaler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScaler.java
deleted file mode 100644
index f3e2790a..00000000
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/JobAutoScaler.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.reconciler.deployment;
-
-import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
-
-/** Per-job Autoscaler instance. */
-public interface JobAutoScaler {
-
- /** Called as part of the reconciliation loop. Returns true if this call led to scaling. */
- void scale(FlinkResourceContext<?> ctx) throws Exception;
-
- /** Called when the custom resource is deleted. */
- void cleanup(FlinkResourceContext<?> ctx);
-}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/NoopJobAutoscalerFactory.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/NoopJobAutoscalerFactory.java
deleted file mode 100644
index 7b022ec4..00000000
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/NoopJobAutoscalerFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.reconciler.deployment;
-
-import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
-import org.apache.flink.kubernetes.operator.utils.EventRecorder;
-
-/** An autoscaler implementation which does nothing. */
-public class NoopJobAutoscalerFactory implements JobAutoScalerFactory, JobAutoScaler {
-
- @Override
- public JobAutoScaler create(EventRecorder eventRecorder) {
- return this;
- }
-
- @Override
- public void scale(FlinkResourceContext<?> ctx) {}
-
- @Override
- public void cleanup(FlinkResourceContext<?> ctx) {}
-}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ReconcilerFactory.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ReconcilerFactory.java
index 85aede5a..beba19dc 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ReconcilerFactory.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ReconcilerFactory.java
@@ -18,15 +18,19 @@
package org.apache.flink.kubernetes.operator.reconciler.deployment;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.autoscaler.JobAutoScaler;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
import org.apache.flink.kubernetes.operator.api.spec.KubernetesDeploymentMode;
import org.apache.flink.kubernetes.operator.api.status.FlinkDeploymentStatus;
+import org.apache.flink.kubernetes.operator.autoscaler.KubernetesJobAutoScalerContext;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.config.Mode;
import org.apache.flink.kubernetes.operator.reconciler.Reconciler;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.kubernetes.operator.utils.StatusRecorder;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -36,7 +40,7 @@ public class ReconcilerFactory {
private final FlinkConfigManager configManager;
private final EventRecorder eventRecorder;
private final StatusRecorder<FlinkDeployment, FlinkDeploymentStatus> deploymentStatusRecorder;
- private final JobAutoScalerFactory autoscalerFactory;
+ private final JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoscaler;
private final Map<Tuple2<Mode, KubernetesDeploymentMode>, Reconciler<FlinkDeployment>>
reconcilerMap;
@@ -44,11 +48,11 @@ public class ReconcilerFactory {
FlinkConfigManager configManager,
EventRecorder eventRecorder,
StatusRecorder<FlinkDeployment, FlinkDeploymentStatus> deploymentStatusRecorder,
- JobAutoScalerFactory autoscalerFactory) {
+ JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoscaler) {
this.configManager = configManager;
this.eventRecorder = eventRecorder;
this.deploymentStatusRecorder = deploymentStatusRecorder;
- this.autoscalerFactory = autoscalerFactory;
+ this.autoscaler = autoscaler;
this.reconcilerMap = new ConcurrentHashMap<>();
}
@@ -63,7 +67,7 @@ public class ReconcilerFactory {
return new SessionReconciler(eventRecorder, deploymentStatusRecorder);
case APPLICATION:
return new ApplicationReconciler(
- eventRecorder, deploymentStatusRecorder, autoscalerFactory);
+ eventRecorder, deploymentStatusRecorder, autoscaler);
default:
throw new UnsupportedOperationException(
String.format("Unsupported running mode: %s", modes.f0));
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/SessionReconciler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/SessionReconciler.java
index 335f6259..b0ebdce4 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/SessionReconciler.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/deployment/SessionReconciler.java
@@ -17,6 +17,7 @@
package org.apache.flink.kubernetes.operator.reconciler.deployment;
+import org.apache.flink.autoscaler.NoopJobAutoscaler;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
import org.apache.flink.kubernetes.operator.api.FlinkSessionJob;
@@ -50,7 +51,7 @@ public class SessionReconciler
public SessionReconciler(
EventRecorder eventRecorder,
StatusRecorder<FlinkDeployment, FlinkDeploymentStatus> statusRecorder) {
- super(eventRecorder, statusRecorder, new NoopJobAutoscalerFactory());
+ super(eventRecorder, statusRecorder, new NoopJobAutoscaler<>());
}
@Override
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/sessionjob/SessionJobReconciler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/sessionjob/SessionJobReconciler.java
index 9521d841..16edab6b 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/sessionjob/SessionJobReconciler.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/sessionjob/SessionJobReconciler.java
@@ -17,6 +17,7 @@
package org.apache.flink.kubernetes.operator.reconciler.sessionjob;
+import org.apache.flink.autoscaler.NoopJobAutoscaler;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
import org.apache.flink.kubernetes.operator.api.FlinkSessionJob;
@@ -27,7 +28,6 @@ import org.apache.flink.kubernetes.operator.api.status.JobManagerDeploymentStatu
import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
import org.apache.flink.kubernetes.operator.reconciler.deployment.AbstractJobReconciler;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.NoopJobAutoscalerFactory;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.kubernetes.operator.utils.StatusRecorder;
import org.apache.flink.runtime.messages.FlinkJobNotFoundException;
@@ -49,7 +49,7 @@ public class SessionJobReconciler
public SessionJobReconciler(
EventRecorder eventRecorder,
StatusRecorder<FlinkSessionJob, FlinkSessionJobStatus> statusRecorder) {
- super(eventRecorder, statusRecorder, new NoopJobAutoscalerFactory());
+ super(eventRecorder, statusRecorder, new NoopJobAutoscaler<>());
}
@Override
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoader.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoader.java
deleted file mode 100644
index 37bac405..00000000
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoader.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.utils;
-
-import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScalerFactory;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.NoopJobAutoscalerFactory;
-import org.apache.flink.util.Preconditions;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ServiceLoader;
-
-/** Loads the active Autoscaler implementation from the classpath. */
-public class AutoscalerLoader {
-
- private static final Logger LOG = LoggerFactory.getLogger(AutoscalerLoader.class);
-
- public static JobAutoScalerFactory loadJobAutoscalerFactory() {
- JobAutoScalerFactory factory = null;
- boolean singleImplementation = true;
-
- for (JobAutoScalerFactory discoveredFactory :
- ServiceLoader.load(JobAutoScalerFactory.class)) {
- LOG.info(
- "Discovered JobAutoScaler factory: {}", discoveredFactory.getClass().getName());
- singleImplementation = factory == null;
- factory = discoveredFactory;
- }
-
- if (factory == null) {
- LOG.info("No JobAutoscaler implementation found. Autoscaling is disabled.");
- return new NoopJobAutoscalerFactory();
- }
-
- Preconditions.checkState(
- singleImplementation,
- "Found multiple implementation for JobAutoScalerFactory. Please ensure only one implementation is present.");
-
- return factory;
- }
-}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventRecorder.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventRecorder.java
index 09c176f4..f29b3a00 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventRecorder.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventRecorder.java
@@ -116,7 +116,7 @@ public class EventRecorder {
public boolean triggerEventByInterval(
AbstractFlinkResource<?, ?> resource,
Type type,
- Reason reason,
+ String reason,
Component component,
String message,
@Nullable String messageKey,
@@ -126,7 +126,7 @@ public class EventRecorder {
client,
resource,
type,
- reason.toString(),
+ reason,
message,
component,
e -> eventListener.accept(resource, e),
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactoryTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactoryTest.java
new file mode 100644
index 00000000..55110bce
--- /dev/null
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFactoryTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.autoscaler.JobAutoScaler;
+import org.apache.flink.autoscaler.JobAutoScalerImpl;
+import org.apache.flink.kubernetes.operator.utils.EventCollector;
+import org.apache.flink.kubernetes.operator.utils.EventRecorder;
+
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
+import lombok.Getter;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** Test loading the default autoscaling implementation from the classpath. */
+@EnableKubernetesMockClient(crud = true)
+public class AutoscalerFactoryTest {
+
+ @Getter private KubernetesClient kubernetesClient;
+
+ @Test
+ void testLoadDefaultImplementation() {
+ JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoScaler =
+ AutoscalerFactory.create(kubernetesClient, new EventRecorder(new EventCollector()));
+ Assertions.assertTrue(autoScaler instanceof JobAutoScalerImpl);
+ }
+}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStoreTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStoreTest.java
new file mode 100644
index 00000000..0e42799e
--- /dev/null
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ConfigMapStoreTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
+import io.fabric8.kubernetes.client.server.mock.KubernetesMockServer;
+import org.junit.jupiter.api.Test;
+
+import java.util.Optional;
+
+import static org.apache.flink.kubernetes.operator.autoscaler.TestingKubernetesAutoscalerUtils.createContext;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/** Test for {@link ConfigMapStore}. */
+@EnableKubernetesMockClient(crud = true)
+public class ConfigMapStoreTest {
+
+ KubernetesClient kubernetesClient;
+
+ KubernetesMockServer mockWebServer;
+
+ @Test
+ void testCaching() {
+ KubernetesJobAutoScalerContext ctx1 = createContext("cr1", kubernetesClient);
+ KubernetesJobAutoScalerContext ctx2 = createContext("cr2", kubernetesClient);
+
+ var configMapStore = new ConfigMapStore(kubernetesClient);
+ assertEquals(0, mockWebServer.getRequestCount());
+
+ String key1 = "key1";
+ String value1 = "value1";
+ String key2 = "key2";
+ String value2 = "value2";
+ String key3 = "key3";
+ String value3 = "value3";
+ assertThat(configMapStore.getSerializedState(ctx1, key1)).isEmpty();
+ assertEquals(1, mockWebServer.getRequestCount());
+
+ // Further gets should not go to K8s
+ assertThat(configMapStore.getSerializedState(ctx1, key2)).isEmpty();
+ assertThat(configMapStore.getSerializedState(ctx1, key3)).isEmpty();
+ assertEquals(1, mockWebServer.getRequestCount());
+
+ assertThat(configMapStore.getConfigMapFromKubernetes(ctx1)).isEmpty();
+ assertEquals(2, mockWebServer.getRequestCount());
+
+ configMapStore.putSerializedState(ctx1, key1, value1);
+ assertEquals(4, mockWebServer.getRequestCount());
+
+ // The put just update the data to cache, and shouldn't request kubernetes.
+ configMapStore.putSerializedState(ctx1, key2, value2);
+ assertEquals(4, mockWebServer.getRequestCount());
+
+ assertThat(configMapStore.getConfigMapFromKubernetes(ctx1)).isPresent();
+ assertThat(configMapStore.getConfigMapFromKubernetes(ctx2)).isEmpty();
+
+ assertThat(configMapStore.getSerializedState(ctx1, key1)).isPresent();
+ assertThat(configMapStore.getSerializedState(ctx2, key1)).isEmpty();
+
+ configMapStore.flush(ctx1);
+ configMapStore.putSerializedState(ctx2, key3, value3);
+ configMapStore.flush(ctx2);
+
+ configMapStore = new ConfigMapStore(kubernetesClient);
+ assertThat(configMapStore.getSerializedState(ctx1, key1)).contains(value1);
+ assertThat(configMapStore.getSerializedState(ctx2, key3)).contains(value3);
+
+ // Removing from the cache should not affect the results
+ configMapStore.removeInfoFromCache(ctx1.getJobKey());
+ assertThat(configMapStore.getSerializedState(ctx1, key1)).contains(value1);
+ assertThat(configMapStore.getSerializedState(ctx2, key3)).contains(value3);
+ }
+
+ @Test
+ void testErrorHandling() {
+ KubernetesJobAutoScalerContext ctx = createContext("cr1", kubernetesClient);
+
+ var configMapStore = new ConfigMapStore(kubernetesClient);
+ // Test for the invalid flush.
+ configMapStore.flush(ctx);
+
+ assertEquals(0, mockWebServer.getRequestCount());
+
+ configMapStore.putSerializedState(ctx, "a", "1");
+ Optional<ConfigMap> configMapOpt = configMapStore.getCache().get(ctx.getJobKey());
+ assertThat(configMapOpt).isPresent();
+ assertEquals(2, mockWebServer.getRequestCount());
+
+ // Modify the autoscaler info in the background
+ var cm = ReconciliationUtils.clone(configMapOpt.get());
+ cm.getData().put("a", "2");
+ kubernetesClient.resource(cm).update();
+
+ // Replace should throw an error due to the modification
+ assertThrows(KubernetesClientException.class, () -> configMapStore.flush(ctx));
+ assertThat(configMapStore.getCache()).isEmpty();
+
+ // Make sure we can get the new version
+ assertThat(configMapStore.getSerializedState(ctx, "a")).contains("2");
+ }
+}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStoreTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStoreTest.java
new file mode 100644
index 00000000..4618a498
--- /dev/null
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/KubernetesAutoScalerStateStoreTest.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import static org.apache.flink.autoscaler.metrics.ScalingHistoryUtils.addToScalingHistoryAndStore;
+import static org.apache.flink.autoscaler.metrics.ScalingHistoryUtils.getTrimmedScalingHistory;
+import static org.apache.flink.autoscaler.metrics.ScalingHistoryUtils.updateVertexList;
+import static org.apache.flink.kubernetes.operator.autoscaler.KubernetesAutoScalerStateStore.serializeEvaluatedMetrics;
+import static org.apache.flink.kubernetes.operator.autoscaler.KubernetesAutoScalerStateStore.serializeScalingHistory;
+import static org.apache.flink.kubernetes.operator.autoscaler.TestingKubernetesAutoscalerUtils.createContext;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/** Test for {@link KubernetesAutoScalerStateStore}. */
+@EnableKubernetesMockClient(crud = true)
+public class KubernetesAutoScalerStateStoreTest {
+
+ KubernetesClient kubernetesClient;
+
+ ConfigMapStore configMapStore;
+
+ KubernetesAutoScalerStateStore stateStore;
+
+ KubernetesJobAutoScalerContext ctx;
+
+ @BeforeEach
+ void setup() {
+ configMapStore = new ConfigMapStore(kubernetesClient);
+ stateStore = new KubernetesAutoScalerStateStore(configMapStore);
+ ctx = createContext("cr1", kubernetesClient);
+ }
+
+ @Test
+ public void testTopologyUpdate() throws Exception {
+ var v1 = new JobVertexID();
+ var v2 = new JobVertexID();
+ var v3 = new JobVertexID();
+
+ var summaries = new HashMap<JobVertexID, ScalingSummary>();
+ summaries.put(v1, new ScalingSummary(1, 2, null));
+ summaries.put(v2, new ScalingSummary(1, 2, null));
+
+ var now = Instant.now();
+
+ addToScalingHistoryAndStore(stateStore, ctx, now, summaries);
+ stateStore.flush(ctx);
+
+ Assertions.assertEquals(
+ summaries.keySet(), getTrimmedScalingHistory(stateStore, ctx, now).keySet());
+ Assertions.assertEquals(
+ summaries.keySet(),
+ getTrimmedScalingHistory(
+ new KubernetesAutoScalerStateStore(configMapStore), ctx, now)
+ .keySet());
+ Assertions.assertEquals(
+ summaries.keySet(),
+ getTrimmedScalingHistory(
+ new KubernetesAutoScalerStateStore(
+ new ConfigMapStore(kubernetesClient)),
+ ctx,
+ now)
+ .keySet());
+
+ updateVertexList(stateStore, ctx, now, Set.of(v2, v3));
+ stateStore.flush(ctx);
+
+ // Expect v1 to be removed
+ Assertions.assertEquals(
+ Set.of(v2), getTrimmedScalingHistory(stateStore, ctx, now).keySet());
+ Assertions.assertEquals(
+ Set.of(v2),
+ getTrimmedScalingHistory(
+ new KubernetesAutoScalerStateStore(configMapStore), ctx, now)
+ .keySet());
+ Assertions.assertEquals(
+ Set.of(v2),
+ getTrimmedScalingHistory(
+ new KubernetesAutoScalerStateStore(
+ new ConfigMapStore(kubernetesClient)),
+ ctx,
+ now)
+ .keySet());
+ }
+
+ @Test
+ public void testHistorySizeConfigs() throws Exception {
+ var v1 = new JobVertexID();
+
+ var history = new HashMap<JobVertexID, ScalingSummary>();
+ history.put(v1, new ScalingSummary(1, 2, null));
+
+ var conf = ctx.getConfiguration();
+ conf.set(AutoScalerOptions.VERTEX_SCALING_HISTORY_COUNT, 2);
+ conf.set(AutoScalerOptions.VERTEX_SCALING_HISTORY_AGE, Duration.ofSeconds(10));
+
+ var now = Instant.now();
+
+ // Verify count based expiration
+ addToScalingHistoryAndStore(stateStore, ctx, now, history);
+ Assertions.assertEquals(1, getTrimmedScalingHistory(stateStore, ctx, now).get(v1).size());
+
+ addToScalingHistoryAndStore(stateStore, ctx, now.plus(Duration.ofSeconds(1)), history);
+ addToScalingHistoryAndStore(stateStore, ctx, now.plus(Duration.ofSeconds(2)), history);
+
+ Assertions.assertEquals(
+ 2,
+ getTrimmedScalingHistory(stateStore, ctx, now.plus(Duration.ofSeconds(2)))
+ .get(v1)
+ .size());
+ Assertions.assertEquals(
+ Set.of(now.plus(Duration.ofSeconds(1)), now.plus(Duration.ofSeconds(2))),
+ getTrimmedScalingHistory(stateStore, ctx, now.plus(Duration.ofSeconds(2)))
+ .get(v1)
+ .keySet());
+
+ // Verify time based expiration
+ addToScalingHistoryAndStore(stateStore, ctx, now.plus(Duration.ofSeconds(15)), history);
+ stateStore.flush(ctx);
+
+ Assertions.assertEquals(
+ 1,
+ getTrimmedScalingHistory(stateStore, ctx, now.plus(Duration.ofSeconds(15)))
+ .get(v1)
+ .size());
+ Assertions.assertEquals(
+ Set.of(now.plus(Duration.ofSeconds(15))),
+ getTrimmedScalingHistory(stateStore, ctx, now.plus(Duration.ofSeconds(15)))
+ .get(v1)
+ .keySet());
+ Assertions.assertEquals(
+ Set.of(now.plus(Duration.ofSeconds(15))),
+ getTrimmedScalingHistory(
+ new KubernetesAutoScalerStateStore(configMapStore),
+ ctx,
+ now.plus(Duration.ofSeconds(15)))
+ .get(v1)
+ .keySet());
+ Assertions.assertEquals(
+ Set.of(now.plus(Duration.ofSeconds(15))),
+ getTrimmedScalingHistory(
+ new KubernetesAutoScalerStateStore(
+ new ConfigMapStore(kubernetesClient)),
+ ctx,
+ now.plus(Duration.ofSeconds(15)))
+ .get(v1)
+ .keySet());
+ }
+
+ @Test
+ public void testCompressionMigration() throws Exception {
+ var jobUpdateTs = Instant.now();
+ var v1 = new JobVertexID();
+
+ var metricHistory = new TreeMap<Instant, CollectedMetrics>();
+ metricHistory.put(
+ jobUpdateTs,
+ new CollectedMetrics(
+ Map.of(v1, Map.of(ScalingMetric.TRUE_PROCESSING_RATE, 1.)), Map.of()));
+
+ var scalingHistory = new HashMap<JobVertexID, SortedMap<Instant, ScalingSummary>>();
+ scalingHistory.put(v1, new TreeMap<>());
+ scalingHistory
+ .get(v1)
+ .put(
+ jobUpdateTs,
+ new ScalingSummary(
+ 1, 2, Map.of(ScalingMetric.LAG, EvaluatedScalingMetric.of(2.))));
+
+ // Store uncompressed data in map to simulate migration
+ configMapStore.putSerializedState(
+ ctx,
+ KubernetesAutoScalerStateStore.COLLECTED_METRICS_KEY,
+ serializeEvaluatedMetrics(metricHistory));
+ configMapStore.putSerializedState(
+ ctx,
+ KubernetesAutoScalerStateStore.SCALING_HISTORY_KEY,
+ serializeScalingHistory(scalingHistory));
+
+ var now = Instant.now();
+
+ Assertions.assertEquals(scalingHistory, getTrimmedScalingHistory(stateStore, ctx, now));
+ assertThat(stateStore.getEvaluatedMetrics(ctx)).hasValue(metricHistory);
+
+ // Override with compressed data
+ var newTs = Instant.now();
+
+ addToScalingHistoryAndStore(stateStore, ctx, newTs, Map.of());
+ stateStore.storeEvaluatedMetrics(ctx, metricHistory);
+
+ // Make sure we can still access everything
+ Assertions.assertEquals(scalingHistory, getTrimmedScalingHistory(stateStore, ctx, newTs));
+ assertThat(stateStore.getEvaluatedMetrics(ctx)).hasValue(metricHistory);
+ }
+
+ @Test
+ public void testMetricsTrimming() throws Exception {
+ var v1 = new JobVertexID();
+ Random rnd = new Random();
+
+ var metricHistory = new TreeMap<Instant, CollectedMetrics>();
+ for (int i = 0; i < 50; i++) {
+ var m = new HashMap<JobVertexID, Map<ScalingMetric, Double>>();
+ for (int j = 0; j < 500; j++) {
+ m.put(
+ new JobVertexID(),
+ Map.of(ScalingMetric.TRUE_PROCESSING_RATE, rnd.nextDouble()));
+ }
+ metricHistory.put(Instant.now(), new CollectedMetrics(m, Collections.emptyMap()));
+ }
+
+ var scalingHistory = new HashMap<JobVertexID, SortedMap<Instant, ScalingSummary>>();
+ scalingHistory.put(v1, new TreeMap<>());
+ scalingHistory
+ .get(v1)
+ .put(
+ Instant.now(),
+ new ScalingSummary(
+ 1, 2, Map.of(ScalingMetric.LAG, EvaluatedScalingMetric.of(2.))));
+
+ addToScalingHistoryAndStore(
+ stateStore,
+ ctx,
+ Instant.now(),
+ Map.of(
+ v1,
+ new ScalingSummary(
+ 1, 2, Map.of(ScalingMetric.LAG, EvaluatedScalingMetric.of(2.)))));
+
+ stateStore.storeEvaluatedMetrics(ctx, metricHistory);
+
+ assertFalse(
+ configMapStore
+ .getSerializedState(
+ ctx,
+ KubernetesAutoScalerStateStore
+ .COLLECTED_METRICS_KEY)
+ .get()
+ .length()
+ + configMapStore
+ .getSerializedState(
+ ctx,
+ KubernetesAutoScalerStateStore.SCALING_HISTORY_KEY)
+ .get()
+ .length()
+ < KubernetesAutoScalerStateStore.MAX_CM_BYTES);
+
+ stateStore.trimHistoryToMaxCmSize(ctx);
+ assertTrue(
+ configMapStore
+ .getSerializedState(
+ ctx,
+ KubernetesAutoScalerStateStore
+ .COLLECTED_METRICS_KEY)
+ .get()
+ .length()
+ + configMapStore
+ .getSerializedState(
+ ctx,
+ KubernetesAutoScalerStateStore.SCALING_HISTORY_KEY)
+ .get()
+ .length()
+ < KubernetesAutoScalerStateStore.MAX_CM_BYTES);
+ }
+
+ @Test
+ public void testDiscardInvalidHistory() throws Exception {
+ configMapStore.putSerializedState(
+ ctx, KubernetesAutoScalerStateStore.COLLECTED_METRICS_KEY, "invalid");
+ configMapStore.putSerializedState(
+ ctx, KubernetesAutoScalerStateStore.SCALING_HISTORY_KEY, "invalid2");
+
+ var now = Instant.now();
+
+ assertThat(
+ configMapStore.getSerializedState(
+ ctx, KubernetesAutoScalerStateStore.COLLECTED_METRICS_KEY))
+ .isPresent();
+ assertThat(stateStore.getEvaluatedMetrics(ctx)).isEmpty();
+ assertThat(
+ configMapStore.getSerializedState(
+ ctx, KubernetesAutoScalerStateStore.COLLECTED_METRICS_KEY))
+ .isEmpty();
+
+ assertThat(
+ configMapStore.getSerializedState(
+ ctx, KubernetesAutoScalerStateStore.SCALING_HISTORY_KEY))
+ .isPresent();
+ Assertions.assertEquals(new TreeMap<>(), getTrimmedScalingHistory(stateStore, ctx, now));
+ assertThat(
+ configMapStore.getSerializedState(
+ ctx, KubernetesAutoScalerStateStore.SCALING_HISTORY_KEY))
+ .isEmpty();
+ }
+}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingAutoscalerFactory.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingAutoscalerFactory.java
deleted file mode 100644
index 486fa96a..00000000
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingAutoscalerFactory.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.autoscaler;
-
-import org.apache.flink.kubernetes.operator.reconciler.deployment.NoopJobAutoscalerFactory;
-
-/** Dummy autoscaler to test the plugin loading for the autoscaler. */
-public class TestingAutoscalerFactory extends NoopJobAutoscalerFactory {}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingKubernetesAutoscalerUtils.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingKubernetesAutoscalerUtils.java
new file mode 100644
index 00000000..9cb92ada
--- /dev/null
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingKubernetesAutoscalerUtils.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.kubernetes.operator.autoscaler;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.client.program.rest.RestClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.kubernetes.operator.TestUtils;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+
+import io.fabric8.kubernetes.client.KubernetesClient;
+
+import javax.annotation.Nonnull;
+
+/** The utils for testing kubernetes autoscaler. */
+public class TestingKubernetesAutoscalerUtils {
+
+ @Nonnull
+ public static KubernetesJobAutoScalerContext createContext(
+ String crName, KubernetesClient kubernetesClient) {
+ var cr = TestUtils.buildApplicationCluster();
+ cr.getMetadata().setName(crName);
+ return new KubernetesJobAutoScalerContext(
+ new JobID(),
+ JobStatus.RUNNING,
+ new Configuration(),
+ new UnregisteredMetricsGroup(),
+ () -> new RestClusterClient<>(new Configuration(), "test-cluster"),
+ cr,
+ kubernetesClient);
+ }
+}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkDeploymentController.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkDeploymentController.java
index e1f3d0ef..b174a29f 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkDeploymentController.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkDeploymentController.java
@@ -25,12 +25,12 @@ import org.apache.flink.kubernetes.operator.TestingFlinkService;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentSpec;
import org.apache.flink.kubernetes.operator.api.status.FlinkDeploymentStatus;
+import org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFactory;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.health.CanaryResourceManager;
import org.apache.flink.kubernetes.operator.metrics.MetricManager;
import org.apache.flink.kubernetes.operator.observer.deployment.FlinkDeploymentObserverFactory;
import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.NoopJobAutoscalerFactory;
import org.apache.flink.kubernetes.operator.reconciler.deployment.ReconcilerFactory;
import org.apache.flink.kubernetes.operator.utils.EventCollector;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
@@ -95,7 +95,8 @@ public class TestingFlinkDeploymentController
configManager,
eventRecorder,
statusRecorder,
- new NoopJobAutoscalerFactory());
+ AutoscalerFactory.create(
+ flinkService.getKubernetesClient(), eventRecorder));
canaryResourceManager = new CanaryResourceManager<>(configManager);
flinkDeploymentController =
new FlinkDeploymentController(
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerTest.java
index bc5c267b..9e507f09 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerTest.java
@@ -19,6 +19,8 @@ package org.apache.flink.kubernetes.operator.reconciler.deployment;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.autoscaler.JobAutoScaler;
+import org.apache.flink.autoscaler.NoopJobAutoscaler;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.Configuration;
@@ -53,9 +55,9 @@ import org.apache.flink.kubernetes.operator.api.status.SavepointInfo;
import org.apache.flink.kubernetes.operator.api.status.SnapshotInfo;
import org.apache.flink.kubernetes.operator.api.status.SnapshotTriggerType;
import org.apache.flink.kubernetes.operator.api.utils.FlinkResourceUtils;
+import org.apache.flink.kubernetes.operator.autoscaler.KubernetesJobAutoScalerContext;
import org.apache.flink.kubernetes.operator.config.FlinkOperatorConfiguration;
import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions;
-import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
import org.apache.flink.kubernetes.operator.exception.RecoveryFailureException;
import org.apache.flink.kubernetes.operator.health.ClusterHealthInfo;
import org.apache.flink.kubernetes.operator.observer.ClusterHealthEvaluator;
@@ -76,6 +78,7 @@ import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
+import io.javaoperatorsdk.operator.processing.event.ResourceID;
import lombok.Getter;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Assertions;
@@ -141,8 +144,7 @@ public class ApplicationReconcilerTest extends OperatorTestBase {
@Override
public void setup() {
appReconciler =
- new ApplicationReconciler(
- eventRecorder, statusRecorder, new NoopJobAutoscalerFactory());
+ new ApplicationReconciler(eventRecorder, statusRecorder, new NoopJobAutoscaler<>());
reconciler = new TestReconcilerAdapter<>(this, appReconciler);
operatorConfig = configManager.getOperatorConfiguration();
executorService = Executors.newDirectExecutorService();
@@ -886,16 +888,15 @@ public class ApplicationReconcilerTest extends OperatorTestBase {
configManager, operatorMetricGroup, flinkService, eventRecorder);
var overrideFunction = new AtomicReference<Consumer<AbstractFlinkSpec>>(s -> {});
- JobAutoScalerFactory autoscalerFactory =
- (r) ->
- new NoopJobAutoscalerFactory() {
- @Override
- public void scale(FlinkResourceContext<?> ctx) {
- overrideFunction.get().accept(ctx.getResource().getSpec());
- }
- };
+ JobAutoScaler<ResourceID, KubernetesJobAutoScalerContext> autoscaler =
+ new NoopJobAutoscaler<>() {
+ @Override
+ public void scale(KubernetesJobAutoScalerContext ctx) {
+ overrideFunction.get().accept(ctx.getResource().getSpec());
+ }
+ };
- appReconciler = new ApplicationReconciler(eventRecorder, statusRecorder, autoscalerFactory);
+ appReconciler = new ApplicationReconciler(eventRecorder, statusRecorder, autoscaler);
var deployment = TestUtils.buildApplicationCluster();
appReconciler.reconcile(ctxFactory.getResourceContext(deployment, context));
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerUpgradeModeTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerUpgradeModeTest.java
index fcde70b6..357e595a 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerUpgradeModeTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/deployment/ApplicationReconcilerUpgradeModeTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.kubernetes.operator.reconciler.deployment;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.autoscaler.NoopJobAutoscaler;
import org.apache.flink.configuration.CheckpointingOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.HighAvailabilityOptions;
@@ -78,7 +79,7 @@ public class ApplicationReconcilerUpgradeModeTest extends OperatorTestBase {
new TestReconcilerAdapter<>(
this,
new ApplicationReconciler(
- eventRecorder, statusRecorder, new NoopJobAutoscalerFactory()));
+ eventRecorder, statusRecorder, new NoopJobAutoscaler<>()));
}
@ParameterizedTest
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoaderTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoaderTest.java
deleted file mode 100644
index cabc25bc..00000000
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/AutoscalerLoaderTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.kubernetes.operator.utils;
-
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.autoscaler.TestingAutoscalerFactory;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScalerFactory;
-import org.apache.flink.kubernetes.operator.reconciler.deployment.NoopJobAutoscalerFactory;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
-
-/** Test loading the default autoscaling implementation from the classpath. */
-public class AutoscalerLoaderTest {
-
- @TempDir public Path temporaryFolder;
-
- @Test
- void testLoadFallbackNoopImplementation() {
- JobAutoScalerFactory factory = AutoscalerLoader.loadJobAutoscalerFactory();
- Assertions.assertTrue(factory instanceof NoopJobAutoscalerFactory);
- }
-
- @Test
- void testLoadCustomImplementation() throws Exception {
- Map<String, String> originalEnv = System.getenv();
- try {
- Map<String, String> systemEnv = new HashMap<>(originalEnv);
- systemEnv.put(
- ConfigConstants.ENV_FLINK_PLUGINS_DIR,
- TestUtils.getTestPluginsRootDir(temporaryFolder));
- TestUtils.setEnv(systemEnv);
-
- JobAutoScalerFactory factory = AutoscalerLoader.loadJobAutoscalerFactory();
- Assertions.assertTrue(factory instanceof TestingAutoscalerFactory);
- } finally {
- TestUtils.setEnv(originalEnv);
- }
- }
-}
diff --git a/pom.xml b/pom.xml
index f18a417e..78dfb9dc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -53,7 +53,6 @@ under the License.
<modules>
<module>flink-kubernetes-standalone</module>
<module>flink-kubernetes-operator</module>
- <module>flink-kubernetes-operator-autoscaler</module>
<module>flink-kubernetes-operator-api</module>
<module>flink-kubernetes-webhook</module>
<module>flink-kubernetes-docs</module>
@@ -94,6 +93,8 @@ under the License.
<okhttp.version>4.11.0</okhttp.version>
<curator-test.version>5.2.0</curator-test.version>
+
+ <assertj.version>3.21.0</assertj.version>
</properties>
<dependencyManagement>
[flink-kubernetes-operator] 02/04: [FLINK-33098][autoscaler] Move non-kubernetes related autoscaler classes to flink-autoscaler module
Posted by gy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
commit b4e958c4700f4e1a1b38512fa09549bbcd78c95d
Author: 1996fanrui <19...@gmail.com>
AuthorDate: Tue Sep 19 11:41:14 2023 +0800
[FLINK-33098][autoscaler] Move non-kubernetes related autoscaler classes to flink-autoscaler module
---
Dockerfile | 6 ++-
docker-entrypoint.sh | 2 +-
flink-autoscaler/pom.xml | 26 ++++++++++
.../flink}/autoscaler/ScalingMetricEvaluator.java | 55 +++++++++++-----------
.../apache/flink}/autoscaler/ScalingSummary.java | 6 +--
.../autoscaler/config/AutoScalerOptions.java | 15 ++++--
.../autoscaler/metrics/CollectedMetricHistory.java | 4 +-
.../autoscaler/metrics/CollectedMetrics.java | 2 +-
.../org/apache/flink}/autoscaler/metrics/Edge.java | 2 +-
.../autoscaler/metrics/EvaluatedScalingMetric.java | 2 +-
.../flink}/autoscaler/metrics/FlinkMetric.java | 2 +-
.../autoscaler/metrics/MetricAggregator.java | 2 +-
.../flink}/autoscaler/metrics/ScalingMetric.java | 2 +-
.../flink}/autoscaler/metrics/ScalingMetrics.java | 8 ++--
.../flink}/autoscaler/topology/JobTopology.java | 2 +-
.../flink}/autoscaler/topology/VertexInfo.java | 2 +-
.../autoscaler/utils/AutoScalerSerDeModule.java | 4 +-
.../flink}/autoscaler/utils/AutoScalerUtils.java | 12 ++---
.../autoscaler/ScalingMetricEvaluatorTest.java | 44 ++++++++---------
.../autoscaler/metrics/ScalingMetricsTest.java | 8 ++--
.../autoscaler/topology}/JobTopologyTest.java | 3 +-
.../autoscaler/utils/AutoScalerUtilsTest.java | 4 +-
.../src/test/resources/log4j2-test.properties | 35 ++++----------
flink-kubernetes-docs/pom.xml | 2 +-
.../configuration/ConfigOptionsDocGenerator.java | 4 +-
flink-kubernetes-operator-autoscaler/pom.xml | 7 +++
.../operator/autoscaler/AutoScalerInfo.java | 7 +--
.../autoscaler/AutoscalerFlinkMetrics.java | 8 ++--
.../operator/autoscaler/JobAutoScalerImpl.java | 9 ++--
.../autoscaler/JobAutoscalerFactoryImpl.java | 1 +
.../operator/autoscaler/JobVertexScaler.java | 29 ++++++------
.../autoscaler/RestApiMetricsCollector.java | 2 +-
.../operator/autoscaler/ScalingExecutor.java | 33 ++++++-------
.../autoscaler/ScalingMetricCollector.java | 16 +++----
.../autoscaler/AutoScalerFlinkMetricsTest.java | 10 ++--
.../operator/autoscaler/AutoScalerInfoTest.java | 9 ++--
.../autoscaler/BacklogBasedScalingTest.java | 9 ++--
.../operator/autoscaler/JobAutoScalerImplTest.java | 19 ++++----
.../operator/autoscaler/JobVertexScalerTest.java | 8 ++--
.../MetricsCollectionAndEvaluationTest.java | 15 +++---
.../autoscaler/RecommendedParallelismTest.java | 17 +++----
.../autoscaler/RestApiMetricsCollectorTest.java | 2 +-
.../operator/autoscaler/ScalingExecutorTest.java | 8 ++--
.../autoscaler/ScalingMetricCollectorTest.java | 9 ++--
.../autoscaler/TestingMetricsCollector.java | 4 +-
45 files changed, 256 insertions(+), 220 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index fe6a85a3..4cfb8f66 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,11 +23,12 @@ WORKDIR /app
COPY . .
-RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-kubernetes-operator-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS
+RUN --mount=type=cache,target=/root/.m2 mvn -ntp clean install -pl flink-kubernetes-standalone,flink-kubernetes-operator-api,flink-kubernetes-operator,flink-autoscaler,flink-kubernetes-operator-autoscaler,flink-kubernetes-webhook -DskipTests=$SKIP_TESTS
RUN cd /app/tools/license; mkdir jars; cd jars; \
cp /app/flink-kubernetes-operator/target/flink-kubernetes-operator-*-shaded.jar . && \
cp /app/flink-kubernetes-operator-autoscaler/target/flink-kubernetes-operator-autoscaler-*.jar . && \
+ cp /app/flink-autoscaler/target/flink-autoscaler-*.jar . && \
cp /app/flink-kubernetes-webhook/target/flink-kubernetes-webhook-*-shaded.jar . && \
cp /app/flink-kubernetes-standalone/target/flink-kubernetes-standalone-*.jar . && \
cp -r /app/flink-kubernetes-operator/target/plugins ./plugins && \
@@ -39,6 +40,7 @@ ENV FLINK_HOME=/opt/flink
ENV FLINK_PLUGINS_DIR=$FLINK_HOME/plugins
ENV OPERATOR_VERSION=1.7-SNAPSHOT
ENV OPERATOR_JAR=flink-kubernetes-operator-$OPERATOR_VERSION-shaded.jar
+ENV FLINK_AUTOSCALER_JAR=flink-autoscaler-$OPERATOR_VERSION.jar
ENV AUTOSCALER_JAR=flink-kubernetes-operator-autoscaler-$OPERATOR_VERSION.jar
ENV WEBHOOK_JAR=flink-kubernetes-webhook-$OPERATOR_VERSION-shaded.jar
ENV KUBERNETES_STANDALONE_JAR=flink-kubernetes-standalone-$OPERATOR_VERSION.jar
@@ -51,6 +53,7 @@ RUN groupadd --system --gid=9999 flink && \
useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink
COPY --from=build /app/flink-kubernetes-operator/target/$OPERATOR_JAR .
+COPY --from=build /app/flink-autoscaler/target/$FLINK_AUTOSCALER_JAR .
COPY --from=build /app/flink-kubernetes-operator-autoscaler/target/$AUTOSCALER_JAR .
COPY --from=build /app/flink-kubernetes-webhook/target/$WEBHOOK_JAR .
COPY --from=build /app/flink-kubernetes-standalone/target/$KUBERNETES_STANDALONE_JAR .
@@ -61,6 +64,7 @@ COPY docker-entrypoint.sh /
RUN chown -R flink:flink $FLINK_HOME && \
chown flink:flink $OPERATOR_JAR && \
+ chown flink:flink $FLINK_AUTOSCALER_JAR && \
chown flink:flink $AUTOSCALER_JAR && \
chown flink:flink $WEBHOOK_JAR && \
chown flink:flink $KUBERNETES_STANDALONE_JAR && \
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index 5b215bee..c46d672b 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -29,7 +29,7 @@ if [ "$1" = "help" ]; then
elif [ "$1" = "operator" ]; then
echo "Starting Operator"
- exec java -cp "./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:./$AUTOSCALER_JAR:$OPERATOR_LIB/*" $LOG_CONFIG $JVM_ARGS org.apache.flink.kubernetes.operator.FlinkOperator
+ exec java -cp "./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:./$FLINK_AUTOSCALER_JAR:./$AUTOSCALER_JAR:$OPERATOR_LIB/*" $LOG_CONFIG $JVM_ARGS org.apache.flink.kubernetes.operator.FlinkOperator
elif [ "$1" = "webhook" ]; then
echo "Starting Webhook"
diff --git a/flink-autoscaler/pom.xml b/flink-autoscaler/pom.xml
index 709e5cca..8e216ee9 100644
--- a/flink-autoscaler/pom.xml
+++ b/flink-autoscaler/pom.xml
@@ -46,6 +46,32 @@ under the License.
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>${lombok.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- TODO FLINK-33098: These jackson dependencies can be replaced with flink shaded jackson. It can be done
+ after the flink-1.18 is released, because the shaded jackson version of flink-1.17 is 2.13.4,
+ it doesn't support loaderOptions. -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
</dependencies>
</project>
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluator.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricEvaluator.java
similarity index 82%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluator.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricEvaluator.java
index af6d1732..e5f9e3cb 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluator.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingMetricEvaluator.java
@@ -15,44 +15,45 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.CollectedMetricHistory;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.autoscaler.metrics.Edge;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetricHistory;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetrics;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.Edge;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.commons.math3.stat.StatUtils;
-import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
import java.util.SortedMap;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.BACKLOG_PROCESSING_LAG_THRESHOLD;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.CATCH_UP_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.CURRENT_PROCESSING_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.LAG;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.LOAD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.MAX_PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SCALE_DOWN_RATE_THRESHOLD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SCALE_UP_RATE_THRESHOLD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SOURCE_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.BACKLOG_PROCESSING_LAG_THRESHOLD;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.CATCH_UP_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.CURRENT_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.LAG;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.LOAD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.MAX_PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_DOWN_RATE_THRESHOLD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_UP_RATE_THRESHOLD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SOURCE_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
/** Job scaling evaluator for autoscaler. */
public class ScalingMetricEvaluator {
@@ -110,7 +111,7 @@ public class ScalingMetricEvaluator {
});
}
- @NotNull
+ @Nonnull
private Map<ScalingMetric, EvaluatedScalingMetric> evaluateMetrics(
Configuration conf,
HashMap<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> scalingOutput,
@@ -154,7 +155,7 @@ public class ScalingMetricEvaluator {
}
@VisibleForTesting
- protected static void computeProcessingRateThresholds(
+ public static void computeProcessingRateThresholds(
Map<ScalingMetric, EvaluatedScalingMetric> metrics,
Configuration conf,
boolean processingBacklog) {
@@ -265,7 +266,7 @@ public class ScalingMetricEvaluator {
Edge edge, SortedMap<Instant, CollectedMetrics> metricsHistory) {
double[] metricValues =
metricsHistory.values().stream()
- .map(m -> m.getOutputRatios())
+ .map(CollectedMetrics::getOutputRatios)
.filter(m -> m.containsKey(edge))
.mapToDouble(m -> m.get(edge))
.filter(d -> !Double.isNaN(d))
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingSummary.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingSummary.java
similarity index 89%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingSummary.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingSummary.java
index 4ff7a6e9..bc25f6c4 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingSummary.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/ScalingSummary.java
@@ -15,10 +15,10 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/config/AutoScalerOptions.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
similarity index 95%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/config/AutoScalerOptions.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
index 276201dc..7a9d1d90 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/config/AutoScalerOptions.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
@@ -15,22 +15,27 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.config;
+package org.apache.flink.autoscaler.config;
+import org.apache.flink.autoscaler.metrics.MetricAggregator;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.MetricAggregator;
import java.time.Duration;
import java.util.List;
-import static org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions.operatorConfig;
-
/** Config options related to the autoscaler module. */
public class AutoScalerOptions {
+ public static final String DEPRECATED_K8S_OP_CONF_PREFIX = "kubernetes.operator.";
+ public static final String AUTOSCALER_CONF_PREFIX = "job.autoscaler.";
+
+ public static ConfigOptions.OptionBuilder operatorConfig(String key) {
+ return ConfigOptions.key(DEPRECATED_K8S_OP_CONF_PREFIX + key);
+ }
+
private static ConfigOptions.OptionBuilder autoScalerConfig(String key) {
- return operatorConfig("job.autoscaler." + key);
+ return operatorConfig(AUTOSCALER_CONF_PREFIX + key);
}
public static final ConfigOption<Boolean> AUTOSCALER_ENABLED =
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/CollectedMetricHistory.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/CollectedMetricHistory.java
similarity index 88%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/CollectedMetricHistory.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/CollectedMetricHistory.java
index 43bbf47b..31a19879 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/CollectedMetricHistory.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/CollectedMetricHistory.java
@@ -15,9 +15,9 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.JobTopology;
import lombok.Data;
import lombok.Setter;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/CollectedMetrics.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/CollectedMetrics.java
similarity index 94%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/CollectedMetrics.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/CollectedMetrics.java
index 97e3f9fe..bfd85ca2 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/CollectedMetrics.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/CollectedMetrics.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
import org.apache.flink.runtime.jobgraph.JobVertexID;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/Edge.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/Edge.java
similarity index 94%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/Edge.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/Edge.java
index c89692a9..1fe938b8 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/Edge.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/Edge.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
import org.apache.flink.runtime.jobgraph.JobVertexID;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/EvaluatedScalingMetric.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/EvaluatedScalingMetric.java
similarity index 95%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/EvaluatedScalingMetric.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/EvaluatedScalingMetric.java
index 23788886..c60adf18 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/EvaluatedScalingMetric.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/EvaluatedScalingMetric.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
import lombok.Data;
import lombok.NoArgsConstructor;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/FlinkMetric.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/FlinkMetric.java
similarity index 97%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/FlinkMetric.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/FlinkMetric.java
index 81609775..7f28d947 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/FlinkMetric.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/FlinkMetric.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/MetricAggregator.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/MetricAggregator.java
similarity index 95%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/MetricAggregator.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/MetricAggregator.java
index c79d4b7d..e147d211 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/MetricAggregator.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/MetricAggregator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetric.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingMetric.java
similarity index 97%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetric.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingMetric.java
index 608e4f1c..b6a3e17d 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetric.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingMetric.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
/**
* Supported scaling metrics. These represent high level metrics computed from Flink job metrics
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetrics.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingMetrics.java
similarity index 97%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetrics.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingMetrics.java
index 592d0602..afaa21ae 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetrics.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/metrics/ScalingMetrics.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/topology/JobTopology.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/topology/JobTopology.java
similarity index 99%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/topology/JobTopology.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/topology/JobTopology.java
index 32643741..49fa11ba 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/topology/JobTopology.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/topology/JobTopology.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.topology;
+package org.apache.flink.autoscaler.topology;
import org.apache.flink.runtime.jobgraph.JobVertexID;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/topology/VertexInfo.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/topology/VertexInfo.java
similarity index 95%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/topology/VertexInfo.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/topology/VertexInfo.java
index a261ca6d..249be598 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/topology/VertexInfo.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/topology/VertexInfo.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.topology;
+package org.apache.flink.autoscaler.topology;
import org.apache.flink.runtime.jobgraph.JobVertexID;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerSerDeModule.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerSerDeModule.java
similarity index 95%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerSerDeModule.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerSerDeModule.java
index fad516a5..ce991403 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerSerDeModule.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerSerDeModule.java
@@ -15,9 +15,9 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.utils;
+package org.apache.flink.autoscaler.utils;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.Edge;
+import org.apache.flink.autoscaler.metrics.Edge;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import com.fasterxml.jackson.core.JsonGenerator;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerUtils.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerUtils.java
similarity index 87%
rename from flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerUtils.java
rename to flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerUtils.java
index 54b6bef3..f011fcec 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerUtils.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/utils/AutoScalerUtils.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.utils;
+package org.apache.flink.autoscaler.utils;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import java.util.ArrayList;
@@ -30,8 +30,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.CATCH_UP_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.CATCH_UP_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
/** AutoScaler utilities. */
public class AutoScalerUtils {
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluatorTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingMetricEvaluatorTest.java
similarity index 86%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluatorTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingMetricEvaluatorTest.java
index c67b27fe..13b19445 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluatorTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/ScalingMetricEvaluatorTest.java
@@ -15,18 +15,18 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.CollectedMetricHistory;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.autoscaler.metrics.Edge;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetricHistory;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetrics;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.Edge;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.VertexInfo;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.junit.jupiter.api.Test;
@@ -39,19 +39,19 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.CATCH_UP_DURATION;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.RESTART_TIME;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.CATCH_UP_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.CURRENT_PROCESSING_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.LAG;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.LOAD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SCALE_DOWN_RATE_THRESHOLD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SCALE_UP_RATE_THRESHOLD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SOURCE_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.CATCH_UP_DURATION;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.RESTART_TIME;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION_BOUNDARY;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.CATCH_UP_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.CURRENT_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.LAG;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.LOAD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_DOWN_RATE_THRESHOLD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_UP_RATE_THRESHOLD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SOURCE_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetricsTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/metrics/ScalingMetricsTest.java
similarity index 98%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetricsTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/metrics/ScalingMetricsTest.java
index 0ae3e2b7..85a72e5a 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetricsTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/metrics/ScalingMetricsTest.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.metrics;
+package org.apache.flink.autoscaler.metrics;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.VertexInfo;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobTopologyTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/topology/JobTopologyTest.java
similarity index 96%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobTopologyTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/topology/JobTopologyTest.java
index f76fadce..f16a2711 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobTopologyTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/topology/JobTopologyTest.java
@@ -15,9 +15,8 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler;
+package org.apache.flink.autoscaler.topology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerUtilsTest.java b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/utils/AutoScalerUtilsTest.java
similarity index 94%
rename from flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerUtilsTest.java
rename to flink-autoscaler/src/test/java/org/apache/flink/autoscaler/utils/AutoScalerUtilsTest.java
index 90588994..a037258a 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerUtilsTest.java
+++ b/flink-autoscaler/src/test/java/org/apache/flink/autoscaler/utils/AutoScalerUtilsTest.java
@@ -15,10 +15,10 @@
* limitations under the License.
*/
-package org.apache.flink.kubernetes.operator.autoscaler.utils;
+package org.apache.flink.autoscaler.utils;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.junit.jupiter.api.Test;
diff --git a/docker-entrypoint.sh b/flink-autoscaler/src/test/resources/log4j2-test.properties
old mode 100755
new mode 100644
similarity index 51%
copy from docker-entrypoint.sh
copy to flink-autoscaler/src/test/resources/log4j2-test.properties
index 5b215bee..47b66644
--- a/docker-entrypoint.sh
+++ b/flink-autoscaler/src/test/resources/log4j2-test.properties
@@ -1,6 +1,4 @@
-#!/usr/bin/env bash
-
-###############################################################################
+################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,28 +14,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-###############################################################################
-
-args=("$@")
-
-cd /flink-kubernetes-operator || exit
-
-if [ "$1" = "help" ]; then
- printf "Usage: $(basename "$0") (operator|webhook)\n"
- printf " Or $(basename "$0") help\n\n"
- exit 0
-elif [ "$1" = "operator" ]; then
- echo "Starting Operator"
-
- exec java -cp "./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:./$AUTOSCALER_JAR:$OPERATOR_LIB/*" $LOG_CONFIG $JVM_ARGS org.apache.flink.kubernetes.operator.FlinkOperator
-elif [ "$1" = "webhook" ]; then
- echo "Starting Webhook"
-
- # Adds the operator shaded jar on the classpath when the webhook starts
- exec java -cp "./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:./$WEBHOOK_JAR:$OPERATOR_LIB/*" $LOG_CONFIG $JVM_ARGS org.apache.flink.kubernetes.operator.admission.FlinkOperatorWebhook
-fi
+################################################################################
-args=("${args[@]}")
+rootLogger.level = INFO
+rootLogger.appenderRef.console.ref = ConsoleAppender
-# Running command in pass-through mode
-exec "${args[@]}"
+# Log all infos to the console
+appender.console.name = ConsoleAppender
+appender.console.type = CONSOLE
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %style{%d}{yellow} %style{%-30c{1.}}{cyan} %highlight{[%-5level]%notEmpty{[%X{resource.namespace}/}%notEmpty{%X{resource.name}]} %msg%n%throwable}
diff --git a/flink-kubernetes-docs/pom.xml b/flink-kubernetes-docs/pom.xml
index 09e8cb38..fd42c792 100644
--- a/flink-kubernetes-docs/pom.xml
+++ b/flink-kubernetes-docs/pom.xml
@@ -45,7 +45,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-kubernetes-operator-autoscaler</artifactId>
+ <artifactId>flink-autoscaler</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGenerator.java b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGenerator.java
index 2fb1e6a4..7f38f1fa 100644
--- a/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGenerator.java
+++ b/flink-kubernetes-docs/src/main/java/org/apache/flink/kubernetes/operator/docs/configuration/ConfigOptionsDocGenerator.java
@@ -76,9 +76,7 @@ public class ConfigOptionsDocGenerator {
new OptionsClassLocation(
"flink-kubernetes-operator",
"org.apache.flink.kubernetes.operator.metrics"),
- new OptionsClassLocation(
- "flink-kubernetes-operator-autoscaler",
- "org.apache.flink.kubernetes.operator.autoscaler.config")
+ new OptionsClassLocation("flink-autoscaler", "org.apache.flink.autoscaler.config")
};
static final String DEFAULT_PATH_PREFIX = "src/main/java";
diff --git a/flink-kubernetes-operator-autoscaler/pom.xml b/flink-kubernetes-operator-autoscaler/pom.xml
index 1156855b..2a6498ca 100644
--- a/flink-kubernetes-operator-autoscaler/pom.xml
+++ b/flink-kubernetes-operator-autoscaler/pom.xml
@@ -36,6 +36,13 @@ under the License.
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-autoscaler</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java
index 55c0b0ac..c37b299f 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java
@@ -18,11 +18,12 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.autoscaler.utils.AutoScalerSerDeModule;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ConfigurationUtils;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetrics;
-import org.apache.flink.kubernetes.operator.autoscaler.utils.AutoScalerSerDeModule;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.util.Preconditions;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java
index 56768e18..eab0f850 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoscalerFlinkMetrics.java
@@ -18,8 +18,8 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -32,8 +32,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
/** Autoscaler metrics for observability. */
public class AutoscalerFlinkMetrics {
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java
index 402f88bf..87603fce 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImpl.java
@@ -19,15 +19,16 @@ package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ConfigurationUtils;
import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
import org.apache.flink.kubernetes.operator.api.lifecycle.ResourceLifecycleState;
import org.apache.flink.kubernetes.operator.api.status.CommonStatus;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScaler;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
@@ -41,9 +42,9 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.initRecommendedParallelism;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.resetRecommendedParallelism;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
/** Application and SessionJob autoscaler. */
public class JobAutoScalerImpl implements JobAutoScaler {
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java
index f6a7afc8..e0934a30 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoscalerFactoryImpl.java
@@ -17,6 +17,7 @@
package org.apache.flink.kubernetes.operator.autoscaler;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScaler;
import org.apache.flink.kubernetes.operator.reconciler.deployment.JobAutoScalerFactory;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScaler.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScaler.java
index fec126af..2fdc3e88 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScaler.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScaler.java
@@ -18,12 +18,13 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.util.Preconditions;
@@ -38,16 +39,16 @@ import java.time.ZoneId;
import java.util.Map;
import java.util.SortedMap;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.MAX_SCALE_DOWN_FACTOR;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.MAX_SCALE_UP_FACTOR;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.SCALE_UP_GRACE_PERIOD;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.VERTEX_MAX_PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.VERTEX_MIN_PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.EXPECTED_PROCESSING_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.MAX_PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.MAX_SCALE_DOWN_FACTOR;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.MAX_SCALE_UP_FACTOR;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALE_UP_GRACE_PERIOD;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.TARGET_UTILIZATION;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.VERTEX_MAX_PARALLELISM;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.VERTEX_MIN_PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.EXPECTED_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.MAX_PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
/** Component responsible for computing vertex parallelism based on the scaling metrics. */
public class JobVertexScaler {
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java
index 158bb5a2..dca52a0c 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java
@@ -18,9 +18,9 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
import org.apache.flink.kubernetes.operator.service.FlinkService;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.EmptyRequestBody;
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java
index 6367381c..32859a43 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java
@@ -18,11 +18,12 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.util.Preconditions;
@@ -39,12 +40,12 @@ import java.util.HashMap;
import java.util.Map;
import java.util.SortedMap;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.EXPECTED_PROCESSING_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SCALE_DOWN_RATE_THRESHOLD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.SCALE_UP_RATE_THRESHOLD;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.EXPECTED_PROCESSING_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_DOWN_RATE_THRESHOLD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.SCALE_UP_RATE_THRESHOLD;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TARGET_DATA_RATE;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
/** Class responsible for executing scaling decisions. */
public class ScalingExecutor {
@@ -128,13 +129,13 @@ public class ScalingExecutor {
Map<JobVertexID, Map<ScalingMetric, EvaluatedScalingMetric>> evaluatedMetrics,
Map<JobVertexID, ScalingSummary> scalingSummaries) {
scalingSummaries.forEach(
- (jobVertexID, scalingSummary) -> {
- evaluatedMetrics
- .get(jobVertexID)
- .put(
- ScalingMetric.RECOMMENDED_PARALLELISM,
- EvaluatedScalingMetric.of(scalingSummary.getNewParallelism()));
- });
+ (jobVertexID, scalingSummary) ->
+ evaluatedMetrics
+ .get(jobVertexID)
+ .put(
+ ScalingMetric.RECOMMENDED_PARALLELISM,
+ EvaluatedScalingMetric.of(
+ scalingSummary.getNewParallelism())));
}
private static String scalingReport(
diff --git a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java
index 3ad1db39..e620fb0e 100644
--- a/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java
+++ b/flink-kubernetes-operator-autoscaler/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java
@@ -19,17 +19,17 @@ package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.CollectedMetricHistory;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetrics;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetricHistory;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetrics;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetrics;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.utils.AutoScalerUtils;
import org.apache.flink.kubernetes.operator.service.FlinkService;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.jobgraph.JobVertexID;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerFlinkMetricsTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerFlinkMetricsTest.java
index 2600bd60..22aa1462 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerFlinkMetricsTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerFlinkMetricsTest.java
@@ -17,10 +17,10 @@
package org.apache.flink.kubernetes.operator.autoscaler;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.TestUtils;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
import org.apache.flink.kubernetes.operator.metrics.TestingMetricListener;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -32,14 +32,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.AVERAGE;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.CURRENT;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.JOB_VERTEX_ID;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.initRecommendedParallelism;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerFlinkMetrics.resetRecommendedParallelism;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.TRUE_PROCESSING_RATE;
import static org.junit.jupiter.api.Assertions.assertEquals;
/** {@link AutoscalerFlinkMetrics} tests. */
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java
index cfc2e7f5..94ada9fb 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfoTest.java
@@ -17,11 +17,12 @@
package org.apache.flink.kubernetes.operator.autoscaler;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetrics;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import com.fasterxml.jackson.core.JsonProcessingException;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java
index 31727983..9bdf777c 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java
@@ -19,14 +19,15 @@ package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.OperatorTestBase;
import org.apache.flink.kubernetes.operator.TestUtils;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.VertexInfo;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
import org.apache.flink.kubernetes.operator.utils.EventCollector;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java
index 9a0c8c44..7316668b 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScalerImplTest.java
@@ -19,16 +19,17 @@ package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.PipelineOptions;
import org.apache.flink.kubernetes.operator.OperatorTestBase;
import org.apache.flink.kubernetes.operator.TestUtils;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.VertexInfo;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext;
import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
@@ -54,8 +55,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
-import static org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.AUTOSCALER_ENABLED;
+import static org.apache.flink.autoscaler.config.AutoScalerOptions.SCALING_ENABLED;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -87,7 +88,7 @@ public class JobAutoScalerImplTest extends OperatorTestBase {
}
@Test
- void testMetricReporting() throws Exception {
+ void testMetricReporting() {
JobVertexID jobVertexID = new JobVertexID();
JobTopology jobTopology = new JobTopology(new VertexInfo(jobVertexID, Set.of(), 1, 10));
@@ -136,7 +137,7 @@ public class JobAutoScalerImplTest extends OperatorTestBase {
}
@Test
- void testErrorReporting() throws Exception {
+ void testErrorReporting() {
var autoscaler = new JobAutoScalerImpl(null, null, null, eventRecorder);
FlinkResourceContext<FlinkDeployment> resourceContext = getResourceContext(app);
ResourceID resourceId = ResourceID.fromResource(app);
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScalerTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScalerTest.java
index 8bff55be..a700f89e 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScalerTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobVertexScalerTest.java
@@ -17,12 +17,14 @@
package org.apache.flink.kubernetes.operator.autoscaler;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.TestUtils;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
import org.apache.flink.kubernetes.operator.utils.EventCollector;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java
index a01195c2..f8cec403 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java
@@ -19,17 +19,18 @@ package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.CollectedMetricHistory;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.TestUtils;
import org.apache.flink.kubernetes.operator.TestingFlinkService;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.CollectedMetricHistory;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.VertexInfo;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
import org.apache.flink.kubernetes.operator.utils.EventCollector;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RecommendedParallelismTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RecommendedParallelismTest.java
index b8d1528b..e11d76e9 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RecommendedParallelismTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RecommendedParallelismTest.java
@@ -19,15 +19,16 @@ package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.OperatorTestBase;
import org.apache.flink.kubernetes.operator.TestUtils;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.VertexInfo;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
import org.apache.flink.kubernetes.operator.utils.EventCollector;
@@ -52,9 +53,9 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.PARALLELISM;
+import static org.apache.flink.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
import static org.apache.flink.kubernetes.operator.autoscaler.AutoscalerTestUtils.getOrCreateInfo;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.PARALLELISM;
-import static org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric.RECOMMENDED_PARALLELISM;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -125,7 +126,7 @@ public class RecommendedParallelismTest extends OperatorTestBase {
}
@Test
- public void endToEnd() throws Exception {
+ public void endToEnd() {
// we start the autoscaler in advisor mode
app.getSpec().getFlinkConfiguration().put(AutoScalerOptions.SCALING_ENABLED.key(), "false");
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java
index 04d83107..9f3cb451 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollectorTest.java
@@ -18,11 +18,11 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions;
import org.apache.flink.kubernetes.operator.TestingFlinkService;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.metrics.AggregatedMetric;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java
index c315300b..d66d10e8 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java
@@ -18,13 +18,15 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.ScalingMetricEvaluator;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.config.AutoScalerOptions;
+import org.apache.flink.autoscaler.metrics.EvaluatedScalingMetric;
+import org.apache.flink.autoscaler.metrics.ScalingMetric;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.TestUtils;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
import org.apache.flink.kubernetes.operator.api.FlinkDeployment;
-import org.apache.flink.kubernetes.operator.autoscaler.config.AutoScalerOptions;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.EvaluatedScalingMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.ScalingMetric;
import org.apache.flink.kubernetes.operator.utils.EventCollector;
import org.apache.flink.kubernetes.operator.utils.EventRecorder;
import org.apache.flink.runtime.jobgraph.JobVertexID;
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollectorTest.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollectorTest.java
index ddf26453..d6c8f14f 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollectorTest.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollectorTest.java
@@ -19,13 +19,13 @@ package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
+import org.apache.flink.autoscaler.topology.VertexInfo;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.TestUtils;
import org.apache.flink.kubernetes.operator.TestingFlinkService;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.VertexInfo;
import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.JobPlanInfo;
@@ -124,8 +124,7 @@ public class ScalingMetricCollectorTest {
new RestApiMetricsCollector() {
@Override
protected Collection<AggregatedMetric> queryAggregatedMetricNames(
- RestClusterClient<?> restClient, JobID jobID, JobVertexID jobVertexID)
- throws Exception {
+ RestClusterClient<?> restClient, JobID jobID, JobVertexID jobVertexID) {
return metricList;
}
};
diff --git a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java
index 9a3c1d13..277ee33b 100644
--- a/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java
+++ b/flink-kubernetes-operator-autoscaler/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java
@@ -18,11 +18,11 @@
package org.apache.flink.kubernetes.operator.autoscaler;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.autoscaler.metrics.FlinkMetric;
+import org.apache.flink.autoscaler.topology.JobTopology;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource;
-import org.apache.flink.kubernetes.operator.autoscaler.metrics.FlinkMetric;
-import org.apache.flink.kubernetes.operator.autoscaler.topology.JobTopology;
import org.apache.flink.kubernetes.operator.service.FlinkService;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.rest.messages.job.JobDetailsInfo;
[flink-kubernetes-operator] 04/04: [FLINK-33098][autoscaler] Removing the operator prefix for all autoscaler options
Posted by gy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
commit b1c40e3578120d5b892a6c80fe0dba8ebca58b6a
Author: 1996fanrui <19...@gmail.com>
AuthorDate: Wed Sep 27 21:06:25 2023 +0800
[FLINK-33098][autoscaler] Removing the operator prefix for all autoscaler options
---
docs/content/docs/custom-resource/autoscaler.md | 28 +++++++-------
docs/content/docs/operations/configuration.md | 2 +
.../generated/auto_scaler_configuration.html | 44 +++++++++++-----------
e2e-tests/data/autoscaler.yaml | 8 ++--
examples/autoscaling/autoscaling-dynamic.yaml | 10 ++---
examples/autoscaling/autoscaling.yaml | 8 ++--
.../flink/autoscaler/config/AutoScalerOptions.java | 29 ++++++++++++--
7 files changed, 77 insertions(+), 52 deletions(-)
diff --git a/docs/content/docs/custom-resource/autoscaler.md b/docs/content/docs/custom-resource/autoscaler.md
index 4140122a..7f67c1a3 100644
--- a/docs/content/docs/custom-resource/autoscaler.md
+++ b/docs/content/docs/custom-resource/autoscaler.md
@@ -123,7 +123,7 @@ The defaults might work reasonably well for many applications, but some tuning m
The autoscaler also supports a passive/metrics-only mode where it only collects and evaluates scaling related performance metrics but does not trigger any job upgrades.
This can be used to gain confidence in the module without any impact on the running applications.
-To disable scaling actions, set: `kubernetes.operator.job.autoscaler.scaling.enabled: "false"`
+To disable scaling actions, set: `job.autoscaler.scaling.enabled: "false"`
{{< /hint >}}
### Job and per operator max parallelism
@@ -140,30 +140,30 @@ It is also possible to set maxParallelism on a per operator level, which can be
### Stabilization and metrics collection intervals
-The autoscaler always looks at average metrics in the collection time window defined by `kubernetes.operator.job.autoscaler.metrics.window`.
+The autoscaler always looks at average metrics in the collection time window defined by `job.autoscaler.metrics.window`.
The size of this window determines how small fluctuations will affect the autoscaler. The larger the window, the more smoothing and stability we get, but we may be slower to react to sudden load changes.
We suggest you experiment with setting this anywhere between 3-60 minutes for best experience.
-To allow jobs to stabilize after recovery users can configure a stabilization window by setting `kubernetes.operator.job.autoscaler.stabilization.interval`.
+To allow jobs to stabilize after recovery users can configure a stabilization window by setting `job.autoscaler.stabilization.interval`.
During this time period no metrics will be collected and no scaling actions will be taken.
### Target utilization and flexible boundaries
-In order to provide stable job performance and some buffer for load fluctuations, the autoscaler allows users to set a target utilization level for the job (`kubernetes.operator.job.autoscaler.target.utilization`).
+In order to provide stable job performance and some buffer for load fluctuations, the autoscaler allows users to set a target utilization level for the job (`job.autoscaler.target.utilization`).
A target of `0.6` means we are targeting 60% utilization/load for the job vertexes.
In general, it's not recommended to set target utilization close to 100% as performance usually degrades as we reach capacity limits in most real world systems.
In addition to the utilization target we can set a utilization boundary, that serves as extra buffer to avoid immediate scaling on load fluctuations.
-Setting `kubernetes.operator.job.autoscaler.target.utilization.boundary: "0.2"` means that we allow 20% deviation from the target utilization before triggering a scaling action.
+Setting `job.autoscaler.target.utilization.boundary: "0.2"` means that we allow 20% deviation from the target utilization before triggering a scaling action.
### Target catch-up duration and restart time
When taking scaling decisions the operator need to account for the extra capacity required to catch up the backlog created during scaling operations.
The amount of extra capacity is determined automatically by the following 2 configs:
- - `kubernetes.operator.job.autoscaler.restart.time` : Time it usually takes to restart the application
- - `kubernetes.operator.job.autoscaler.catch-up.duration` : Time to job is expected to catch up after scaling
+ - `job.autoscaler.restart.time` : Time it usually takes to restart the application
+ - `job.autoscaler.catch-up.duration` : Time to job is expected to catch up after scaling
In the future the autoscaler may be able to automatically determine the restart time, but the target catch-up duration depends on the users SLO.
@@ -175,13 +175,13 @@ We suggest setting this based on your actual objective, such us 10,30,60 minutes
...
flinkVersion: v1_17
flinkConfiguration:
- kubernetes.operator.job.autoscaler.enabled: "true"
- kubernetes.operator.job.autoscaler.stabilization.interval: 1m
- kubernetes.operator.job.autoscaler.metrics.window: 5m
- kubernetes.operator.job.autoscaler.target.utilization: "0.6"
- kubernetes.operator.job.autoscaler.target.utilization.boundary: "0.2"
- kubernetes.operator.job.autoscaler.restart.time: 2m
- kubernetes.operator.job.autoscaler.catch-up.duration: 5m
+ job.autoscaler.enabled: "true"
+ job.autoscaler.stabilization.interval: 1m
+ job.autoscaler.metrics.window: 5m
+ job.autoscaler.target.utilization: "0.6"
+ job.autoscaler.target.utilization.boundary: "0.2"
+ job.autoscaler.restart.time: 2m
+ job.autoscaler.catch-up.duration: 5m
pipeline.max-parallelism: "720"
```
diff --git a/docs/content/docs/operations/configuration.md b/docs/content/docs/operations/configuration.md
index 48ad892b..7343faa8 100644
--- a/docs/content/docs/operations/configuration.md
+++ b/docs/content/docs/operations/configuration.md
@@ -130,6 +130,8 @@ These options can be configured on both an operator and a per-resource level. Wh
Like other resource options these can be configured on both an operator and a per-resource level. When set under `spec.flinkConfiguration` for the Flink resources it will override the default value provided in the operator default configuration (`flink-conf.yaml`).
+> Note: The option prefix `kubernetes.operator.` was removed in FLIP-334, because the autoscaler module was decoupled from flink-kubernetes-operator.
+
{{< generated/auto_scaler_configuration >}}
### System Metrics Configuration
diff --git a/docs/layouts/shortcodes/generated/auto_scaler_configuration.html b/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
index bc35f57d..0b2d5b35 100644
--- a/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
+++ b/docs/layouts/shortcodes/generated/auto_scaler_configuration.html
@@ -9,133 +9,133 @@
</thead>
<tbody>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.backlog-processing.lag-threshold</h5></td>
+ <td><h5>job.autoscaler.backlog-processing.lag-threshold</h5></td>
<td style="word-wrap: break-word;">5 min</td>
<td>Duration</td>
<td>Lag threshold which will prevent unnecessary scalings while removing the pending messages responsible for the lag.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.catch-up.duration</h5></td>
+ <td><h5>job.autoscaler.catch-up.duration</h5></td>
<td style="word-wrap: break-word;">15 min</td>
<td>Duration</td>
<td>The target duration for fully processing any backlog after a scaling operation. Set to 0 to disable backlog based scaling.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.enabled</h5></td>
+ <td><h5>job.autoscaler.enabled</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
<td>Enable job autoscaler module.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.flink.client.timeout</h5></td>
+ <td><h5>job.autoscaler.flink.rest-client.timeout</h5></td>
<td style="word-wrap: break-word;">10 s</td>
<td>Duration</td>
<td>The timeout for waiting the flink rest client to return.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.history.max.age</h5></td>
+ <td><h5>job.autoscaler.history.max.age</h5></td>
<td style="word-wrap: break-word;">86400000 ms</td>
<td>Duration</td>
<td>Maximum age for past scaling decisions to retain.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.history.max.count</h5></td>
+ <td><h5>job.autoscaler.history.max.count</h5></td>
<td style="word-wrap: break-word;">3</td>
<td>Integer</td>
<td>Maximum number of past scaling decisions to retain per vertex.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.metrics.busy-time.aggregator</h5></td>
+ <td><h5>job.autoscaler.metrics.busy-time.aggregator</h5></td>
<td style="word-wrap: break-word;">MAX</td>
<td><p>Enum</p></td>
<td>Metric aggregator to use for busyTime metrics. This affects how true processing/output rate will be computed. Using max allows us to handle jobs with data skew more robustly, while avg may provide better stability when we know that the load distribution is even.<br /><br />Possible values:<ul><li>"AVG"</li><li>"MAX"</li><li>"MIN"</li></ul></td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.metrics.window</h5></td>
+ <td><h5>job.autoscaler.metrics.window</h5></td>
<td style="word-wrap: break-word;">10 min</td>
<td>Duration</td>
<td>Scaling metrics aggregation window size.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.restart.time</h5></td>
+ <td><h5>job.autoscaler.restart.time</h5></td>
<td style="word-wrap: break-word;">3 min</td>
<td>Duration</td>
<td>Expected restart time to be used until the operator can determine it reliably from history.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.scale-down.max-factor</h5></td>
+ <td><h5>job.autoscaler.scale-down.max-factor</h5></td>
<td style="word-wrap: break-word;">0.6</td>
<td>Double</td>
<td>Max scale down factor. 1 means no limit on scale down, 0.6 means job can only be scaled down with 60% of the original parallelism.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.scale-up.grace-period</h5></td>
+ <td><h5>job.autoscaler.scale-up.grace-period</h5></td>
<td style="word-wrap: break-word;">1 h</td>
<td>Duration</td>
<td>Duration in which no scale down of a vertex is allowed after it has been scaled up.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.scale-up.max-factor</h5></td>
+ <td><h5>job.autoscaler.scale-up.max-factor</h5></td>
<td style="word-wrap: break-word;">100000.0</td>
<td>Double</td>
<td>Max scale up factor. 2.0 means job can only be scaled up with 200% of the current parallelism.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.scaling.effectiveness.detection.enabled</h5></td>
+ <td><h5>job.autoscaler.scaling.effectiveness.detection.enabled</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
<td>Whether to enable detection of ineffective scaling operations and allowing the autoscaler to block further scale ups.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.scaling.effectiveness.threshold</h5></td>
+ <td><h5>job.autoscaler.scaling.effectiveness.threshold</h5></td>
<td style="word-wrap: break-word;">0.1</td>
<td>Double</td>
<td>Processing rate increase threshold for detecting ineffective scaling threshold. 0.1 means if we do not accomplish at least 10% of the desired capacity increase with scaling, the action is marked ineffective.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.scaling.enabled</h5></td>
+ <td><h5>job.autoscaler.scaling.enabled</h5></td>
<td style="word-wrap: break-word;">true</td>
<td>Boolean</td>
<td>Enable vertex scaling execution by the autoscaler. If disabled, the autoscaler will only collect metrics and evaluate the suggested parallelism for each vertex but will not upgrade the jobs.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.scaling.report.interval</h5></td>
+ <td><h5>job.autoscaler.scaling.report.interval</h5></td>
<td style="word-wrap: break-word;">30 min</td>
<td>Duration</td>
<td>Time interval to resend the identical event</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.stabilization.interval</h5></td>
+ <td><h5>job.autoscaler.stabilization.interval</h5></td>
<td style="word-wrap: break-word;">5 min</td>
<td>Duration</td>
<td>Stabilization period in which no new scaling will be executed</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.target.utilization</h5></td>
+ <td><h5>job.autoscaler.target.utilization</h5></td>
<td style="word-wrap: break-word;">0.7</td>
<td>Double</td>
<td>Target vertex utilization</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.target.utilization.boundary</h5></td>
+ <td><h5>job.autoscaler.target.utilization.boundary</h5></td>
<td style="word-wrap: break-word;">0.4</td>
<td>Double</td>
<td>Target vertex utilization boundary. Scaling won't be performed if the current processing rate is within [target_rate / (target_utilization - boundary), (target_rate / (target_utilization + boundary)]</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.vertex.exclude.ids</h5></td>
+ <td><h5>job.autoscaler.vertex.exclude.ids</h5></td>
<td style="word-wrap: break-word;"></td>
<td>List<String></td>
<td>A (semicolon-separated) list of vertex ids in hexstring for which to disable scaling. Caution: For non-sink vertices this will still scale their downstream operators until https://issues.apache.org/jira/browse/FLINK-31215 is implemented.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.vertex.max-parallelism</h5></td>
+ <td><h5>job.autoscaler.vertex.max-parallelism</h5></td>
<td style="word-wrap: break-word;">200</td>
<td>Integer</td>
<td>The maximum parallelism the autoscaler can use. Note that this limit will be ignored if it is higher than the max parallelism configured in the Flink config or directly on each operator.</td>
</tr>
<tr>
- <td><h5>kubernetes.operator.job.autoscaler.vertex.min-parallelism</h5></td>
+ <td><h5>job.autoscaler.vertex.min-parallelism</h5></td>
<td style="word-wrap: break-word;">1</td>
<td>Integer</td>
<td>The minimum parallelism the autoscaler can use.</td>
diff --git a/e2e-tests/data/autoscaler.yaml b/e2e-tests/data/autoscaler.yaml
index ce41f325..6f53bbe9 100644
--- a/e2e-tests/data/autoscaler.yaml
+++ b/e2e-tests/data/autoscaler.yaml
@@ -36,10 +36,10 @@ spec:
state.checkpoints.dir: file:///opt/flink/volume/flink-cp
state.savepoints.dir: file:///opt/flink/volume/flink-sp
- kubernetes.operator.job.autoscaler.enabled: "true"
- kubernetes.operator.job.autoscaler.scaling.enabled: "true"
- kubernetes.operator.job.autoscaler.stabilization.interval: "5s"
- kubernetes.operator.job.autoscaler.metrics.window: "1m"
+ job.autoscaler.enabled: "true"
+ job.autoscaler.scaling.enabled: "true"
+ job.autoscaler.stabilization.interval: "5s"
+ job.autoscaler.metrics.window: "1m"
jobmanager.scheduler: adaptive
serviceAccount: flink
diff --git a/examples/autoscaling/autoscaling-dynamic.yaml b/examples/autoscaling/autoscaling-dynamic.yaml
index 92d2d159..b6bb3d04 100644
--- a/examples/autoscaling/autoscaling-dynamic.yaml
+++ b/examples/autoscaling/autoscaling-dynamic.yaml
@@ -24,11 +24,11 @@ spec:
image: autoscaling-example
flinkVersion: v1_18
flinkConfiguration:
- kubernetes.operator.job.autoscaler.enabled: "true"
- kubernetes.operator.job.autoscaler.stabilization.interval: "1m"
- kubernetes.operator.job.autoscaler.metrics.window: "15m"
- kubernetes.operator.job.autoscaler.target.utilization: "0.5"
- kubernetes.operator.job.autoscaler.target.utilization.boundary: "0.3"
+ job.autoscaler.enabled: "true"
+ job.autoscaler.stabilization.interval: "1m"
+ job.autoscaler.metrics.window: "15m"
+ job.autoscaler.target.utilization: "0.5"
+ job.autoscaler.target.utilization.boundary: "0.3"
pipeline.max-parallelism: "32"
taskmanager.numberOfTaskSlots: "4"
serviceAccount: flink
diff --git a/examples/autoscaling/autoscaling.yaml b/examples/autoscaling/autoscaling.yaml
index 26edaaad..a973ccd3 100644
--- a/examples/autoscaling/autoscaling.yaml
+++ b/examples/autoscaling/autoscaling.yaml
@@ -24,9 +24,9 @@ spec:
image: autoscaling-example
flinkVersion: v1_18
flinkConfiguration:
- kubernetes.operator.job.autoscaler.enabled: "true"
- kubernetes.operator.job.autoscaler.stabilization.interval: "1m"
- kubernetes.operator.job.autoscaler.metrics.window: "3m"
+ job.autoscaler.enabled: "true"
+ job.autoscaler.stabilization.interval: "1m"
+ job.autoscaler.metrics.window: "3m"
pipeline.max-parallelism: "24"
taskmanager.numberOfTaskSlots: "4"
state.savepoints.dir: file:///flink-data/savepoints
@@ -35,7 +35,7 @@ spec:
high-availability.storageDir: file:///flink-data/ha
execution.checkpointing.interval: "1m"
jobmanager.scheduler: adaptive
- kubernetes.operator.job.autoscaler.target.utilization.boundary: "0.1"
+ job.autoscaler.target.utilization.boundary: "0.1"
serviceAccount: flink
jobManager:
resource:
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
index f22ea2f0..6c74b5d2 100644
--- a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/config/AutoScalerOptions.java
@@ -30,12 +30,12 @@ public class AutoScalerOptions {
public static final String DEPRECATED_K8S_OP_CONF_PREFIX = "kubernetes.operator.";
public static final String AUTOSCALER_CONF_PREFIX = "job.autoscaler.";
- private static String operatorConfigKey(String key) {
- return DEPRECATED_K8S_OP_CONF_PREFIX + key;
+ private static String deprecatedOperatorConfigKey(String key) {
+ return DEPRECATED_K8S_OP_CONF_PREFIX + AUTOSCALER_CONF_PREFIX + key;
}
private static String autoScalerConfigKey(String key) {
- return operatorConfigKey(AUTOSCALER_CONF_PREFIX + key);
+ return AUTOSCALER_CONF_PREFIX + key;
}
private static ConfigOptions.OptionBuilder autoScalerConfig(String key) {
@@ -46,12 +46,14 @@ public class AutoScalerOptions {
autoScalerConfig("enabled")
.booleanType()
.defaultValue(false)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("enabled"))
.withDescription("Enable job autoscaler module.");
public static final ConfigOption<Boolean> SCALING_ENABLED =
autoScalerConfig("scaling.enabled")
.booleanType()
.defaultValue(true)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("scaling.enabled"))
.withDescription(
"Enable vertex scaling execution by the autoscaler. If disabled, the autoscaler will only collect metrics and evaluate the suggested parallelism for each vertex but will not upgrade the jobs.");
@@ -59,12 +61,14 @@ public class AutoScalerOptions {
autoScalerConfig("metrics.window")
.durationType()
.defaultValue(Duration.ofMinutes(10))
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("metrics.window"))
.withDescription("Scaling metrics aggregation window size.");
public static final ConfigOption<Duration> STABILIZATION_INTERVAL =
autoScalerConfig("stabilization.interval")
.durationType()
.defaultValue(Duration.ofMinutes(5))
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("stabilization.interval"))
.withDescription(
"Stabilization period in which no new scaling will be executed");
@@ -72,12 +76,14 @@ public class AutoScalerOptions {
autoScalerConfig("target.utilization")
.doubleType()
.defaultValue(0.7)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("target.utilization"))
.withDescription("Target vertex utilization");
public static final ConfigOption<Double> TARGET_UTILIZATION_BOUNDARY =
autoScalerConfig("target.utilization.boundary")
.doubleType()
.defaultValue(0.4)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("target.utilization.boundary"))
.withDescription(
"Target vertex utilization boundary. Scaling won't be performed if the current processing rate is within [target_rate / (target_utilization - boundary), (target_rate / (target_utilization + boundary)]");
@@ -85,6 +91,7 @@ public class AutoScalerOptions {
autoScalerConfig("scale-up.grace-period")
.durationType()
.defaultValue(Duration.ofHours(1))
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("scale-up.grace-period"))
.withDescription(
"Duration in which no scale down of a vertex is allowed after it has been scaled up.");
@@ -92,12 +99,14 @@ public class AutoScalerOptions {
autoScalerConfig("vertex.min-parallelism")
.intType()
.defaultValue(1)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("vertex.min-parallelism"))
.withDescription("The minimum parallelism the autoscaler can use.");
public static final ConfigOption<Integer> VERTEX_MAX_PARALLELISM =
autoScalerConfig("vertex.max-parallelism")
.intType()
.defaultValue(200)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("vertex.max-parallelism"))
.withDescription(
"The maximum parallelism the autoscaler can use. Note that this limit will be ignored if it is higher than the max parallelism configured in the Flink config or directly on each operator.");
@@ -105,6 +114,7 @@ public class AutoScalerOptions {
autoScalerConfig("scale-down.max-factor")
.doubleType()
.defaultValue(0.6)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("scale-down.max-factor"))
.withDescription(
"Max scale down factor. 1 means no limit on scale down, 0.6 means job can only be scaled down with 60% of the original parallelism.");
@@ -112,6 +122,7 @@ public class AutoScalerOptions {
autoScalerConfig("scale-up.max-factor")
.doubleType()
.defaultValue(100000.)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("scale-up.max-factor"))
.withDescription(
"Max scale up factor. 2.0 means job can only be scaled up with 200% of the current parallelism.");
@@ -119,6 +130,7 @@ public class AutoScalerOptions {
autoScalerConfig("catch-up.duration")
.durationType()
.defaultValue(Duration.ofMinutes(15))
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("catch-up.duration"))
.withDescription(
"The target duration for fully processing any backlog after a scaling operation. Set to 0 to disable backlog based scaling.");
@@ -126,6 +138,7 @@ public class AutoScalerOptions {
autoScalerConfig("restart.time")
.durationType()
.defaultValue(Duration.ofMinutes(3))
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("restart.time"))
.withDescription(
"Expected restart time to be used until the operator can determine it reliably from history.");
@@ -133,6 +146,8 @@ public class AutoScalerOptions {
autoScalerConfig("backlog-processing.lag-threshold")
.durationType()
.defaultValue(Duration.ofMinutes(5))
+ .withDeprecatedKeys(
+ deprecatedOperatorConfigKey("backlog-processing.lag-threshold"))
.withDescription(
"Lag threshold which will prevent unnecessary scalings while removing the pending messages responsible for the lag.");
@@ -140,6 +155,8 @@ public class AutoScalerOptions {
autoScalerConfig("scaling.effectiveness.detection.enabled")
.booleanType()
.defaultValue(false)
+ .withDeprecatedKeys(
+ deprecatedOperatorConfigKey("scaling.effectiveness.detection.enabled"))
.withDescription(
"Whether to enable detection of ineffective scaling operations and allowing the autoscaler to block further scale ups.");
@@ -147,6 +164,8 @@ public class AutoScalerOptions {
autoScalerConfig("scaling.effectiveness.threshold")
.doubleType()
.defaultValue(0.1)
+ .withDeprecatedKeys(
+ deprecatedOperatorConfigKey("scaling.effectiveness.threshold"))
.withDescription(
"Processing rate increase threshold for detecting ineffective scaling threshold. 0.1 means if we do not accomplish at least 10% of the desired capacity increase with scaling, the action is marked ineffective.");
@@ -154,6 +173,7 @@ public class AutoScalerOptions {
autoScalerConfig("history.max.count")
.intType()
.defaultValue(3)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("history.max.count"))
.withDescription(
"Maximum number of past scaling decisions to retain per vertex.");
@@ -161,12 +181,14 @@ public class AutoScalerOptions {
autoScalerConfig("history.max.age")
.durationType()
.defaultValue(Duration.ofHours(24))
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("history.max.age"))
.withDescription("Maximum age for past scaling decisions to retain.");
public static final ConfigOption<MetricAggregator> BUSY_TIME_AGGREGATOR =
autoScalerConfig("metrics.busy-time.aggregator")
.enumType(MetricAggregator.class)
.defaultValue(MetricAggregator.MAX)
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("metrics.busy-time.aggregator"))
.withDescription(
"Metric aggregator to use for busyTime metrics. This affects how true processing/output rate will be computed. Using max allows us to handle jobs with data skew more robustly, while avg may provide better stability when we know that the load distribution is even.");
@@ -175,6 +197,7 @@ public class AutoScalerOptions {
.stringType()
.asList()
.defaultValues()
+ .withDeprecatedKeys(deprecatedOperatorConfigKey("vertex.exclude.ids"))
.withDescription(
"A (semicolon-separated) list of vertex ids in hexstring for which to disable scaling. Caution: For non-sink vertices this will still scale their downstream operators until https://issues.apache.org/jira/browse/FLINK-31215 is implemented.");
[flink-kubernetes-operator] 01/04: [FLINK-33097][autoscaler] Initialize the generic autoscaler module and interfaces
Posted by gy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
commit 5c76d01c79a91a255b40e0f08d384d7b0326bea3
Author: 1996fanrui <19...@gmail.com>
AuthorDate: Mon Sep 18 16:04:50 2023 +0800
[FLINK-33097][autoscaler] Initialize the generic autoscaler module and interfaces
---
flink-autoscaler/pom.xml | 51 +++++++++++++++
.../org/apache/flink/autoscaler/JobAutoScaler.java | 35 ++++++++++
.../flink/autoscaler/JobAutoScalerContext.java | 62 ++++++++++++++++++
.../autoscaler/event/AutoScalerEventHandler.java | 55 ++++++++++++++++
.../flink/autoscaler/realizer/ScalingRealizer.java | 36 +++++++++++
.../autoscaler/state/AutoScalerStateStore.java | 74 ++++++++++++++++++++++
pom.xml | 1 +
7 files changed, 314 insertions(+)
diff --git a/flink-autoscaler/pom.xml b/flink-autoscaler/pom.xml
new file mode 100644
index 00000000..709e5cca
--- /dev/null
+++ b/flink-autoscaler/pom.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-kubernetes-operator-parent</artifactId>
+ <version>1.7-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-autoscaler</artifactId>
+ <name>Flink Autoscaler</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime</artifactId>
+ <version>${flink.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>${flink.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScaler.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScaler.java
new file mode 100644
index 00000000..ff2b7331
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScaler.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * The generic Autoscaler.
+ *
+ * @param <KEY> The job key.
+ */
+@Internal
+public interface JobAutoScaler<KEY, Context extends JobAutoScalerContext<KEY>> {
+
+ /** Called as part of the reconciliation loop. */
+ void scale(Context context) throws Exception;
+
+ /** Called when the job is deleted. */
+ void cleanup(KEY key);
+}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerContext.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerContext.java
new file mode 100644
index 00000000..9b60291c
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/JobAutoScalerContext.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.client.program.rest.RestClusterClient;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.util.function.SupplierWithException;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.ToString;
+
+import javax.annotation.Nullable;
+
+/**
+ * The job autoscaler context, it includes all details related to the current job.
+ *
+ * @param <KEY> The job key.
+ */
+@Experimental
+@AllArgsConstructor
+@ToString
+public class JobAutoScalerContext<KEY> {
+
+ /** The identifier of each flink job. */
+ @Getter private final KEY jobKey;
+
+ /** The jobId and jobStatus can be null when the job isn't started. */
+ @Nullable @Getter private final JobID jobID;
+
+ @Nullable @Getter private final JobStatus jobStatus;
+
+ @Getter private final Configuration configuration;
+
+ @Getter private final MetricGroup metricGroup;
+
+ @ToString.Exclude
+ private final SupplierWithException<RestClusterClient<String>, Exception> restClientSupplier;
+
+ public RestClusterClient<String> getRestClusterClient() throws Exception {
+ return restClientSupplier.get();
+ }
+}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/event/AutoScalerEventHandler.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/event/AutoScalerEventHandler.java
new file mode 100644
index 00000000..9fafc686
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/event/AutoScalerEventHandler.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.event;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+
+/**
+ * Handler all loggable events during scaling.
+ *
+ * @param <KEY> The job key.
+ * @param <Context> Instance of JobAutoScalerContext.
+ */
+@Experimental
+public interface AutoScalerEventHandler<KEY, Context extends JobAutoScalerContext<KEY>> {
+
+ /**
+ * Handle the event.
+ *
+ * @param interval When interval is great than 0, events that repeat within the interval will be
+ * ignored.
+ */
+ void handleEvent(
+ Context context,
+ Type type,
+ String reason,
+ String message,
+ @Nullable String messageKey,
+ @Nullable Duration interval);
+
+ /** The type of the events. */
+ enum Type {
+ Normal,
+ Warning
+ }
+}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/realizer/ScalingRealizer.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/realizer/ScalingRealizer.java
new file mode 100644
index 00000000..36184dad
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/realizer/ScalingRealizer.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.realizer;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+
+import java.util.Map;
+
+/**
+ * The Scaling Realizer is responsible for managing scaling actions.
+ *
+ * @param <KEY> The job key.
+ * @param <Context> Instance of JobAutoScalerContext.
+ */
+@Experimental
+public interface ScalingRealizer<KEY, Context extends JobAutoScalerContext<KEY>> {
+
+ /** Update job's parallelism to parallelismOverrides. */
+ void realize(Context context, Map<String, String> parallelismOverrides);
+}
diff --git a/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/AutoScalerStateStore.java b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/AutoScalerStateStore.java
new file mode 100644
index 00000000..4551ef54
--- /dev/null
+++ b/flink-autoscaler/src/main/java/org/apache/flink/autoscaler/state/AutoScalerStateStore.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.autoscaler.state;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.autoscaler.JobAutoScalerContext;
+import org.apache.flink.autoscaler.ScalingSummary;
+import org.apache.flink.autoscaler.metrics.CollectedMetrics;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.Optional;
+import java.util.SortedMap;
+
+/**
+ * The state store is responsible for storing all state during scaling.
+ *
+ * @param <KEY> The job key.
+ * @param <Context> Instance of JobAutoScalerContext.
+ */
+@Experimental
+public interface AutoScalerStateStore<KEY, Context extends JobAutoScalerContext<KEY>> {
+
+ void storeScalingHistory(
+ Context jobContext, Map<JobVertexID, SortedMap<Instant, ScalingSummary>> scalingHistory)
+ throws Exception;
+
+ Optional<Map<JobVertexID, SortedMap<Instant, ScalingSummary>>> getScalingHistory(
+ Context jobContext) throws Exception;
+
+ void removeScalingHistory(Context jobContext) throws Exception;
+
+ void storeEvaluatedMetrics(
+ Context jobContext, SortedMap<Instant, CollectedMetrics> evaluatedMetrics)
+ throws Exception;
+
+ Optional<SortedMap<Instant, CollectedMetrics>> getEvaluatedMetrics(Context jobContext)
+ throws Exception;
+
+ void removeEvaluatedMetrics(Context jobContext) throws Exception;
+
+ void storeParallelismOverrides(Context jobContext, Map<String, String> parallelismOverrides)
+ throws Exception;
+
+ Optional<Map<String, String>> getParallelismOverrides(Context jobContext) throws Exception;
+
+ void removeParallelismOverrides(Context jobContext) throws Exception;
+
+ /**
+ * Flushing is needed because we just save data in cache for all store methods. For less write
+ * operations, we flush the cached data to the physical storage only after all operations have
+ * been performed.
+ */
+ void flush(Context jobContext) throws Exception;
+
+ /** Clean up all information related to the current job. */
+ void removeInfoFromCache(KEY jobKey);
+}
diff --git a/pom.xml b/pom.xml
index d90c72de..f18a417e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,6 +57,7 @@ under the License.
<module>flink-kubernetes-operator-api</module>
<module>flink-kubernetes-webhook</module>
<module>flink-kubernetes-docs</module>
+ <module>flink-autoscaler</module>
<module>examples/flink-sql-runner-example</module>
<module>examples/flink-beam-example</module>
<module>examples/kubernetes-client-examples</module>