You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by mc...@apache.org on 2020/10/21 18:36:25 UTC
[openwhisk] branch mcdan/ephmeral-scale-factor updated (40025c5 ->
f76a6b3)
This is an automated email from the ASF dual-hosted git repository.
mcdan pushed a change to branch mcdan/ephmeral-scale-factor
in repository https://gitbox.apache.org/repos/asf/openwhisk.git.
discard 40025c5 Add ability to scale Ephemeral storage along with memory, similar to CPU.
new f76a6b3 Add ability to scale Ephemeral storage along with memory, similar to CPU.
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (40025c5)
\
N -- N -- N refs/heads/mcdan/ephmeral-scale-factor (f76a6b3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../containerpool/kubernetes/WhiskPodBuilder.scala | 8 +++----
.../kubernetes/test/WhiskPodBuilderTests.scala | 25 +++-------------------
2 files changed, 7 insertions(+), 26 deletions(-)
[openwhisk] 01/01: Add ability to scale Ephemeral storage along
with memory, similar to CPU.
Posted by mc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mcdan pushed a commit to branch mcdan/ephmeral-scale-factor
in repository https://gitbox.apache.org/repos/asf/openwhisk.git
commit f76a6b311bfb13c02f08c51ca3a7b3662c32a913
Author: mcweeney <mc...@adobe.com>
AuthorDate: Wed Oct 21 14:07:59 2020 -0400
Add ability to scale Ephemeral storage along with memory, similar to CPU.
---
core/invoker/src/main/resources/application.conf | 4 +++-
.../kubernetes/KubernetesClient.scala | 2 +-
.../containerpool/kubernetes/WhiskPodBuilder.scala | 7 ++++++-
.../kubernetes/test/WhiskPodBuilderTests.scala | 22 +++++++++++++++++++++-
4 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/core/invoker/src/main/resources/application.conf b/core/invoker/src/main/resources/application.conf
index a11f454..35c6fc9 100644
--- a/core/invoker/src/main/resources/application.conf
+++ b/core/invoker/src/main/resources/application.conf
@@ -110,10 +110,12 @@ whisk {
#}
#if missing, the pod will be created without ephermal disk request/limit
- #if specified, the pod will be created with ephemeral-storage request+limit set
+ #if specified, the pod will be created with ephemeral-storage request+limit set or using the scale factor
+ #as a multiple of the request memory. If both are set scale-factor takes precedence.
#See: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#local-ephemeral-storage
#ephemeral-storage {
# limit = 2 g
+ # scale-factor = 2.0
#}
#enable PodDisruptionBudget creation for pods? (will include same labels as pods, and specify minAvailable=1 to prevent termination of action pods during maintenance)
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala
index ca96dd4..b44b2c2 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala
@@ -74,7 +74,7 @@ case class KubernetesCpuScalingConfig(millicpus: Int, memory: ByteSize, maxMilli
/**
* Configuration for kubernetes ephemeral storage limit for the action container
*/
-case class KubernetesEphemeralStorageConfig(limit: ByteSize)
+case class KubernetesEphemeralStorageConfig(limit: ByteSize, scaleFactor: Double)
/**
* Exception to indicate a pod took too long to become ready.
diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala
index b5eda4d..da3b0c5 100644
--- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala
+++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala
@@ -108,7 +108,12 @@ class WhiskPodBuilder(client: NamespacedKubernetesClient, config: KubernetesClie
.getOrElse(Map.empty)
val diskLimit = config.ephemeralStorage
- .map(diskConfig => Map("ephemeral-storage" -> new Quantity(diskConfig.limit.toMB + "Mi")))
+ .map(diskConfig =>
+ if (diskConfig.scaleFactor > 0) {
+ Map("ephemeral-storage" -> new Quantity(diskConfig.scaleFactor * memory.toMB + "Mi"))
+ } else {
+ Map("ephemeral-storage" -> new Quantity(diskConfig.limit.toMB + "Mi"))
+ })
.getOrElse(Map.empty)
//In container its assumed that env, port, resource limits are set explicitly
diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala
index 93bc4b3..93117de 100644
--- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala
+++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala
@@ -133,7 +133,7 @@ class WhiskPodBuilderTests extends FlatSpec with Matchers with KubeClientSupport
Some(KubernetesCpuScalingConfig(300, 3.MB, 1000)),
false,
None,
- Some(KubernetesEphemeralStorageConfig(1.GB)))
+ Some(KubernetesEphemeralStorageConfig(1.GB, 0)))
val builder = new WhiskPodBuilder(kubeClient, config)
val (pod, _) = builder.buildPodSpec(name, testImage, 2.MB, Map("foo" -> "bar"), Map("fooL" -> "barV"), config)
@@ -143,6 +143,26 @@ class WhiskPodBuilderTests extends FlatSpec with Matchers with KubeClientSupport
c.getResources.getRequests.asScala.get("ephemeral-storage").map(_.getAmount) shouldBe Some("1024Mi")
}
}
+ it should "scale ephemeral storage when scale factor is given" in {
+ val config = KubernetesClientConfig(
+ KubernetesClientTimeoutConfig(1.second, 1.second),
+ KubernetesInvokerNodeAffinity(false, "k", "v"),
+ true,
+ None,
+ None,
+ Some(KubernetesCpuScalingConfig(300, 3.MB, 1000)),
+ false,
+ None,
+ Some(KubernetesEphemeralStorageConfig(1.GB, 1.25)))
+ val builder = new WhiskPodBuilder(kubeClient, config)
+
+ val (pod, _) = builder.buildPodSpec(name, testImage, 2.MB, Map("foo" -> "bar"), Map("fooL" -> "barV"), config)
+ withClue(Serialization.asYaml(pod)) {
+ val c = getActionContainer(pod)
+ c.getResources.getLimits.asScala.get("ephemeral-storage").map(_.getAmount) shouldBe Some("2.5Mi")
+ c.getResources.getRequests.asScala.get("ephemeral-storage").map(_.getAmount) shouldBe Some("2.5Mi")
+ }
+ }
it should "extend existing pod template" in {
val template = """