[GitHub] cbickel closed pull request #3126: Throttle log messages when rescheduling containers

[GitBox <gi...@apache.org>]

cbickel closed pull request #3126: Throttle log messages when rescheduling containers
URL: https://github.com/apache/incubator-openwhisk/pull/3126
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala
index 5177026fed..1254ba8c75 100644
--- a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala
+++ b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerPool.scala
@@ -71,6 +71,7 @@ class ContainerPool(childFactory: ActorRefFactory => ActorRef,
   var freePool = immutable.Map.empty[ActorRef, ContainerData]
   var busyPool = immutable.Map.empty[ActorRef, ContainerData]
   var prewarmedPool = immutable.Map.empty[ActorRef, ContainerData]
+  val logRetriesInterval = 1000
 
   prewarmConfig.foreach { config =>
     logging.info(this, s"pre-warming ${config.count} ${config.exec.kind} containers")(TransactionId.invokerWarmup)
@@ -110,8 +111,15 @@ class ContainerPool(childFactory: ActorRefFactory => ActorRef,
           freePool = freePool - actor
           actor ! r // forwards the run request to the container
         case None =>
-          logging.error(this, "Rescheduling Run message, too many message in the pool")(r.msg.transid)
-          self ! r
+          if (r.retryCount % logRetriesInterval == 0) {
+            logging.error(
+              this,
+              s"Rescheduling Run message, too many message in the pool, freePoolSize: ${freePool.size}, " +
+                s"busyPoolSize: ${busyPool.size}, maxActiveContainers $maxActiveContainers, userNamespace: ${r.msg.user.namespace}, action: ${r.action}, retries: ${r.retryCount}")(
+              r.msg.transid)
+          }
+          val retryRun = Run(r.action, r.msg, r.retryCount + 1)
+          self ! retryRun
       }
 
     // Container is free to take more work
diff --git a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
index b76a60d8c0..efd7976199 100644
--- a/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
+++ b/core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
@@ -61,7 +61,7 @@ case class WarmedData(container: Container,
 
 // Events received by the actor
 case class Start(exec: CodeExec[_], memoryLimit: ByteSize)
-case class Run(action: ExecutableWhiskAction, msg: ActivationMessage)
+case class Run(action: ExecutableWhiskAction, msg: ActivationMessage, retryCount: Long = 0)
 case object Remove
 
 // Events sent by the actor


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services