You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/01/18 18:16:21 UTC
[GitHub] chetanmeh commented on issue #3195: When running Invoker on "Bare-Metal" it cannot read log files
chetanmeh commented on issue #3195: When running Invoker on "Bare-Metal" it cannot read log files
URL: https://github.com/apache/incubator-openwhisk/issues/3195#issuecomment-358734535
> There is also a swallowed exception that happens if you run the Invoker as a non-root user, still trying to track that one down.
@mcdan I tried to track that down and it appears to be coming in collectLogs calling itself. Following rough patch demonstrates and fixes that
```patch
Index: core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala (revision 925500cf8d34bb75bebd077ea057af236eba0b01)
+++ core/invoker/src/main/scala/whisk/core/containerpool/ContainerProxy.scala (date 1516298930000)
@@ -38,6 +38,8 @@
import whisk.core.entity.ExecManifest.ImageName
import whisk.http.Messages
+import scala.util.Try
+
// States
sealed trait ContainerState
case object Uninitialized extends ContainerState
@@ -380,18 +382,25 @@
val activationWithLogs: Future[Either[ActivationLogReadingError, WhiskActivation]] = activation
.flatMap { activation =>
val start = tid.started(this, LoggingMarkers.INVOKER_COLLECT_LOGS)
- collectLogs(tid, job.msg.user, activation, container, job.action)
- .andThen {
- case Success(_) => tid.finished(this, start)
- case Failure(t) => tid.failed(this, start, s"reading logs failed: $t")
- }
- .map(logs => Right(activation.withLogs(logs)))
- .recover {
- case LogCollectingException(logs) =>
- Left(ActivationLogReadingError(activation.withLogs(logs)))
- case _ =>
- Left(ActivationLogReadingError(activation.withLogs(ActivationLogs(Vector(Messages.logFailure)))))
- }
+ Try(
+ collectLogs(tid, job.msg.user, activation, container, job.action)
+ .andThen {
+ case Success(_) => tid.finished(this, start)
+ case Failure(t) => tid.failed(this, start, s"reading logs failed: $t")
+ }
+ .map(logs => Right(activation.withLogs(logs)))
+ .recover {
+ case LogCollectingException(logs) =>
+ Left(ActivationLogReadingError(activation.withLogs(logs)))
+ case _ =>
+ Left(ActivationLogReadingError(activation.withLogs(ActivationLogs(Vector(Messages.logFailure)))))
+ }) match {
+ case Success(e) => e
+ case Failure(t) =>
+ tid.failed(this, start, s"reading logs failed: $t")
+ Future.successful(
+ Left(ActivationLogReadingError(activation.withLogs(ActivationLogs(Vector(Messages.logFailure))))))
+ }
}
// Storing the record. Entirely asynchronous and not waited upon.
Index: tests/src/test/scala/whisk/core/containerpool/test/ContainerProxyTests.scala
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
--- tests/src/test/scala/whisk/core/containerpool/test/ContainerProxyTests.scala (revision 925500cf8d34bb75bebd077ea057af236eba0b01)
+++ tests/src/test/scala/whisk/core/containerpool/test/ContainerProxyTests.scala (date 1516299060000)
@@ -507,6 +507,41 @@
}
}
+ it should "complete the transaction and destroy the container if log reading failed terminally - v2" in {
+ val container = new TestContainer
+ val factory = createFactory(Future.successful(container))
+ val acker = createAcker
+ val store = createStore
+ val collector = LoggedFunction {
+ (transid: TransactionId,
+ user: Identity,
+ activation: WhiskActivation,
+ container: Container,
+ action: ExecutableWhiskAction) =>
+ throw new Exception
+ }
+
+ val machine =
+ childActorOf(ContainerProxy.props(factory, acker, store, collector, InstanceId(0), pauseGrace = timeout))
+ registerCallback(machine)
+ machine ! Run(action, message)
+ expectMsg(Transition(machine, Uninitialized, Running))
+ expectMsg(ContainerRemoved) // The message is sent as soon as the container decides to destroy itself
+ expectMsg(Transition(machine, Running, Removing))
+
+ awaitAssert {
+ factory.calls should have size 1
+ container.initializeCount shouldBe 1
+ container.runCount shouldBe 1
+ collector.calls should have size 1
+ container.destroyCount shouldBe 1
+ acker.calls should have size 1
+ acker.calls(0)._2.response shouldBe ActivationResponse.success()
+ store.calls should have size 1
+ store.calls(0)._2.logs shouldBe ActivationLogs(Vector(Messages.logFailure))
+ }
+ }
+
it should "resend the job to the parent if resuming a container fails" in within(timeout) {
val container = new TestContainer {
override def resume()(implicit transid: TransactionId) = {
```
With this following log entry is seen
```
[2018-01-18T23:40:19.846Z] INFO [pool-2-thread-13] akka.actor.ActorSystemImpl - [#tid_49] [ContainerProxy] [marker:invoker_collectLogs_start:669]
[2018-01-18T23:40:19.848Z] WARN [pool-2-thread-13] akka.actor.ActorSystemImpl - [#tid_49] [ContainerProxy] reading logs failed: java.lang.IllegalArgumentException: Path '/project/workdir//data/openwhisk/invoker/containers/afc36bff93783c2dcd125743e7349b16e754b8d3fb8ff87e525eba94fb03dc32/afc36bff93783c2dcd125743e7349b16e754b8d3fb8ff87e525eba94fb03dc32-json.log' does not exist [marker:invoker_collectLogs_error:670:1]
[2018-01-18T23:40:19.849Z] INFO [pool-2-thread-13] whisk.core.invoker.Invoker$ - [#tid_49] [InvokerReactive] recording the activation result to the data store
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services