You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pekko.apache.org by fa...@apache.org on 2026/05/22 07:44:55 UTC
(pekko-projection) branch main updated: synch up code with akka-projection-r2dbc 1.4.0 (#484)
This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pekko-projection.git
The following commit(s) were added to refs/heads/main by this push:
new 09ebb42 synch up code with akka-projection-r2dbc 1.4.0 (#484)
09ebb42 is described below
commit 09ebb425f5a0d3754a1f626d4f7db660fc9de279
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Fri May 22 08:44:49 2026 +0100
synch up code with akka-projection-r2dbc 1.4.0 (#484)
* Port r2dbc changes from akka-projection 1.4.x
- Add warnAboutFilteredEventsInFlow setting to R2dbcProjectionSettings
- Update R2dbcOffsetStore with slice field in Record, sizeAfterEvict in State,
Validation sealed trait (Accepted/Duplicate/RejectedSeqNr/RejectedBacktrackingSeqNr)
- Update R2dbcProjectionImpl with new offset store validation API and
warnAboutFilteredEventsInFlow check in adaptedHandlerForFlow
- Add R2dbcReplication helpers (scaladsl and javadsl)
- Update BySliceSourceProviderAdapter, R2dbcHandlerAdapter, handler traits
- Port updated integration test files
Co-authored-by: Copilot <22...@users.noreply.github.com>
Co-authored-by: pjfanning <11...@users.noreply.github.com>
* Fix logback-test.xml and javadsl scaladoc references
Co-authored-by: Copilot <22...@users.noreply.github.com>
Co-authored-by: pjfanning <11...@users.noreply.github.com>
* Fix R2dbcOffsetStore.fromConfig call and ExecutionContexts.parasitic in r2dbc module
Agent-Logs-Url: https://github.com/pjfanning/incubator-pekko-projection/sessions/2e8fd0eb-1908-4b34-a538-181f6fafc2fe
Co-authored-by: pjfanning <11...@users.noreply.github.com>
* refactor
* some issues
* more updates
* update
* Update R2dbcOffsetStoreSpec.scala
* Fix compile error: use false for fromBacktracking in UpdatedDurableState case
* Update TestDbLifecycle.scala
* compile warnings
* Update R2dbcProjectionImpl.scala
* Create synch-akka-projection-r2dbc-1.4.0.excludes
* compile issues
* test runtime issue
* Update Dependencies.scala
* some more changes
* Update TestClock.scala
* more issues
* revert some changes
* Update R2dbcOffsetStoreSpec.scala
---------
Co-authored-by: copilot-swe-agent[bot] <19...@users.noreply.github.com>
Co-authored-by: pjfanning <11...@users.noreply.github.com>
---
build.sbt | 3 +-
project/Dependencies.scala | 16 +-
.../r2dbc/DurableStateEndToEndSpec.scala | 102 +++--
.../projection/r2dbc/EventSourcedChaosSpec.scala | 35 +-
.../r2dbc/EventSourcedEndToEndSpec.scala | 211 ++++++----
.../projection/r2dbc/EventSourcedPubSubSpec.scala | 40 +-
.../projection/r2dbc/R2dbcOffsetStoreSpec.scala | 9 +-
.../r2dbc/R2dbcOffsetStoreStateSpec.scala | 117 ++++--
.../projection/r2dbc/R2dbcProjectionSpec.scala | 128 +++---
.../r2dbc/R2dbcTimestampOffsetProjectionSpec.scala | 42 +-
.../r2dbc/R2dbcTimestampOffsetStoreSpec.scala | 450 +++++++++++++++------
.../apache/pekko/projection/r2dbc/TestClock.scala | 5 +-
.../apache/pekko/projection/r2dbc/TestData.scala | 5 +-
.../pekko/projection/r2dbc/TestDbLifecycle.scala | 13 +-
.../r2dbc/TestSourceProviderWithInput.scala | 2 +-
.../synch-akka-projection-r2dbc-1.4.0.excludes | 30 ++
r2dbc/src/main/resources/reference.conf | 4 +
.../projection/r2dbc/R2dbcProjectionSettings.scala | 29 +-
.../internal/BySliceSourceProviderAdapter.scala | 12 +-
.../projection/r2dbc/internal/OffsetPidSeqNr.scala | 3 +-
.../r2dbc/internal/R2dbcHandlerAdapter.scala | 2 +-
.../r2dbc/internal/R2dbcOffsetStore.scala | 421 ++++++++++---------
.../r2dbc/internal/R2dbcProjectionImpl.scala | 300 +++++++++-----
.../projection/r2dbc/javadsl/R2dbcHandler.scala | 2 +-
.../projection/r2dbc/javadsl/R2dbcProjection.scala | 8 +-
.../r2dbc/javadsl/R2dbcReplication.scala | 44 ++
.../projection/r2dbc/scaladsl/R2dbcHandler.scala | 2 +-
.../r2dbc/scaladsl/R2dbcProjection.scala | 3 +-
.../r2dbc/scaladsl/R2dbcReplication.scala | 41 ++
29 files changed, 1349 insertions(+), 730 deletions(-)
diff --git a/build.sbt b/build.sbt
index 0c78273..07b2651 100644
--- a/build.sbt
+++ b/build.sbt
@@ -188,6 +188,7 @@ lazy val r2dbc =
.settings(
name := "pekko-projection-r2dbc")
.dependsOn(core)
+ .dependsOn(grpc)
lazy val grpcTest =
Project(id = "grpc-test", base = file("grpc-test"))
@@ -222,7 +223,7 @@ lazy val grpcIntTest =
lazy val r2dbcIntTest =
Project(id = "r2dbc-int-test", base = file("r2dbc-int-test"))
.disablePlugins(MimaPlugin)
- .settings(Dependencies.r2dbc)
+ .settings(Dependencies.r2dbcIntTest)
.settings(
name := "pekko-projection-r2dbc-int-test",
publish / skip := true,
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index c961e21..970e302 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -80,6 +80,7 @@ object Dependencies {
object Test {
val pekkoDiscovery = "org.apache.pekko" %% "pekko-discovery" % Versions.pekko % "test"
+ val pekkoDistributedData = "org.apache.pekko" %% "pekko-distributed-data" % Versions.pekko % "test"
val pekkoSerializationJackson = "org.apache.pekko" %% "pekko-serialization-jackson" % Versions.pekko % "test"
val pekkoTypedTestkit = Compile.pekkoTypedTestkit
val pekkoStreamTestkit = Compile.pekkoStreamTestkit
@@ -253,9 +254,20 @@ object Dependencies {
Compile.pekkoPersistenceQuery,
Compile.r2dbcSpi,
Compile.r2dbcPool,
- Compile.r2dbcPostgres % "provided,test",
- Compile.r2dbcMysql % "provided,test",
+ Compile.r2dbcPostgres % "provided",
+ Compile.r2dbcMysql % "provided")
+
+ val r2dbcIntTest =
+ deps ++= Seq(
+ "org.apache.pekko" %% "pekko-persistence-r2dbc" % Versions.pekkoPersistenceR2dbc,
+ Compile.pekkoPersistenceQuery,
+ Compile.r2dbcSpi,
+ Compile.r2dbcPool,
+ Compile.r2dbcPostgres % "test",
+ Compile.r2dbcMysql % "test",
Test.pekkoSerializationJackson,
+ Test.pekkoDiscovery,
+ Test.pekkoDistributedData,
Test.pekkoTypedTestkit % "test",
Test.pekkoStreamTestkit % "test",
Test.logback % "test",
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/DurableStateEndToEndSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/DurableStateEndToEndSpec.scala
index a53226a..59db8b5 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/DurableStateEndToEndSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/DurableStateEndToEndSpec.scala
@@ -8,13 +8,12 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
import java.util.UUID
-
import scala.concurrent.Future
import org.apache.pekko
@@ -25,6 +24,7 @@ import pekko.actor.typed.ActorRef
import pekko.actor.typed.ActorSystem
import pekko.actor.typed.Behavior
import pekko.actor.typed.scaladsl.Behaviors
+import pekko.actor.typed.scaladsl.LoggerOps
import pekko.persistence.query.DurableStateChange
import pekko.persistence.query.UpdatedDurableState
import pekko.persistence.r2dbc.state.scaladsl.R2dbcDurableStateStore
@@ -69,32 +69,31 @@ object DurableStateEndToEndSpec {
def apply(pid: PersistenceId): Behavior[Command] = {
Behaviors.setup { context =>
- DurableStateBehavior[Command, Any](
- persistenceId = pid,
- "",
- { (_, command) =>
- command match {
- case command: Persist =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- DurableStateBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload)
- case command: PersistWithAck =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- DurableStateBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload).thenRun(_ => command.replyTo ! Done)
- case Ping(replyTo) =>
- replyTo ! Done
- Effect.none
- case Stop(replyTo) =>
- replyTo ! Done
- Effect.stop()
- }
+ DurableStateBehavior[Command, Any](persistenceId = pid, "",
+ {
+ (_, command) =>
+ command match {
+ case command: Persist =>
+ context.log.debugN(
+ "Persist [{}], pid [{}], seqNr [{}]",
+ command.payload,
+ pid.id,
+ DurableStateBehavior.lastSequenceNumber(context) + 1)
+ Effect.persist(command.payload)
+ case command: PersistWithAck =>
+ context.log.debugN(
+ "Persist [{}], pid [{}], seqNr [{}]",
+ command.payload,
+ pid.id,
+ DurableStateBehavior.lastSequenceNumber(context) + 1)
+ Effect.persist(command.payload).thenRun(_ => command.replyTo ! Done)
+ case Ping(replyTo) =>
+ replyTo ! Done
+ Effect.none
+ case Stop(replyTo) =>
+ replyTo ! Done
+ Effect.stop()
+ }
})
}
}
@@ -109,7 +108,7 @@ object DurableStateEndToEndSpec {
override def process(session: R2dbcSession, envelope: DurableStateChange[String]): Future[Done] = {
envelope match {
case upd: UpdatedDurableState[String] =>
- log.debug("{} Processed {} revision {}", projectionId.key, upd.value, upd.revision: java.lang.Long)
+ log.debugN("{} Processed {} revision {}", projectionId.key, upd.value, upd.revision)
case _ =>
}
processed :+= envelope
@@ -155,12 +154,8 @@ class DurableStateEndToEndSpec
sliceRanges.map { range =>
val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
val sourceProvider =
- DurableStateSourceProvider.changesBySlices[String](
- system,
- R2dbcDurableStateStore.Identifier,
- entityType,
- range.min,
- range.max)
+ DurableStateSourceProvider
+ .changesBySlices[String](system, R2dbcDurableStateStore.Identifier, entityType, range.min, range.max)
val projection = R2dbcProjection
.exactlyOnce(
projectionId,
@@ -228,28 +223,29 @@ class DurableStateEndToEndSpec
n += 1
}
- handlers.foreach { case (projectionId, handler) =>
- (0 until numberOfEntities).foreach { p =>
- val persistenceId = PersistenceId(entityType, s"p$p")
- val slice = DurableStateSourceProvider.sliceForPersistenceId(
- system,
- R2dbcDurableStateStore.Identifier,
- persistenceId.id)
- withClue(s"projectionId $projectionId, persistenceId $persistenceId, slice $slice: ") {
- if (handler.sliceRange.contains(slice)) {
- eventually {
- val updates = handler.processed.collect {
- case upd: UpdatedDurableState[String] if upd.persistenceId == persistenceId.id => upd
+ handlers.foreach {
+ case (projectionId, handler) =>
+ (0 until numberOfEntities).foreach { p =>
+ val persistenceId = PersistenceId(entityType, s"p$p")
+ val slice = DurableStateSourceProvider.sliceForPersistenceId(
+ system,
+ R2dbcDurableStateStore.Identifier,
+ persistenceId.id)
+ withClue(s"projectionId $projectionId, persistenceId $persistenceId, slice $slice: ") {
+ if (handler.sliceRange.contains(slice)) {
+ eventually {
+ val updates = handler.processed.collect {
+ case upd: UpdatedDurableState[String] if upd.persistenceId == persistenceId.id => upd
+ }
+ val revision = revisionPerEntity(p)
+ updates.last.revision shouldBe revision
+ updates.last.value shouldBe s"s$p-$revision"
+ // processed events in right order
+ updates shouldBe updates.sortBy(_.revision)
}
- val revision = revisionPerEntity(p)
- updates.last.revision shouldBe revision
- updates.last.value shouldBe s"s$p-$revision"
- // processed events in right order
- updates shouldBe updates.sortBy(_.revision)
}
}
}
- }
}
projections.foreach(_ ! ProjectionBehavior.Stop)
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedChaosSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedChaosSpec.scala
index d839b8f..273e341 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedChaosSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedChaosSpec.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -27,6 +27,7 @@ import pekko.actor.testkit.typed.scaladsl.LogCapturing
import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
import pekko.actor.typed.ActorRef
import pekko.actor.typed.ActorSystem
+import pekko.actor.typed.scaladsl.LoggerOps
import pekko.persistence.query.typed.EventEnvelope
import pekko.persistence.r2dbc.query.scaladsl.R2dbcReadJournal
import pekko.persistence.typed.PersistenceId
@@ -80,20 +81,20 @@ object EventSourcedChaosSpec {
val failCount = failEvents.getOrDefault(envelope.eventOption, 0)
if (failCount > 0) {
failEvents.put(envelope.event, failCount - 1)
- log.debug(
+ log.debugN(
"{} Fail event [{}], pid [{}], seqNr [{}]",
projectionId.key,
envelope.event,
envelope.persistenceId,
- envelope.sequenceNr: java.lang.Long)
+ envelope.sequenceNr)
throw TestException(s"Fail event [${envelope.event}]")
} else {
- log.debug(
+ log.debugN(
"{} Processed event [{}], pid [{}], seqNr [{}]",
projectionId.key,
envelope.event,
envelope.persistenceId,
- envelope.sequenceNr: java.lang.Long)
+ envelope.sequenceNr)
probe ! Processed(projectionId, envelope)
Future.successful(Done)
}
@@ -130,7 +131,8 @@ class EventSourcedChaosSpec
"handle all events exactlyOnce" in {
val rnd = new Random(seed)
- val numberOfIterations = system.settings.config.getInt("pekko.projection.r2dbc.ChaosSpec.nr-iterations")
+ val numberOfIterations =
+ system.settings.config.getInt("pekko.projection.r2dbc.ChaosSpec.nr-iterations")
val numberOfEntities = 20 + rnd.nextInt(80)
val numberOfProjections = 1 << rnd.nextInt(4) // 1 to 8 projections
val entityType = nextEntityType()
@@ -199,7 +201,7 @@ class EventSourcedChaosSpec
(1 to expectedEventCounts).foreach { _ =>
// not using receiveMessages(expectedEvents) for better logging in case of failure
try {
- processed :+= processedProbe.receiveMessage(30.seconds)
+ processed :+= processedProbe.receiveMessage(15.seconds)
} catch {
case e: AssertionError =>
val missing = expectedEvents.diff(processed.map(_.envelope.event))
@@ -221,13 +223,14 @@ class EventSourcedChaosSpec
val logMsg =
s"Iteration #$iteration. Processed [$expectedEventCounts] events in [$numberOfProjections] projections from [${byPid.size}] pids."
log.debug(logMsg)
- byPid.foreach { case (_, processedByPid) =>
- // all events of a pid must be processed by the same projection instance
- processedByPid.map(_.projectionId).toSet.size shouldBe 1
- // processed events in right order
- val seqNrs = processedByPid.map(_.envelope.sequenceNr).toVector
- (seqNrs.last - seqNrs.head + 1) shouldBe processedByPid.size
- seqNrs shouldBe (seqNrs.head to seqNrs.last).toVector
+ byPid.foreach {
+ case (_, processedByPid) =>
+ // all events of a pid must be processed by the same projection instance
+ processedByPid.map(_.projectionId).toSet.size shouldBe 1
+ // processed events in right order
+ val seqNrs = processedByPid.map(_.envelope.sequenceNr).toVector
+ (seqNrs.last - seqNrs.head + 1) shouldBe processedByPid.size
+ seqNrs shouldBe (seqNrs.head to seqNrs.last).toVector
}
info(logMsg)
@@ -279,11 +282,11 @@ class EventSourcedChaosSpec
val failCount = 1 + rnd.nextInt(3)
val i = rnd.nextInt(events.size)
failEvents.put(events(i), failCount)
- log.debug(
+ log.debugN(
"Persisting events [{}], it will fail [{}] in projection [{}] times",
events.mkString(", "),
events(i),
- failCount: java.lang.Integer)
+ failCount)
} else {
log.debug("Persisting events [{}]", events.mkString(", "))
}
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedEndToEndSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedEndToEndSpec.scala
index 7402d02..5aa1c50 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedEndToEndSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedEndToEndSpec.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -23,16 +23,16 @@ import org.apache.pekko
import pekko.Done
import pekko.actor.testkit.typed.scaladsl.LogCapturing
import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
+import pekko.actor.testkit.typed.scaladsl.TestProbe
import pekko.actor.typed.ActorRef
import pekko.actor.typed.ActorSystem
import pekko.actor.typed.Behavior
import pekko.actor.typed.scaladsl.Behaviors
+import pekko.actor.typed.scaladsl.LoggerOps
import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.r2dbc.Dialect
+import pekko.persistence.r2dbc.JournalSettings
import pekko.persistence.r2dbc.QuerySettings
-import pekko.persistence.r2dbc.internal.PayloadCodec
-import pekko.persistence.r2dbc.internal.PayloadCodec.RichStatement
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
+import pekko.persistence.r2dbc.internal.Sql.Interpolation
import pekko.persistence.r2dbc.query.scaladsl.R2dbcReadJournal
import pekko.persistence.typed.PersistenceId
import pekko.persistence.typed.scaladsl.Effect
@@ -75,42 +75,40 @@ object EventSourcedEndToEndSpec {
def apply(pid: PersistenceId): Behavior[Command] = {
Behaviors.setup { context =>
- EventSourcedBehavior[Command, Any, String](
- persistenceId = pid,
- "",
- { (_, command) =>
- command match {
- case command: Persist =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- EventSourcedBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload)
- case command: PersistWithAck =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- EventSourcedBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload).thenRun(_ => command.replyTo ! Done)
- case command: PersistAll =>
- if (context.log.isDebugEnabled)
- context.log.debug(
- "PersistAll [{}], pid [{}], seqNr [{}]",
- command.payloads.mkString(","),
+ EventSourcedBehavior[Command, Any, String](persistenceId = pid, "",
+ {
+ (_, command) =>
+ command match {
+ case command: Persist =>
+ context.log.debugN(
+ "Persist [{}], pid [{}], seqNr [{}]",
+ command.payload,
pid.id,
- EventSourcedBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payloads)
- case Ping(replyTo) =>
- replyTo ! Done
- Effect.none
- case Stop(replyTo) =>
- replyTo ! Done
- Effect.stop()
- }
- },
- (_, _) => "")
+ EventSourcedBehavior.lastSequenceNumber(context) + 1)
+ Effect.persist(command.payload)
+ case command: PersistWithAck =>
+ context.log.debugN(
+ "Persist [{}], pid [{}], seqNr [{}]",
+ command.payload,
+ pid.id,
+ EventSourcedBehavior.lastSequenceNumber(context) + 1)
+ Effect.persist(command.payload).thenRun(_ => command.replyTo ! Done)
+ case command: PersistAll =>
+ if (context.log.isDebugEnabled)
+ context.log.debugN(
+ "PersistAll [{}], pid [{}], seqNr [{}]",
+ command.payloads.mkString(","),
+ pid.id,
+ EventSourcedBehavior.lastSequenceNumber(context) + 1)
+ Effect.persist(command.payloads)
+ case Ping(replyTo) =>
+ replyTo ! Done
+ Effect.none
+ case Stop(replyTo) =>
+ replyTo ! Done
+ Effect.stop()
+ }
+ }, (_, _) => "")
}
}
}
@@ -122,7 +120,7 @@ object EventSourcedEndToEndSpec {
private val log = LoggerFactory.getLogger(getClass)
override def process(session: R2dbcSession, envelope: EventEnvelope[String]): Future[Done] = {
- log.debug("{} Processed {}", projectionId.key: Any, envelope.event: Any)
+ log.debug2("{} Processed {}", projectionId.key, envelope.event)
probe ! Processed(projectionId, envelope)
Future.successful(Done)
}
@@ -142,10 +140,10 @@ class EventSourcedEndToEndSpec
private val log = LoggerFactory.getLogger(getClass)
+ private val journalSettings = JournalSettings(system.settings.config.getConfig("pekko.persistence.r2dbc.journal"))
private val querySettings = QuerySettings(system.settings.config.getConfig("pekko.persistence.r2dbc.query"))
private val projectionSettings = R2dbcProjectionSettings(system)
private val stringSerializer = SerializationExtension(system).serializerFor(classOf[String])
- private implicit val journalPayloadCodec: PayloadCodec = querySettings.journalPayloadCodec
override protected def beforeAll(): Unit = {
super.beforeAll()
@@ -153,10 +151,9 @@ class EventSourcedEndToEndSpec
// to be able to store events with specific timestamps
private def writeEvent(persistenceId: String, seqNr: Long, timestamp: Instant, event: String): Unit = {
- log.debug("Write test event [{}] [{}] [{}] at time [{}]", persistenceId, seqNr: java.lang.Long, event, timestamp)
- implicit val dialect: Dialect = projectionSettings.dialect
+ log.debugN("Write test event [{}] [{}] [{}] at time [{}]", persistenceId, seqNr, event, timestamp)
val insertEventSql = sql"""
- INSERT INTO ${querySettings.journalTableWithSchema}
+ INSERT INTO ${journalSettings.journalTableWithSchema}
(slice, entity_type, persistence_id, seq_nr, db_timestamp, writer, adapter_manifest, event_ser_id, event_ser_manifest, event_payload)
VALUES (?, ?, ?, ?, ?, '', '', ?, '', ?)"""
@@ -172,7 +169,7 @@ class EventSourcedEndToEndSpec
.bind(3, seqNr)
.bind(4, timestamp)
.bind(5, stringSerializer.identifier)
- .bindPayload(6, stringSerializer.toBinary(event))
+ .bind(6, stringSerializer.toBinary(event))
}
result.futureValue shouldBe 1
}
@@ -187,12 +184,8 @@ class EventSourcedEndToEndSpec
sliceRanges.map { range =>
val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
val sourceProvider =
- EventSourcedProvider.eventsBySlices[String](
- system,
- R2dbcReadJournal.Identifier,
- entityType,
- range.min,
- range.max)
+ EventSourcedProvider
+ .eventsBySlices[String](system, R2dbcReadJournal.Identifier, entityType, range.min, range.max)
val projection = R2dbcProjection
.exactlyOnce(
projectionId,
@@ -209,6 +202,40 @@ class EventSourcedEndToEndSpec
"e" + (template + s).takeRight(5)
}
+ private def assertEventsProcessed(
+ expectedEvents: Vector[String],
+ processedProbe: TestProbe[Processed],
+ verifyProjectionId: Boolean): Unit = {
+ val expectedNumberOfEvents = expectedEvents.size
+ var processed = Vector.empty[Processed]
+
+ (1 to expectedNumberOfEvents).foreach { _ =>
+ // not using receiveMessages(expectedEvents) for better logging in case of failure
+ try {
+ processed :+= processedProbe.receiveMessage(15.seconds)
+ } catch {
+ case e: AssertionError =>
+ val missing = expectedEvents.diff(processed.map(_.envelope.event))
+ log.error(s"Processed [${processed.size}] events, but expected [$expectedNumberOfEvents]. " +
+ s"Missing [${missing.mkString(",")}]. " +
+ s"Received [${processed.map(p =>
+ s"(${p.envelope.event}, ${p.envelope.persistenceId}, ${p.envelope.sequenceNr})").mkString(", ")}]. ")
+ throw e
+ }
+ }
+
+ if (verifyProjectionId) {
+ val byPid = processed.groupBy(_.envelope.persistenceId)
+ byPid.foreach {
+ case (_, processedByPid) =>
+ // all events of a pid must be processed by the same projection instance
+ processedByPid.map(_.projectionId).toSet.size shouldBe 1
+ // processed events in right order
+ processedByPid.map(_.envelope.sequenceNr).toVector shouldBe (1 to processedByPid.size).toVector
+ }
+ }
+ }
+
"A R2DBC projection with eventsBySlices source" must {
"handle all events exactlyOnce" in {
@@ -218,7 +245,7 @@ class EventSourcedEndToEndSpec
val entities = (0 until numberOfEntities).map { n =>
val persistenceId = PersistenceId(entityType, s"p$n")
- spawn(Persister(persistenceId), s"p$n")
+ spawn(Persister(persistenceId), s"$entityType-p$n")
}
// write some before starting the projections
@@ -267,31 +294,65 @@ class EventSourcedEndToEndSpec
n += 1
}
- var processed = Vector.empty[Processed]
val expectedEvents = (1 to numberOfEvents).map(mkEvent).toVector
- (1 to numberOfEvents).foreach { _ =>
- // not using receiveMessages(expectedEvents) for better logging in case of failure
- try {
- processed :+= processedProbe.receiveMessage(30.seconds)
- } catch {
- case e: AssertionError =>
- val missing = expectedEvents.diff(processed.map(_.envelope.event))
- log.error(s"Processed [${processed.size}] events, but expected [$numberOfEvents]. " +
- s"Missing [${missing.mkString(",")}]. " +
- s"Received [${processed.map(p =>
- s"(${p.envelope.event}, ${p.envelope.persistenceId}, ${p.envelope.sequenceNr})").mkString(", ")}]. ")
- throw e
- }
+ assertEventsProcessed(expectedEvents, processedProbe, verifyProjectionId = true)
+
+ projections.foreach(_ ! ProjectionBehavior.Stop)
+ }
+
+ "support change of slice distribution" in {
+ val numberOfEntities = 20
+ val numberOfEvents = numberOfEntities * 10
+ val entityType = nextEntityType()
+
+ val entities = (0 until numberOfEntities).map { n =>
+ val persistenceId = PersistenceId(entityType, s"p$n")
+ spawn(Persister(persistenceId), s"$entityType-p$n")
}
- val byPid = processed.groupBy(_.envelope.persistenceId)
- byPid.foreach { case (_, processedByPid) =>
- // all events of a pid must be processed by the same projection instance
- processedByPid.map(_.projectionId).toSet.size shouldBe 1
- // processed events in right order
- processedByPid.map(_.envelope.sequenceNr).toVector shouldBe (1 to processedByPid.size).toVector
+ val projectionName = UUID.randomUUID().toString
+ val processedProbe = createTestProbe[Processed]()
+ var projections = startProjections(entityType, projectionName, nrOfProjections = 4, processedProbe.ref)
+
+ (1 to numberOfEvents).foreach { n =>
+ val p = n % numberOfEntities
+ entities(p) ! Persister.Persist(mkEvent(n))
+
+ if (n % 10 == 0)
+ Thread.sleep(50)
+ else if (n % 25 == 0)
+ Thread.sleep(1500)
+
+ // stop projections
+ if (n == numberOfEvents / 4) {
+ val probe = createTestProbe()
+ projections.foreach { ref =>
+ ref ! ProjectionBehavior.Stop
+ probe.expectTerminated(ref)
+ }
+ }
+
+ // resume projections again but with more nrOfProjections
+ if (n == (numberOfEvents / 4) + 20)
+ projections = startProjections(entityType, projectionName, nrOfProjections = 8, processedProbe.ref)
+
+ // stop projections
+ if (n == numberOfEvents * 3 / 4) {
+ val probe = createTestProbe()
+ projections.foreach { ref =>
+ ref ! ProjectionBehavior.Stop
+ probe.expectTerminated(ref)
+ }
+ }
+
+ // resume projections again but with less nrOfProjections
+ if (n == (numberOfEvents * 3 / 4) + 20)
+ projections = startProjections(entityType, projectionName, nrOfProjections = 2, processedProbe.ref)
}
+ val expectedEvents = (1 to numberOfEvents).map(mkEvent).toVector
+ assertEventsProcessed(expectedEvents, processedProbe, verifyProjectionId = false)
+
projections.foreach(_ ! ProjectionBehavior.Stop)
}
@@ -330,12 +391,14 @@ class EventSourcedEndToEndSpec
// but backtracking can fill in the gaps, backtracking will pick up pid3 seqNr 8 and 9
writeEvent(pid3, 8L, startTime.plusMillis(3), "e3-8")
val possibleDelay =
- querySettings.backtrackingBehindCurrentTime + querySettings.refreshInterval + processedProbe.remainingOrDefault
+ querySettings.backtrackingBehindCurrentTime + querySettings.refreshInterval +
+ processedProbe.remainingOrDefault
processedProbe.receiveMessage(possibleDelay).envelope.event shouldBe "e3-8"
processedProbe.receiveMessage(possibleDelay).envelope.event shouldBe "e3-9"
projection ! ProjectionBehavior.Stop
}
+
}
}
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedPubSubSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedPubSubSpec.scala
index 17f6967..cfb60b9 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedPubSubSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedPubSubSpec.scala
@@ -8,13 +8,12 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
import java.util.UUID
-
import scala.concurrent.ExecutionContext
import scala.concurrent.Future
import scala.concurrent.duration._
@@ -121,12 +120,8 @@ class EventSourcedPubSubSpec
sliceRanges.map { range =>
val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
val sourceProvider =
- EventSourcedProvider.eventsBySlices[String](
- system,
- R2dbcReadJournal.Identifier,
- entityType,
- range.min,
- range.max)
+ EventSourcedProvider
+ .eventsBySlices[String](system, R2dbcReadJournal.Identifier, entityType, range.min, range.max)
val projection = R2dbcProjection
.exactlyOnce(
projectionId,
@@ -150,7 +145,7 @@ class EventSourcedPubSubSpec
(1 to numberOfEvents).foreach { _ =>
// not using receiveMessages(expectedEvents) for better logging in case of failure
try {
- processed :+= processedProbe.receiveMessage(30.seconds)
+ processed :+= processedProbe.receiveMessage(25.seconds)
} catch {
case e: AssertionError =>
val missing = expectedEvents.diff(processed.map(_.envelope.event))
@@ -227,19 +222,20 @@ class EventSourcedPubSubSpec
processed ++= expectProcessed(processedProbe, numberOfEvents - 10 + 1, numberOfEvents)
val byPid = processed.groupBy(_.envelope.persistenceId)
- byPid.foreach { case (pid, processedByPid) =>
- // all events of a pid must be processed by the same projection instance
- processedByPid.map(_.projectionId).toSet.size shouldBe 1
- // processed events in right order
- processedByPid.map(_.envelope.sequenceNr) shouldBe (1 to processedByPid.size).toVector
-
- val viaPubSub =
- processedByPid.filter(p =>
- p.envelope.offset.asInstanceOf[TimestampOffset].timestamp ==
- p.envelope.offset
- .asInstanceOf[TimestampOffset]
- .readTimestamp)
- log.info("via pub-sub {}: {}", pid: Any, viaPubSub.map(_.envelope.sequenceNr).mkString(", "): Any)
+ byPid.foreach {
+ case (pid, processedByPid) =>
+ // all events of a pid must be processed by the same projection instance
+ processedByPid.map(_.projectionId).toSet.size shouldBe 1
+ // processed events in right order
+ processedByPid.map(_.envelope.sequenceNr) shouldBe (1 to processedByPid.size).toVector
+
+ val viaPubSub =
+ processedByPid.filter(p =>
+ p.envelope.offset.asInstanceOf[TimestampOffset].timestamp ==
+ p.envelope.offset
+ .asInstanceOf[TimestampOffset]
+ .readTimestamp)
+ log.info2("via pub-sub {}: {}", pid, viaPubSub.map(_.envelope.sequenceNr).mkString(", "))
}
val countViaPubSub = processed.count(p =>
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreSpec.scala
index 5e123d3..0276314 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreSpec.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -22,8 +22,7 @@ import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
import pekko.actor.typed.ActorSystem
import pekko.persistence.query.Sequence
import pekko.persistence.query.TimeBasedUUID
-import pekko.persistence.r2dbc.Dialect
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
+import pekko.persistence.r2dbc.internal.Sql.Interpolation
import pekko.projection.MergeableOffset
import pekko.projection.ProjectionId
import pekko.projection.internal.ManagementState
@@ -50,7 +49,6 @@ class R2dbcOffsetStoreSpec
private val table = settings.offsetTableWithSchema
- implicit val dialect: Dialect = settings.dialect
def selectLastSql: String =
sql"SELECT * FROM $table WHERE projection_name = ? AND projection_key = ?"
@@ -62,7 +60,7 @@ class R2dbcOffsetStoreSpec
.createStatement(selectLastSql)
.bind(0, projectionId.name)
.bind(1, projectionId.key),
- row => Instant.ofEpochMilli(row.get[java.lang.Long]("last_updated", classOf[java.lang.Long])))
+ row => Instant.ofEpochMilli(row.get("last_updated", classOf[java.lang.Long]).longValue()))
.futureValue
.getOrElse(throw new RuntimeException(s"no records found for $projectionId"))
}
@@ -257,7 +255,6 @@ class R2dbcOffsetStoreSpec
"read and save paused" in {
val projectionId = genRandomProjectionId()
val offsetStore = createOffsetStore(projectionId)
-
offsetStore.readManagementState().futureValue shouldBe None
offsetStore.savePaused(paused = true).futureValue
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreStateSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreStateSpec.scala
index 531d8df..f79eb82 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreStateSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreStateSpec.scala
@@ -8,13 +8,17 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
+import java.time.Instant
+
import org.apache.pekko
+import pekko.projection.r2dbc.internal.R2dbcOffsetStore.Pid
import pekko.projection.r2dbc.internal.R2dbcOffsetStore.Record
+import pekko.projection.r2dbc.internal.R2dbcOffsetStore.SeqNr
import pekko.projection.r2dbc.internal.R2dbcOffsetStore.State
import org.scalatest.TestSuite
import org.scalatest.matchers.should.Matchers
@@ -22,17 +26,26 @@ import org.scalatest.wordspec.AnyWordSpec
class R2dbcOffsetStoreStateSpec extends AnyWordSpec with TestSuite with Matchers {
+ def createRecord(pid: Pid, seqNr: SeqNr, timestamp: Instant): Record = {
+ val slice = math.abs(pid.hashCode % 1024)
+ Record(slice, pid, seqNr, timestamp)
+ }
+
"R2dbcOffsetStore.State" should {
"add records and keep track of pids and latest offset" in {
val t0 = TestClock.nowMillis().instant()
val state1 = State.empty
- .add(Vector(Record("p1", 1, t0), Record("p1", 2, t0.plusMillis(1)), Record("p1", 3, t0.plusMillis(2))))
+ .add(
+ Vector(
+ createRecord("p1", 1, t0),
+ createRecord("p1", 2, t0.plusMillis(1)),
+ createRecord("p1", 3, t0.plusMillis(2))))
state1.byPid("p1").seqNr shouldBe 3L
state1.latestTimestamp shouldBe t0.plusMillis(2)
state1.latestOffset.get.seen shouldBe Map("p1" -> 3L)
state1.oldestTimestamp shouldBe t0
- val state2 = state1.add(Vector(Record("p2", 2, t0.plusMillis(1))))
+ val state2 = state1.add(Vector(createRecord("p2", 2, t0.plusMillis(1))))
state2.byPid("p1").seqNr shouldBe 3L
state2.byPid("p2").seqNr shouldBe 2L
// latest not updated because timestamp of p2 was before latest
@@ -40,7 +53,7 @@ class R2dbcOffsetStoreStateSpec extends AnyWordSpec with TestSuite with Matchers
state2.latestOffset.get.seen shouldBe Map("p1" -> 3L)
state2.oldestTimestamp shouldBe t0
- val state3 = state2.add(Vector(Record("p3", 10, t0.plusMillis(3))))
+ val state3 = state2.add(Vector(createRecord("p3", 10, t0.plusMillis(3))))
state3.byPid("p1").seqNr shouldBe 3L
state3.byPid("p2").seqNr shouldBe 2L
state3.byPid("p3").seqNr shouldBe 10L
@@ -53,7 +66,12 @@ class R2dbcOffsetStoreStateSpec extends AnyWordSpec with TestSuite with Matchers
"include highest seqNr in seen of latestOffset" in {
val t0 = TestClock.nowMillis().instant()
val records =
- Vector(Record("p4", 9, t0), Record("p2", 2, t0), Record("p3", 5, t0), Record("p2", 1, t0), Record("p1", 3, t0))
+ Vector(
+ createRecord("p4", 9, t0),
+ createRecord("p2", 2, t0),
+ createRecord("p3", 5, t0),
+ createRecord("p2", 1, t0),
+ createRecord("p1", 3, t0))
val state = State(records)
state.byPid("p2").seqNr shouldBe 2L
// p2 -> 2 should be included even though p2 -> 1 was added afterwards (same timestamp)
@@ -61,47 +79,94 @@ class R2dbcOffsetStoreStateSpec extends AnyWordSpec with TestSuite with Matchers
}
"evict old" in {
+ // these pids have the same slice 645, otherwise it will keep one for each slice
+ val p1 = "p500"
+ val p2 = "p621"
+ val p3 = "p742"
+ val p4 = "p863"
+ val p5 = "p984"
+
val t0 = TestClock.nowMillis().instant()
val state1 = State.empty
.add(
Vector(
- Record("p1", 1, t0),
- Record("p2", 2, t0.plusMillis(1)),
- Record("p3", 3, t0.plusMillis(2)),
- Record("p4", 4, t0.plusMillis(3)),
- Record("p5", 5, t0.plusMillis(4))))
- state1.latestOffset.get.seen shouldBe Map("p5" -> 5L)
+ createRecord(p1, 1, t0),
+ createRecord(p2, 2, t0.plusMillis(1)),
+ createRecord(p3, 3, t0.plusMillis(2)),
+ createRecord(p4, 4, t0.plusMillis(3)),
+ createRecord(p5, 5, t0.plusMillis(4))))
+ state1.latestOffset.get.seen shouldBe Map(p5 -> 5L)
state1.oldestTimestamp shouldBe t0
- state1.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map(
- "p1" -> 1L,
- "p2" -> 2L,
- "p3" -> 3L,
- "p4" -> 4L,
- "p5" -> 5L)
+ state1.byPid
+ .map { case (pid, r) => pid -> r.seqNr } shouldBe Map(p1 -> 1L, p2 -> 2L, p3 -> 3L, p4 -> 4L, p5 -> 5L)
val state2 = state1.evict(t0.plusMillis(2), keepNumberOfEntries = 1)
- state2.latestOffset.get.seen shouldBe Map("p5" -> 5L)
+ state2.latestOffset.get.seen shouldBe Map(p5 -> 5L)
state2.oldestTimestamp shouldBe t0.plusMillis(2)
- state2.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map("p3" -> 3L, "p4" -> 4L, "p5" -> 5L)
+ state2.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map(p3 -> 3L, p4 -> 4L, p5 -> 5L)
// keep all
state1.evict(t0.plusMillis(2), keepNumberOfEntries = 100) shouldBe state1
// keep 4
val state3 = state1.evict(t0.plusMillis(2), keepNumberOfEntries = 4)
- state3.latestOffset.get.seen shouldBe Map("p5" -> 5L)
+ state3.latestOffset.get.seen shouldBe Map(p5 -> 5L)
state3.oldestTimestamp shouldBe t0.plusMillis(1)
- state3.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map("p2" -> 2L, "p3" -> 3L, "p4" -> 4L, "p5" -> 5L)
+ state3.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map(p2 -> 2L, p3 -> 3L, p4 -> 4L, p5 -> 5L)
+ }
+
+ "evict old but keep latest for each slice" in {
+ val t0 = TestClock.nowMillis().instant()
+ val state1 = State.empty
+ .add(
+ Vector(
+ createRecord("p1", 1, t0),
+ createRecord("p92", 2, t0.plusMillis(1)),
+ createRecord("p108", 3, t0.plusMillis(2)),
+ createRecord("p4", 4, t0.plusMillis(3)),
+ createRecord("p5", 5, t0.plusMillis(4))))
+
+ state1.byPid("p1").slice shouldBe 449
+ state1.byPid("p92").slice shouldBe 905
+ state1.byPid("p108").slice shouldBe 905 // same slice as p92
+ state1.byPid("p4").slice shouldBe 452
+ state1.byPid("p5").slice shouldBe 453
+
+ state1.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map(
+ "p1" -> 1L,
+ "p92" -> 2L,
+ "p108" -> 3L,
+ "p4" -> 4L,
+ "p5" -> 5L)
+ state1.latestOffset.get.seen shouldBe Map("p5" -> 5L)
+ state1.oldestTimestamp shouldBe t0
+
+ val state2 = state1.evict(t0.plusMillis(2), keepNumberOfEntries = 1)
+ // note that p92 is evicted because it has same slice as p108
+ // p1 is kept because keeping one for each slice
+ state2.byPid
+ .map { case (pid, r) => pid -> r.seqNr } shouldBe Map("p1" -> 1L, "p108" -> 3L, "p4" -> 4L, "p5" -> 5L)
+ state2.latestOffset.get.seen shouldBe Map("p5" -> 5L)
+ state2.oldestTimestamp shouldBe t0
+
+ val state3 = state1.evict(t0.plusMillis(10), keepNumberOfEntries = 1)
+ // still keeping one for each slice
+ state3.byPid
+ .map { case (pid, r) => pid -> r.seqNr } shouldBe Map("p1" -> 1L, "p108" -> 3L, "p4" -> 4L, "p5" -> 5L)
}
"find duplicate" in {
val t0 = TestClock.nowMillis().instant()
val state =
- State(Vector(Record("p1", 1, t0), Record("p2", 2, t0.plusMillis(1)), Record("p3", 3, t0.plusMillis(2))))
- state.isDuplicate(Record("p1", 1, t0)) shouldBe true
- state.isDuplicate(Record("p1", 2, t0.plusMillis(10))) shouldBe false
- state.isDuplicate(Record("p2", 1, t0)) shouldBe true
- state.isDuplicate(Record("p4", 4, t0.plusMillis(10))) shouldBe false
+ State(
+ Vector(
+ createRecord("p1", 1, t0),
+ createRecord("p2", 2, t0.plusMillis(1)),
+ createRecord("p3", 3, t0.plusMillis(2))))
+ state.isDuplicate(createRecord("p1", 1, t0)) shouldBe true
+ state.isDuplicate(createRecord("p1", 2, t0.plusMillis(10))) shouldBe false
+ state.isDuplicate(createRecord("p2", 1, t0)) shouldBe true
+ state.isDuplicate(createRecord("p4", 4, t0.plusMillis(10))) shouldBe false
}
}
}
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSpec.scala
index 42fbfca..2aa118d 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSpec.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -17,7 +17,6 @@ import java.util.UUID
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicReference
-import scala.annotation.nowarn
import scala.annotation.tailrec
import scala.collection.immutable
import scala.concurrent.Await
@@ -34,8 +33,8 @@ import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
import pekko.actor.typed.ActorRef
import pekko.actor.typed.ActorSystem
import pekko.persistence.r2dbc.Dialect
+import pekko.persistence.r2dbc.internal.Sql.Interpolation
import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
import pekko.projection.HandlerRecoveryStrategy
import pekko.projection.OffsetVerification
import pekko.projection.OffsetVerification.VerificationFailure
@@ -99,7 +98,7 @@ object R2dbcProjectionSpec {
val table = "projection_spec_model"
val createTableSql: String =
- s"""|CREATE table IF NOT EXISTS $table (
+ s"""|CREATE table IF NOT EXISTS "$table" (
| id VARCHAR(255) NOT NULL,
| concatenated VARCHAR(255) NOT NULL,
| PRIMARY KEY(id)
@@ -112,8 +111,6 @@ object R2dbcProjectionSpec {
private val logger = LoggerFactory.getLogger(this.getClass)
- implicit private val dialect: Dialect = settings.dialect
-
def concatToText(id: String, payload: String): Future[Done] = {
val savedStrOpt = findById(id)
@@ -139,22 +136,22 @@ object R2dbcProjectionSpec {
private def upsert(concatStr: ConcatStr): Future[Done] = {
logger.debug("TestRepository.upsert: [{}]", concatStr)
- val stmtSql = dialect match {
+ val stmtSql = settings.dialect match {
case Dialect.Postgres | Dialect.Yugabyte =>
sql"""
- INSERT INTO $table (id, concatenated) VALUES (?, ?)
- ON CONFLICT (id)
- DO UPDATE SET
- id = excluded.id,
- concatenated = excluded.concatenated
- """
+ INSERT INTO "$table" (id, concatenated) VALUES (?, ?)
+ ON CONFLICT (id)
+ DO UPDATE SET
+ id = excluded.id,
+ concatenated = excluded.concatenated
+ """
case Dialect.MySQL =>
sql"""
- INSERT INTO $table (id, concatenated) VALUES (?, ?) AS excluded
- ON DUPLICATE KEY UPDATE
- id = excluded.id,
- concatenated = excluded.concatenated
- """
+ INSERT INTO $table (id, concatenated) VALUES (?, ?) AS excluded
+ ON DUPLICATE KEY UPDATE
+ id = excluded.id,
+ concatenated = excluded.concatenated
+ """
}
val stmt = session
.createStatement(stmtSql)
@@ -182,7 +179,6 @@ object R2dbcProjectionSpec {
}
}
-@nowarn
class R2dbcProjectionSpec
extends ScalaTestWithActorTestKit(TestConfig.config)
with AnyWordSpecLike
@@ -203,11 +199,9 @@ class R2dbcProjectionSpec
override protected def beforeAll(): Unit = {
super.beforeAll()
- Await.result(
- r2dbcExecutor.executeDdl("beforeAll createTable") { conn =>
+ Await.result(r2dbcExecutor.executeDdl("beforeAll createTable") { conn =>
conn.createStatement(TestRepository.createTableSql)
- },
- 10.seconds)
+ }, 10.seconds)
Await.result(
r2dbcExecutor.updateOne("beforeAll delete")(_.createStatement(s"delete from ${TestRepository.table}")),
10.seconds)
@@ -283,7 +277,7 @@ class R2dbcProjectionSpec
"A R2DBC exactly-once projection" must {
"persist projection and offset in the same write operation (transactional)" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -302,7 +296,7 @@ class R2dbcProjectionSpec
}
"skip failing events when using RecoveryStrategy.skip" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -325,7 +319,7 @@ class R2dbcProjectionSpec
}
"store offset for failing events when using RecoveryStrategy.skip" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -348,7 +342,7 @@ class R2dbcProjectionSpec
}
"skip failing events after retrying when using RecoveryStrategy.retryAndSkip" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -395,7 +389,7 @@ class R2dbcProjectionSpec
}
"fail after retrying when using RecoveryStrategy.retryAndFail" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -422,7 +416,7 @@ class R2dbcProjectionSpec
}
"restart from previous offset - fail with throwing an exception" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -450,7 +444,7 @@ class R2dbcProjectionSpec
}
"restart from previous offset - fail with bad insert on user code" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -488,9 +482,9 @@ class R2dbcProjectionSpec
"verify offsets before and after processing an envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
case class ProbeMessage(str: String, offset: Long)
val verificationProbe = createTestProbe[ProbeMessage]("verification")
@@ -530,9 +524,9 @@ class R2dbcProjectionSpec
}
"skip record if offset verification fails before processing envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val testVerification = (offset: Long) => {
if (offset == 3L)
@@ -556,9 +550,9 @@ class R2dbcProjectionSpec
}
"skip record if offset verification fails after processing envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val testVerification = (offset: Long) => {
if (offset == 3L)
@@ -584,7 +578,7 @@ class R2dbcProjectionSpec
"A R2DBC grouped projection" must {
"persist projection and offset in the same write operation (transactional)" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -627,7 +621,7 @@ class R2dbcProjectionSpec
}
"handle grouped async projection and store offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -663,7 +657,7 @@ class R2dbcProjectionSpec
"A R2DBC at-least-once projection" must {
"persist projection and offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -684,7 +678,7 @@ class R2dbcProjectionSpec
}
"skip failing events when using RecoveryStrategy.skip, save after 1" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -708,7 +702,7 @@ class R2dbcProjectionSpec
}
"skip failing events when using RecoveryStrategy.skip, save after 2" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -732,7 +726,7 @@ class R2dbcProjectionSpec
}
"restart from previous offset - handler throwing an exception, save after 1" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -777,7 +771,7 @@ class R2dbcProjectionSpec
}
"restart from previous offset - handler throwing an exception, save after 2" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -828,13 +822,12 @@ class R2dbcProjectionSpec
}
"save offset after number of elements" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
- import pekko.actor.typed.scaladsl.adapter._
val sourceProbe = new AtomicReference[TestPublisher.Probe[Envelope]]()
- val source = TestSource[Envelope]()(system.toClassic).mapMaterializedValue { probe =>
+ val source = TestSource[Envelope]().mapMaterializedValue { probe =>
sourceProbe.set(probe)
NotUsed
}
@@ -873,13 +866,12 @@ class R2dbcProjectionSpec
}
"save offset after idle duration" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
- import pekko.actor.typed.scaladsl.adapter._
val sourceProbe = new AtomicReference[TestPublisher.Probe[Envelope]]()
- val source = TestSource[Envelope]()(system.toClassic).mapMaterializedValue { probe =>
+ val source = TestSource[Envelope]().mapMaterializedValue { probe =>
sourceProbe.set(probe)
NotUsed
}
@@ -919,9 +911,9 @@ class R2dbcProjectionSpec
}
"verify offsets before processing an envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val verifiedProbe = createTestProbe[Long]()
val testVerification = (offset: Long) => {
@@ -949,9 +941,9 @@ class R2dbcProjectionSpec
}
"skip record if offset verification fails before processing envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val testVerification = (offset: Long) => {
if (offset == 3L)
@@ -976,7 +968,7 @@ class R2dbcProjectionSpec
}
"handle async projection and store offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -1009,7 +1001,7 @@ class R2dbcProjectionSpec
"A R2DBC flow projection" must {
"persist projection and offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -1087,9 +1079,9 @@ class R2dbcProjectionSpec
}
"call start and stop of the handler" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val handlerProbe = createTestProbe[String]()
val handler = new LifecycleHandler(handlerProbe.ref)
@@ -1125,9 +1117,9 @@ class R2dbcProjectionSpec
}
"call start and stop of the handler when using TestKit.runWithTestSink" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val handlerProbe = createTestProbe[String]()
val handler = new LifecycleHandler(handlerProbe.ref)
@@ -1164,9 +1156,9 @@ class R2dbcProjectionSpec
}
"call start and stop of handler when restarted" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val handlerProbe = createTestProbe[String]()
@volatile var _handler: Option[LifecycleHandler] = None
@@ -1235,9 +1227,9 @@ class R2dbcProjectionSpec
}
"call start and stop of handler when failed but no restart" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val handlerProbe = createTestProbe[String]()
val failOnceOnOffset = new AtomicInteger(4)
@@ -1263,9 +1255,9 @@ class R2dbcProjectionSpec
}
"be able to stop when retrying" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
+ createOffsetStore(projectionId)
val handlerProbe = createTestProbe[String]()
val handler = new LifecycleHandler(handlerProbe.ref, alwaysFailOnOffset = 4)
@@ -1295,7 +1287,7 @@ class R2dbcProjectionSpec
"R2dbcProjection management" must {
"restart from beginning when offset is cleared" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -1327,7 +1319,7 @@ class R2dbcProjectionSpec
}
"restart from updated offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
@@ -1360,7 +1352,7 @@ class R2dbcProjectionSpec
}
"pause projection" in {
- implicit val entityId: String = UUID.randomUUID().toString
+ implicit val entityId = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetProjectionSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetProjectionSpec.scala
index b16cbe6..0451967 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetProjectionSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetProjectionSpec.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -18,7 +18,6 @@ import java.time.{ Duration => JDuration }
import java.util.UUID
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicInteger
-
import scala.annotation.tailrec
import scala.collection.immutable
import scala.concurrent.Await
@@ -38,8 +37,8 @@ import pekko.persistence.query.TimestampOffset
import pekko.persistence.query.typed.EventEnvelope
import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
import pekko.persistence.query.typed.scaladsl.LoadEventQuery
-import pekko.persistence.typed.PersistenceId
import pekko.persistence.r2dbc.internal.EnvelopeOrigin
+import pekko.persistence.typed.PersistenceId
import pekko.projection.BySlicesSourceProvider
import pekko.projection.HandlerRecoveryStrategy
import pekko.projection.ProjectionBehavior
@@ -162,11 +161,9 @@ class R2dbcTimestampOffsetProjectionSpec
override protected def beforeAll(): Unit = {
super.beforeAll()
- Await.result(
- r2dbcExecutor.executeDdl("beforeAll createTable") { conn =>
+ Await.result(r2dbcExecutor.executeDdl("beforeAll createTable") { conn =>
conn.createStatement(TestRepository.createTableSql)
- },
- 10.seconds)
+ }, 10.seconds)
Await.result(
r2dbcExecutor.updateOne("beforeAll delete")(_.createStatement(s"delete from ${TestRepository.table}")),
10.seconds)
@@ -286,12 +283,8 @@ class R2dbcTimestampOffsetProjectionSpec
}
}
- def now(): Instant = {
- TestClock.nowMicros().instant()
- }
-
def createEnvelopesWithDuplicates(pid1: Pid, pid2: Pid): Vector[EventEnvelope[String]] = {
- val startTime = now()
+ val startTime = TestClock.nowMicros().instant()
Vector(
createEnvelope(pid1, 1, startTime, "e1-1"),
@@ -346,13 +339,14 @@ class R2dbcTimestampOffsetProjectionSpec
Future.successful(Done)
else {
val repo = TestRepository(session, settings)
- val results = envelopes.groupBy(_.persistenceId).map { case (pid, envs) =>
- repo.findById(pid).flatMap { existing =>
- val newConcatStr = envs.foldLeft(existing.getOrElse(ConcatStr(pid, ""))) { (acc, env) =>
- acc.concat(env.event)
+ val results = envelopes.groupBy(_.persistenceId).map {
+ case (pid, envs) =>
+ repo.findById(pid).flatMap { existing =>
+ val newConcatStr = envs.foldLeft(existing.getOrElse(ConcatStr(pid, ""))) { (acc, env) =>
+ acc.concat(env.event)
+ }
+ repo.update(pid, newConcatStr.text)
}
- repo.update(pid, newConcatStr.text)
- }
}
Future.sequence(results).map(_ => Done)
}
@@ -557,7 +551,7 @@ class R2dbcTimestampOffsetProjectionSpec
val pid2 = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- val startTime = now()
+ val startTime = TestClock.nowMicros().instant()
val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
val sourceProvider1 = createSourceProvider(envelopes1)
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider1)
@@ -669,7 +663,7 @@ class R2dbcTimestampOffsetProjectionSpec
val pid2 = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- val startTime = now()
+ val startTime = TestClock.nowMicros().instant()
val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
val sourceProvider1 = createBacktrackingSourceProvider(envelopes1)
implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider1)
@@ -818,7 +812,7 @@ class R2dbcTimestampOffsetProjectionSpec
val pid2 = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- val startTime = now()
+ val startTime = TestClock.nowMicros().instant()
val sourceProvider = new TestSourceProviderWithInput()
implicit val offsetStore: R2dbcOffsetStore =
R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
@@ -955,7 +949,7 @@ class R2dbcTimestampOffsetProjectionSpec
val pid2 = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- val startTime = now()
+ val startTime = TestClock.nowMicros().instant()
val sourceProvider = new TestSourceProviderWithInput()
implicit val offsetStore: R2dbcOffsetStore =
R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
@@ -1159,7 +1153,7 @@ class R2dbcTimestampOffsetProjectionSpec
val pid1 = UUID.randomUUID().toString
val pid2 = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- val startTime = now()
+ val startTime = TestClock.nowMicros().instant()
val sourceProvider = new TestSourceProviderWithInput()
implicit val offsetStore: R2dbcOffsetStore =
R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
@@ -1301,7 +1295,7 @@ class R2dbcTimestampOffsetProjectionSpec
val pid2 = UUID.randomUUID().toString
val projectionId = genRandomProjectionId()
- val startTime = now()
+ val startTime = TestClock.nowMicros().instant()
val sourceProvider = new TestSourceProviderWithInput()
implicit val offsetStore: R2dbcOffsetStore =
R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetStoreSpec.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetStoreSpec.scala
index b22fc82..c9d6d83 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetStoreSpec.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetStoreSpec.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -28,15 +28,14 @@ import pekko.persistence.query.UpdatedDurableState
import pekko.persistence.query.typed.EventEnvelope
import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
import pekko.persistence.query.typed.scaladsl.LoadEventQuery
-import pekko.persistence.typed.PersistenceId
import pekko.persistence.r2dbc.internal.EnvelopeOrigin
+import pekko.persistence.typed.PersistenceId
import pekko.projection.BySlicesSourceProvider
import pekko.projection.ProjectionId
import pekko.projection.internal.ManagementState
import pekko.projection.r2dbc.internal.OffsetPidSeqNr
import pekko.projection.r2dbc.internal.R2dbcOffsetStore
import pekko.projection.r2dbc.internal.R2dbcOffsetStore.Pid
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore.Record
import pekko.projection.r2dbc.internal.R2dbcOffsetStore.SeqNr
import com.typesafe.config.ConfigFactory
import org.scalatest.wordspec.AnyWordSpecLike
@@ -62,7 +61,8 @@ class R2dbcTimestampOffsetStoreSpec
.parseString("""
# to be able to test eviction
pekko.projection.r2dbc.offset-store.keep-number-of-entries = 0
- """).withFallback(TestConfig.config))
+ """)
+ .withFallback(TestConfig.config))
with AnyWordSpecLike
with TestDbLifecycle
with TestData
@@ -350,6 +350,7 @@ class R2dbcTimestampOffsetStoreSpec
}
"filter duplicates" in {
+ import R2dbcOffsetStore.Validation._
val projectionId = genRandomProjectionId()
val offsetStore = createOffsetStore(projectionId)
@@ -367,26 +368,30 @@ class R2dbcTimestampOffsetStoreSpec
val offset3 = TimestampOffset(clock.instant(), Map("p5" -> 10L))
offsetStore.saveOffset(OffsetPidSeqNr(offset3, "p5", 10L)).futureValue
- offsetStore.isDuplicate(Record("p5", 10, offset3.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p1", 4, offset2.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p3", 6, offset2.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p4", 9, offset2.timestamp)) shouldBe true
+ def env(pid: Pid, seqNr: SeqNr, timestamp: Instant): EventEnvelope[String] =
+ createEnvelope(pid, seqNr, timestamp, "evt")
- offsetStore.isDuplicate(Record("p1", 3, offset1.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p2", 1, offset1.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p3", 5, offset1.timestamp)) shouldBe true
+ offsetStore.validate(env("p5", 10, offset3.timestamp)).futureValue shouldBe Duplicate
+ offsetStore.validate(env("p1", 4, offset2.timestamp)).futureValue shouldBe Duplicate
+ offsetStore.validate(env("p3", 6, offset2.timestamp)).futureValue shouldBe Duplicate
+ offsetStore.validate(env("p4", 9, offset2.timestamp)).futureValue shouldBe Duplicate
- offsetStore.isDuplicate(Record("p1", 2, offset1.timestamp.minusMillis(1))) shouldBe true
- offsetStore.isDuplicate(Record("p5", 9, offset3.timestamp.minusMillis(1))) shouldBe true
+ offsetStore.validate(env("p1", 3, offset1.timestamp)).futureValue shouldBe Duplicate
+ offsetStore.validate(env("p2", 1, offset1.timestamp)).futureValue shouldBe Duplicate
+ offsetStore.validate(env("p3", 5, offset1.timestamp)).futureValue shouldBe Duplicate
- offsetStore.isDuplicate(Record("p5", 11, offset3.timestamp)) shouldBe false
- offsetStore.isDuplicate(Record("p5", 12, offset3.timestamp.plusMillis(1))) shouldBe false
+ offsetStore.validate(env("p1", 2, offset1.timestamp.minusMillis(1))).futureValue shouldBe Duplicate
+ offsetStore.validate(env("p5", 9, offset3.timestamp.minusMillis(1))).futureValue shouldBe Duplicate
- offsetStore.isDuplicate(Record("p6", 1, offset3.timestamp.plusMillis(2))) shouldBe false
- offsetStore.isDuplicate(Record("p7", 1, offset3.timestamp.minusMillis(1))) shouldBe false
+ offsetStore.validate(env("p5", 11, offset3.timestamp)).futureValue shouldNot be(Duplicate)
+ offsetStore.validate(env("p5", 12, offset3.timestamp.plusMillis(1))).futureValue shouldNot be(Duplicate)
+
+ offsetStore.validate(env("p6", 1, offset3.timestamp.plusMillis(2))).futureValue shouldNot be(Duplicate)
+ offsetStore.validate(env("p7", 1, offset3.timestamp.minusMillis(1))).futureValue shouldNot be(Duplicate)
}
"accept known sequence numbers and reject unknown" in {
+ import R2dbcOffsetStore.Validation._
val projectionId = genRandomProjectionId()
val eventTimestampQueryClock = TestClock.nowMicros()
val offsetStore = createOffsetStore(projectionId, eventTimestampQueryClock = eventTimestampQueryClock)
@@ -399,97 +404,84 @@ class R2dbcTimestampOffsetStoreSpec
// seqNr 1 is always accepted
val env1 = createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")
- offsetStore.isAccepted(env1).futureValue shouldBe true
- offsetStore.isAccepted(backtrackingEnvelope(env1)).futureValue shouldBe true
+ offsetStore.validate(env1).futureValue shouldBe Accepted
+ offsetStore.validate(backtrackingEnvelope(env1)).futureValue shouldBe Accepted
// but not if already inflight, seqNr 1 was accepted
offsetStore.addInflight(env1)
val env1Later = createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")
- offsetStore.isAccepted(env1Later).futureValue shouldBe false
- offsetStore.isAccepted(backtrackingEnvelope(env1Later)).futureValue shouldBe false
+ offsetStore.validate(env1Later).futureValue shouldBe Duplicate
+ offsetStore.validate(backtrackingEnvelope(env1Later)).futureValue shouldBe Duplicate
// subsequent seqNr is accepted
val env2 = createEnvelope("p4", 2L, startTime.plusMillis(2), "e4-2")
- offsetStore.isAccepted(env2).futureValue shouldBe true
- offsetStore.isAccepted(backtrackingEnvelope(env2)).futureValue shouldBe true
+ offsetStore.validate(env2).futureValue shouldBe Accepted
+ offsetStore.validate(backtrackingEnvelope(env2)).futureValue shouldBe Accepted
offsetStore.addInflight(env2)
// but not when gap
val envP4SeqNr4 = createEnvelope("p4", 4L, startTime.plusMillis(3), "e4-4")
- offsetStore.isAccepted(envP4SeqNr4).futureValue shouldBe false
+ offsetStore.validate(envP4SeqNr4).futureValue shouldBe RejectedSeqNr
// hard reject when gap from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(envP4SeqNr4))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unexpected sequence number.*")
+ offsetStore.validate(backtrackingEnvelope(envP4SeqNr4)).futureValue shouldBe RejectedBacktrackingSeqNr
// reject filtered event when gap
- offsetStore.isAccepted(filteredEnvelope(envP4SeqNr4)).futureValue shouldBe false
+ offsetStore.validate(filteredEnvelope(envP4SeqNr4)).futureValue shouldBe RejectedSeqNr
// hard reject when filtered event with gap from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(filteredEnvelope(envP4SeqNr4)))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unexpected sequence number.*")
+ offsetStore
+ .validate(backtrackingEnvelope(filteredEnvelope(envP4SeqNr4)))
+ .futureValue shouldBe RejectedBacktrackingSeqNr
// and not if later already inflight, seqNr 2 was accepted
- offsetStore.isAccepted(createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")).futureValue shouldBe false
+ offsetStore.validate(createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")).futureValue shouldBe Duplicate
// +1 to known is accepted
val env3 = createEnvelope("p1", 4L, startTime.plusMillis(4), "e1-4")
- offsetStore.isAccepted(env3).futureValue shouldBe true
+ offsetStore.validate(env3).futureValue shouldBe Accepted
// but not same
- offsetStore.isAccepted(createEnvelope("p3", 5L, startTime, "e3-5")).futureValue shouldBe false
+ offsetStore.validate(createEnvelope("p3", 5L, startTime, "e3-5")).futureValue shouldBe Duplicate
// but not same, even if it's 1
- offsetStore.isAccepted(createEnvelope("p2", 1L, startTime, "e2-1")).futureValue shouldBe false
+ offsetStore.validate(createEnvelope("p2", 1L, startTime, "e2-1")).futureValue shouldBe Duplicate
// and not less
- offsetStore.isAccepted(createEnvelope("p3", 4L, startTime, "e3-4")).futureValue shouldBe false
+ offsetStore.validate(createEnvelope("p3", 4L, startTime, "e3-4")).futureValue shouldBe Duplicate
offsetStore.addInflight(env3)
// and then it's not accepted again
- offsetStore.isAccepted(env3).futureValue shouldBe false
- offsetStore.isAccepted(backtrackingEnvelope(env3)).futureValue shouldBe false
+ offsetStore.validate(env3).futureValue shouldBe Duplicate
+ offsetStore.validate(backtrackingEnvelope(env3)).futureValue shouldBe Duplicate
// and not when later seqNr is inflight
- offsetStore.isAccepted(env2).futureValue shouldBe false
- offsetStore.isAccepted(backtrackingEnvelope(env2)).futureValue shouldBe false
+ offsetStore.validate(env2).futureValue shouldBe Duplicate
+ offsetStore.validate(backtrackingEnvelope(env2)).futureValue shouldBe Duplicate
// +1 to known, and then also subsequent are accepted (needed for grouped)
val env4 = createEnvelope("p3", 6L, startTime.plusMillis(5), "e3-6")
- offsetStore.isAccepted(env4).futureValue shouldBe true
+ offsetStore.validate(env4).futureValue shouldBe Accepted
offsetStore.addInflight(env4)
val env5 = createEnvelope("p3", 7L, startTime.plusMillis(6), "e3-7")
- offsetStore.isAccepted(env5).futureValue shouldBe true
+ offsetStore.validate(env5).futureValue shouldBe Accepted
offsetStore.addInflight(env5)
val env6 = createEnvelope("p3", 8L, startTime.plusMillis(7), "e3-8")
- offsetStore.isAccepted(env6).futureValue shouldBe true
+ offsetStore.validate(env6).futureValue shouldBe Accepted
offsetStore.addInflight(env6)
// reject unknown
val env7 = createEnvelope("p5", 7L, startTime.plusMillis(8), "e5-7")
- offsetStore.isAccepted(env7).futureValue shouldBe false
- (offsetStore.isAccepted(backtrackingEnvelope(env7)).failed.futureValue.getMessage should fullyMatch)
- .regex("Rejected envelope from backtracking.*unknown sequence number.*")
+ offsetStore.validate(env7).futureValue shouldBe RejectedSeqNr
+ offsetStore.validate(backtrackingEnvelope(env7)).futureValue shouldBe RejectedBacktrackingSeqNr
// but ok when previous is old
eventTimestampQueryClock.setInstant(startTime.minusSeconds(3600))
val env8 = createEnvelope("p5", 7L, startTime.plusMillis(5), "e5-7")
- offsetStore.isAccepted(env8).futureValue shouldBe true
+ offsetStore.validate(env8).futureValue shouldBe Accepted
eventTimestampQueryClock.setInstant(startTime)
offsetStore.addInflight(env8)
// and subsequent seqNr is accepted
val env9 = createEnvelope("p5", 8L, startTime.plusMillis(9), "e5-8")
- offsetStore.isAccepted(env9).futureValue shouldBe true
+ offsetStore.validate(env9).futureValue shouldBe Accepted
offsetStore.addInflight(env9)
// reject unknown filtered
val env10 = filteredEnvelope(createEnvelope("p6", 7L, startTime.plusMillis(10), "e6-7"))
- offsetStore.isAccepted(env10).futureValue shouldBe false
+ offsetStore.validate(env10).futureValue shouldBe RejectedSeqNr
// hard reject when unknown from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(env10))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unknown sequence number.*")
+ offsetStore.validate(backtrackingEnvelope(env10)).futureValue shouldBe RejectedBacktrackingSeqNr
// hard reject when unknown filtered event from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(filteredEnvelope(env10)))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unknown sequence number.*")
+ offsetStore
+ .validate(backtrackingEnvelope(filteredEnvelope(env10)))
+ .futureValue shouldBe RejectedBacktrackingSeqNr
// it's keeping the inflight that are not in the "stored" state
offsetStore.getInflight() shouldBe Map("p1" -> 4L, "p3" -> 8, "p4" -> 2L, "p5" -> 8)
@@ -504,6 +496,7 @@ class R2dbcTimestampOffsetStoreSpec
}
"update inflight on error and re-accept element" in {
+ import R2dbcOffsetStore.Validation._
val projectionId = genRandomProjectionId()
val offsetStore = createOffsetStore(projectionId)
@@ -514,7 +507,7 @@ class R2dbcTimestampOffsetStoreSpec
val envelope3 = createEnvelope("p1", 3L, startTime.plusMillis(2), "e1-2")
// seqNr 1 is always accepted
- offsetStore.isAccepted(envelope1).futureValue shouldBe true
+ offsetStore.validate(envelope1).futureValue shouldBe Accepted
offsetStore.addInflight(envelope1)
offsetStore.getInflight() shouldBe Map("p1" -> 1L)
offsetStore
@@ -523,19 +516,19 @@ class R2dbcTimestampOffsetStoreSpec
offsetStore.getInflight() shouldBe empty
// seqNr 2 is accepts since it follows seqNr 1 that is stored in state
- offsetStore.isAccepted(envelope2).futureValue shouldBe true
+ offsetStore.validate(envelope2).futureValue shouldBe Accepted
// simulate envelope processing error by not adding envelope2 to inflight
// seqNr 3 is not accepted, still waiting for seqNr 2
- offsetStore.isAccepted(envelope3).futureValue shouldBe false
+ offsetStore.validate(envelope3).futureValue shouldBe RejectedSeqNr
// offer seqNr 2 once again
- offsetStore.isAccepted(envelope2).futureValue shouldBe true
+ offsetStore.validate(envelope2).futureValue shouldBe Accepted
offsetStore.addInflight(envelope2)
offsetStore.getInflight() shouldBe Map("p1" -> 2L)
// offer seqNr 3 once more
- offsetStore.isAccepted(envelope3).futureValue shouldBe true
+ offsetStore.validate(envelope3).futureValue shouldBe Accepted
offsetStore.addInflight(envelope3)
offsetStore.getInflight() shouldBe Map("p1" -> 3L)
@@ -546,7 +539,8 @@ class R2dbcTimestampOffsetStoreSpec
offsetStore.getInflight() shouldBe empty
}
- "filter accepted" in {
+ "mapIsAccepted" in {
+ import R2dbcOffsetStore.Validation._
val projectionId = genRandomProjectionId()
val startTime = TestClock.nowMicros().instant()
val offsetStore = createOffsetStore(projectionId)
@@ -567,11 +561,17 @@ class R2dbcTimestampOffsetStoreSpec
// but not when previous is unknown
val env5 = createEnvelope("p3", 7L, startTime.plusMillis(5), "e3-7")
- offsetStore.filterAccepted(List(env1, env2, env3, env4, env5)).futureValue shouldBe List(env1, env2, env4)
+ offsetStore.validateAll(List(env1, env2, env3, env4, env5)).futureValue shouldBe List(
+ env1 -> Accepted,
+ env2 -> Accepted,
+ env3 -> RejectedSeqNr,
+ env4 -> Accepted,
+ env5 -> RejectedSeqNr)
}
"accept new revisions for durable state" in {
+ import R2dbcOffsetStore.Validation._
val projectionId = genRandomProjectionId()
val offsetStore = createOffsetStore(projectionId)
@@ -583,50 +583,50 @@ class R2dbcTimestampOffsetStoreSpec
// seqNr 1 is always accepted
val env1 = createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1")
- offsetStore.isAccepted(env1).futureValue shouldBe true
+ offsetStore.validate(env1).futureValue shouldBe Accepted
// but not if already inflight, seqNr 1 was accepted
offsetStore.addInflight(env1)
offsetStore
- .isAccepted(createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1"))
- .futureValue shouldBe false
+ .validate(createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1"))
+ .futureValue shouldBe Duplicate
// subsequent seqNr is accepted
val env2 = createUpdatedDurableState("p4", 2L, startTime.plusMillis(2), "s4-2")
- offsetStore.isAccepted(env2).futureValue shouldBe true
+ offsetStore.validate(env2).futureValue shouldBe Accepted
offsetStore.addInflight(env2)
// and also ok with gap
offsetStore
- .isAccepted(createUpdatedDurableState("p4", 4L, startTime.plusMillis(3), "s4-4"))
- .futureValue shouldBe true
+ .validate(createUpdatedDurableState("p4", 4L, startTime.plusMillis(3), "s4-4"))
+ .futureValue shouldBe Accepted
// and not if later already inflight, seqNr 2 was accepted
offsetStore
- .isAccepted(createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1"))
- .futureValue shouldBe false
+ .validate(createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1"))
+ .futureValue shouldBe Duplicate
// greater than known is accepted
val env3 = createUpdatedDurableState("p1", 4L, startTime.plusMillis(4), "s1-4")
- offsetStore.isAccepted(env3).futureValue shouldBe true
+ offsetStore.validate(env3).futureValue shouldBe Accepted
// but not same
- offsetStore.isAccepted(createUpdatedDurableState("p3", 5L, startTime, "s3-5")).futureValue shouldBe false
+ offsetStore.validate(createUpdatedDurableState("p3", 5L, startTime, "s3-5")).futureValue shouldBe Duplicate
// but not same, even if it's 1
- offsetStore.isAccepted(createUpdatedDurableState("p2", 1L, startTime, "s2-1")).futureValue shouldBe false
+ offsetStore.validate(createUpdatedDurableState("p2", 1L, startTime, "s2-1")).futureValue shouldBe Duplicate
// and not less
- offsetStore.isAccepted(createUpdatedDurableState("p3", 4L, startTime, "s3-4")).futureValue shouldBe false
+ offsetStore.validate(createUpdatedDurableState("p3", 4L, startTime, "s3-4")).futureValue shouldBe Duplicate
offsetStore.addInflight(env3)
// greater than known, and then also subsequent are accepted (needed for grouped)
val env4 = createUpdatedDurableState("p3", 8L, startTime.plusMillis(5), "s3-6")
- offsetStore.isAccepted(env4).futureValue shouldBe true
+ offsetStore.validate(env4).futureValue shouldBe Accepted
offsetStore.addInflight(env4)
val env5 = createUpdatedDurableState("p3", 9L, startTime.plusMillis(6), "s3-7")
- offsetStore.isAccepted(env5).futureValue shouldBe true
+ offsetStore.validate(env5).futureValue shouldBe Accepted
offsetStore.addInflight(env5)
val env6 = createUpdatedDurableState("p3", 20L, startTime.plusMillis(7), "s3-8")
- offsetStore.isAccepted(env6).futureValue shouldBe true
+ offsetStore.validate(env6).futureValue shouldBe Accepted
offsetStore.addInflight(env6)
// accept unknown
val env7 = createUpdatedDurableState("p5", 7L, startTime.plusMillis(8), "s5-7")
- offsetStore.isAccepted(env7).futureValue shouldBe true
+ offsetStore.validate(env7).futureValue shouldBe Accepted
offsetStore.addInflight(env7)
// it's keeping the inflight that are not in the "stored" state
@@ -650,50 +650,143 @@ class R2dbcTimestampOffsetStoreSpec
val startTime = TestClock.nowMicros().instant()
log.debug("Start time [{}]", startTime)
- offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map("p1" -> 1L)), "p1", 1L)).futureValue
+ // these pids have the same slice 645, otherwise it will keep one for each slice
+ val p1 = "p500"
+ val p2 = "p621"
+ val p3 = "p742"
+ val p4 = "p863"
+ val p5 = "p984"
+ val p6 = "p3080"
+ val p7 = "p4290"
+ val p8 = "p20180"
+
+ offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map(p1 -> 1L)), p1, 1L)).futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map(p2 -> 1L)), p2, 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map(p3 -> 1L)), p3, 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(evictInterval), Map(p4 -> 1L)), p4, 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(
+ OffsetPidSeqNr(
+ TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(1)), Map(p4 -> 1L)),
+ p4,
+ 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(
+ OffsetPidSeqNr(
+ TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(2)), Map(p5 -> 1L)),
+ p5,
+ 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(
+ OffsetPidSeqNr(
+ TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(3)), Map(p6 -> 1L)),
+ p6,
+ 3L))
+ .futureValue
+ offsetStore.getState().size shouldBe 6
+
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(10)), Map(p7 -> 1L)), p7, 1L))
+ .futureValue
+ offsetStore.getState().size shouldBe 7 // nothing evicted yet
+
+ offsetStore
+ .saveOffset(
+ OffsetPidSeqNr(
+ TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).minusSeconds(3)), Map(p8 -> 1L)),
+ p8,
+ 1L))
+ .futureValue
+ offsetStore.getState().size shouldBe 8 // still nothing evicted yet
+
+ offsetStore
+ .saveOffset(
+ OffsetPidSeqNr(
+ TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(1)), Map(p8 -> 2L)),
+ p8,
+ 2L))
+ .futureValue
+ offsetStore.getState().byPid.keySet shouldBe Set(p5, p6, p7, p8)
+
+ offsetStore
+ .saveOffset(
+ OffsetPidSeqNr(
+ TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(20)), Map(p8 -> 3L)),
+ p8,
+ 3L))
+ .futureValue
+ offsetStore.getState().byPid.keySet shouldBe Set(p7, p8)
+ }
+
+ "evict old records but keep latest for each slice" in {
+ val projectionId = genRandomProjectionId()
+ val evictSettings = settings.withTimeWindow(JDuration.ofSeconds(100)).withEvictInterval(JDuration.ofSeconds(10))
+ import evictSettings._
+ val offsetStore = createOffsetStore(projectionId, evictSettings)
+
+ val startTime = TestClock.nowMicros().instant()
+ log.debug("Start time [{}]", startTime)
+
+ val p1 = "p500" // slice 645
+ val p2 = "p92" // slice 905
+ val p3 = "p108" // slice 905
+ val p4 = "p863" // slice 645
+ val p5 = "p984" // slice 645
+ val p6 = "p3080" // slice 645
+ val p7 = "p4290" // slice 645
+ val p8 = "p20180" // slice 645
+
+ offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map(p1 -> 1L)), p1, 1L)).futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map("p2" -> 1L)), "p2", 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map(p2 -> 1L)), p2, 1L))
.futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map("p3" -> 1L)), "p3", 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map(p3 -> 1L)), p3, 1L))
.futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(evictInterval), Map("p4" -> 1L)), "p4", 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(evictInterval), Map(p4 -> 1L)), p4, 1L))
.futureValue
offsetStore
.saveOffset(
OffsetPidSeqNr(
- TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(1)), Map("p4" -> 1L)),
- "p4",
+ TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(1)), Map(p4 -> 1L)),
+ p4,
1L))
.futureValue
offsetStore
.saveOffset(
OffsetPidSeqNr(
- TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(2)), Map("p5" -> 1L)),
- "p5",
+ TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(2)), Map(p5 -> 1L)),
+ p5,
1L))
.futureValue
offsetStore
.saveOffset(
OffsetPidSeqNr(
- TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(3)), Map("p6" -> 1L)),
- "p6",
+ TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(3)), Map(p6 -> 1L)),
+ p6,
1L))
.futureValue
offsetStore.getState().size shouldBe 6
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(10)), Map("p7" -> 1L)), "p7",
- 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(10)), Map(p7 -> 1L)), p7, 1L))
.futureValue
offsetStore.getState().size shouldBe 7 // nothing evicted yet
offsetStore
.saveOffset(
OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).minusSeconds(3)), Map("p8" -> 1L)),
- "p8",
+ TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).minusSeconds(3)), Map(p8 -> 1L)),
+ p8,
1L))
.futureValue
offsetStore.getState().size shouldBe 8 // still nothing evicted yet
@@ -701,20 +794,21 @@ class R2dbcTimestampOffsetStoreSpec
offsetStore
.saveOffset(
OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(1)), Map("p8" -> 2L)),
- "p8",
+ TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(1)), Map(p8 -> 2L)),
+ p8,
2L))
.futureValue
- offsetStore.getState().byPid.keySet shouldBe Set("p5", "p6", "p7", "p8")
+ // also keeping p3 ("p108") for slice 905
+ offsetStore.getState().byPid.keySet shouldBe Set(p3, p5, p6, p7, p8)
offsetStore
.saveOffset(
OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(20)), Map("p8" -> 3L)),
- "p8",
+ TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(20)), Map(p8 -> 3L)),
+ p8,
3L))
.futureValue
- offsetStore.getState().byPid.keySet shouldBe Set("p7", "p8")
+ offsetStore.getState().byPid.keySet shouldBe Set(p3, p7, p8)
}
"delete old records" in {
@@ -726,27 +820,89 @@ class R2dbcTimestampOffsetStoreSpec
val startTime = TestClock.nowMicros().instant()
log.debug("Start time [{}]", startTime)
- offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map("p1" -> 1L)), "p1", 1L)).futureValue
+ // these pids have the same slice 645, otherwise it will keep one for each slice
+ val p1 = "p500"
+ val p2 = "p621"
+ val p3 = "p742"
+ val p4 = "p863"
+ val p5 = "p984"
+ val p6 = "p3080"
+ val p7 = "p4290"
+ val p8 = "p20180"
+
+ offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map(p1 -> 1L)), p1, 1L)).futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map(p2 -> 1L)), p2, 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map(p3 -> 1L)), p3, 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(3)), Map(p4 -> 1L)), p4, 1L))
+ .futureValue
+ offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 0
+ offsetStore.readOffset().futureValue // this will load from database
+ offsetStore.getState().size shouldBe 4
+
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(2)), Map(p5 -> 1L)), p5, 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(1)), Map(p6 -> 1L)), p6, 1L))
+ .futureValue
+ // nothing deleted yet
+ offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 0
+ offsetStore.readOffset().futureValue // this will load from database
+ offsetStore.getState().size shouldBe 6
+
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(1)), Map(p7 -> 1L)), p7, 1L))
+ .futureValue
+ offsetStore
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(3)), Map(p8 -> 1L)), p8, 1L))
+ .futureValue
+ offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 3
+ offsetStore.readOffset().futureValue // this will load from database
+ offsetStore.getState().byPid.keySet shouldBe Set(p4, p5, p6, p7, p8)
+ }
+
+ "delete old records but keep latest for each slice" in {
+ val projectionId = genRandomProjectionId()
+ val deleteSettings = settings.withTimeWindow(JDuration.ofSeconds(100))
+ import deleteSettings._
+ val offsetStore = createOffsetStore(projectionId, deleteSettings)
+
+ val startTime = TestClock.nowMicros().instant()
+ log.debug("Start time [{}]", startTime)
+
+ val p1 = "p500" // slice 645
+ val p2 = "p92" // slice 905
+ val p3 = "p108" // slice 905
+ val p4 = "p863" // slice 645
+ val p5 = "p984" // slice 645
+ val p6 = "p3080" // slice 645
+ val p7 = "p4290" // slice 645
+ val p8 = "p20180" // slice 645
+
+ offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map(p1 -> 1L)), p1, 1L)).futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map("p2" -> 1L)), "p2", 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map(p2 -> 1L)), p2, 1L))
.futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map("p3" -> 1L)), "p3", 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map(p3 -> 1L)), p3, 1L))
.futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(3)), Map("p4" -> 1L)), "p4", 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(3)), Map(p4 -> 1L)), p4, 1L))
.futureValue
offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 0
offsetStore.readOffset().futureValue // this will load from database
offsetStore.getState().size shouldBe 4
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(2)), Map("p5" -> 1L)), "p5",
- 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(2)), Map(p5 -> 1L)), p5, 1L))
.futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(1)), Map("p6" -> 1L)), "p6",
- 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(1)), Map(p6 -> 1L)), p6, 1L))
.futureValue
// nothing deleted yet
offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 0
@@ -754,47 +910,87 @@ class R2dbcTimestampOffsetStoreSpec
offsetStore.getState().size shouldBe 6
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(1)), Map("p7" -> 1L)), "p7",
- 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(1)), Map(p7 -> 1L)), p7, 1L))
.futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(2)), Map("p8" -> 1L)), "p8",
- 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(3)), Map(p8 -> 1L)), p8, 1L))
.futureValue
offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 2
offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().byPid.keySet shouldBe Set("p3", "p4", "p5", "p6", "p7", "p8")
+ // p3 is kept for slice 905
+ offsetStore.getState().byPid.keySet shouldBe Set(p3, p4, p5, p6, p7, p8)
+ }
+
+ "delete many old records" in {
+ // windowSeconds and totalMillis can be increase for longer/more testing
+ val windowSeconds = 3
+ val totalMillis = 5 * 1000
+
+ val projectionId = genRandomProjectionId()
+ val deleteSettings = settings
+ .withTimeWindow(JDuration.ofSeconds(windowSeconds))
+ .withKeepNumberOfEntries(2000)
+ .withDeleteInterval(JDuration.ofHours(1)) // don't run the scheduled deletes
+ val offsetStore = createOffsetStore(projectionId, deleteSettings)
+
+ val startTime = TestClock.nowMicros().instant()
+ log.debug("Start time [{}]", startTime)
+
+ val storedSlices = (1 to totalMillis / 10).flatMap { m =>
+ val offsets = (1 to 10).map { n =>
+ val pid = s"p$m-$n"
+ OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofMillis(m * 10 + n)), Map(pid -> 1L)), pid, 1L)
+ }
+ offsetStore.saveOffsets(offsets).futureValue
+ if (m % (totalMillis / 100) == 0) {
+ val t0 = System.nanoTime()
+ val deleted = offsetStore.deleteOldTimestampOffsets().futureValue
+ println(s"# ${m * 10} deleted $deleted, took ${(System.nanoTime() - t0) / 1000 / 1000} ms")
+ }
+ offsets.map(o => persistenceExt.sliceForPersistenceId(o.pidSeqNr.get._1)).toSet
+ }.toSet
+
+ offsetStore.readOffset().futureValue // this will load from database
+ val readSlices = offsetStore.getState().byPid.keySet.map(pid => persistenceExt.sliceForPersistenceId(pid))
+ readSlices shouldBe storedSlices
}
"periodically delete old records" in {
val projectionId = genRandomProjectionId()
- val deleteSettings = settings.withTimeWindow(JDuration.ofSeconds(100)).withDeleteInterval(JDuration.ofMillis(500))
+ val deleteSettings =
+ settings
+ .withTimeWindow(JDuration.ofSeconds(100))
+ .withDeleteInterval(JDuration.ofMillis(500))
import deleteSettings._
val offsetStore = createOffsetStore(projectionId, deleteSettings)
val startTime = TestClock.nowMicros().instant()
log.debug("Start time [{}]", startTime)
- offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map("p1" -> 1L)), "p1", 1L)).futureValue
+ // these pids have the same slice 645, otherwise it will keep one for each slice
+ val p1 = "p500"
+ val p2 = "p621"
+ val p3 = "p742"
+
+ offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map(p1 -> 1L)), p1, 1L)).futureValue
offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(1)), Map("p2" -> 1L)), "p2",
- 1L))
+ .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(1)), Map(p2 -> 1L)), p2, 1L))
.futureValue
eventually {
offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().byPid.keySet shouldBe Set("p2")
+ offsetStore.getState().byPid.keySet shouldBe Set(p2)
}
offsetStore
.saveOffset(
OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.multipliedBy(2).plusSeconds(2)), Map("p3" -> 1L)),
- "p3",
+ TimestampOffset(startTime.plus(timeWindow.multipliedBy(2).plusSeconds(2)), Map(p3 -> 1L)),
+ p3,
1L))
.futureValue
eventually {
offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().byPid.keySet shouldBe Set("p3")
+ offsetStore.getState().byPid.keySet shouldBe Set(p3)
}
}
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestClock.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestClock.scala
index 209a3bc..b6922c5 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestClock.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestClock.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -21,7 +21,8 @@ import java.time.ZoneOffset
import java.time.temporal.ChronoUnit
import java.time.temporal.TemporalUnit
-import org.apache.pekko.annotation.InternalApi
+import org.apache.pekko
+import pekko.annotation.InternalApi
object TestClock {
def nowMillis(): TestClock = new TestClock(ChronoUnit.MILLIS)
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestData.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestData.scala
index bc2e039..5203c7f 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestData.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestData.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
@@ -16,7 +16,8 @@ package org.apache.pekko.projection.r2dbc
import java.util.UUID
import java.util.concurrent.atomic.AtomicLong
-import org.apache.pekko.projection.ProjectionId
+import org.apache.pekko
+import pekko.projection.ProjectionId
object TestData {
private val start = 0L // could be something more unique, like currentTimeMillis
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestDbLifecycle.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestDbLifecycle.scala
index 552c5c6..536c0fc 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestDbLifecycle.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestDbLifecycle.scala
@@ -8,13 +8,14 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
import scala.concurrent.Await
import scala.concurrent.duration._
+
import org.apache.pekko
import pekko.actor.typed.ActorSystem
import pekko.persistence.Persistence
@@ -37,11 +38,7 @@ trait TestDbLifecycle extends BeforeAndAfterAll { this: Suite =>
lazy val r2dbcExecutor: R2dbcExecutor = {
new R2dbcExecutor(
- // making sure that test harness does not initialize connection factory for the plugin that is being tested
- ConnectionFactoryProvider(typedSystem)
- .connectionFactoryFor("test.connection-factory",
- typedSystem.settings.config.getConfig(r2dbcProjectionSettings.useConnectionFactory).atPath(
- "test.connection-factory")),
+ ConnectionFactoryProvider(typedSystem).connectionFactoryFor(r2dbcProjectionSettings.useConnectionFactory),
LoggerFactory.getLogger(getClass),
r2dbcProjectionSettings.logDbCallsExceeding)(typedSystem.executionContext, typedSystem)
}
@@ -49,9 +46,9 @@ trait TestDbLifecycle extends BeforeAndAfterAll { this: Suite =>
lazy val persistenceExt: Persistence = Persistence(typedSystem)
override protected def beforeAll(): Unit = {
- lazy val journalSettings: JournalSettings =
+ val journalSettings: JournalSettings =
new JournalSettings(typedSystem.settings.config.getConfig("pekko.persistence.r2dbc.journal"))
- lazy val stateSettings: StateSettings =
+ val stateSettings: StateSettings =
new StateSettings(typedSystem.settings.config.getConfig("pekko.persistence.r2dbc.state"))
Await.result(
r2dbcExecutor.updateOne("beforeAll delete")(
diff --git a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestSourceProviderWithInput.scala b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestSourceProviderWithInput.scala
index a6e6b7e..eb22d8e 100644
--- a/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestSourceProviderWithInput.scala
+++ b/r2dbc-int-test/src/test/scala/org/apache/pekko/projection/r2dbc/TestSourceProviderWithInput.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc
diff --git a/r2dbc/src/main/mima-filters/2.0.x.backwards.excludes/synch-akka-projection-r2dbc-1.4.0.excludes b/r2dbc/src/main/mima-filters/2.0.x.backwards.excludes/synch-akka-projection-r2dbc-1.4.0.excludes
new file mode 100644
index 0000000..247569d
--- /dev/null
+++ b/r2dbc/src/main/mima-filters/2.0.x.backwards.excludes/synch-akka-projection-r2dbc-1.4.0.excludes
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Caused by synching changes from akka-projection-r2dbc 1.4.0
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.apply")
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.apply")
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#Record.copy")
+ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#Record.copy$default$*")
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#Record.this")
+ProblemFilters.exclude[MissingTypesProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore$Record$")
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#Record.apply")
+ProblemFilters.exclude[IncompatibleSignatureProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#Record.unapply")
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#State.copy")
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#State.this")
+ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#State.apply")
+ProblemFilters.exclude[IncompatibleSignatureProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#State.unapply")
diff --git a/r2dbc/src/main/resources/reference.conf b/r2dbc/src/main/resources/reference.conf
index 67ffe1d..6f8ff0b 100644
--- a/r2dbc/src/main/resources/reference.conf
+++ b/r2dbc/src/main/resources/reference.conf
@@ -48,5 +48,9 @@ pekko.projection.r2dbc {
# Set to "off" to disable this logging.
# Set to 0 to log all calls.
log-db-calls-exceeding = 300 ms
+
+ # Warn if a filtered event is emitted in an atLeastOnceFlow projection,
+ # since such events cannot be skipped.
+ warn-about-filtered-events-in-flow = true
}
//#projection-config
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSettings.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSettings.scala
index 8428826..954b1cd 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSettings.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSettings.scala
@@ -47,7 +47,8 @@ object R2dbcProjectionSettings {
keepNumberOfEntries = config.getInt("offset-store.keep-number-of-entries"),
evictInterval = config.getDuration("offset-store.evict-interval"),
deleteInterval = config.getDuration("offset-store.delete-interval"),
- logDbCallsExceeding
+ logDbCallsExceeding,
+ warnAboutFilteredEventsInFlow = config.getBoolean("warn-about-filtered-events-in-flow")
)
}
@@ -85,7 +86,8 @@ object R2dbcProjectionSettings {
keepNumberOfEntries: Int,
evictInterval: JDuration,
deleteInterval: JDuration,
- logDbCallsExceeding: FiniteDuration
+ logDbCallsExceeding: FiniteDuration,
+ warnAboutFilteredEventsInFlow: Boolean = true
): R2dbcProjectionSettings = new R2dbcProjectionSettings(
Dialect.Postgres,
schema,
@@ -97,7 +99,8 @@ object R2dbcProjectionSettings {
keepNumberOfEntries,
evictInterval,
deleteInterval,
- logDbCallsExceeding
+ logDbCallsExceeding,
+ warnAboutFilteredEventsInFlow
)
}
@@ -112,12 +115,13 @@ final class R2dbcProjectionSettings private (
val keepNumberOfEntries: Int,
val evictInterval: JDuration,
val deleteInterval: JDuration,
- val logDbCallsExceeding: FiniteDuration
+ val logDbCallsExceeding: FiniteDuration,
+ val warnAboutFilteredEventsInFlow: Boolean
) extends Serializable {
override def toString: String =
s"R2dbcProjectionSettings($dialect, $schema, $offsetTable, $timestampOffsetTable, $managementTable, " +
- s"$useConnectionFactory, $timeWindow, $keepNumberOfEntries, $evictInterval, $deleteInterval, $logDbCallsExceeding)"
+ s"$useConnectionFactory, $timeWindow, $keepNumberOfEntries, $evictInterval, $deleteInterval, $logDbCallsExceeding, $warnAboutFilteredEventsInFlow)"
override def equals(other: Any): Boolean =
other match {
@@ -127,7 +131,8 @@ final class R2dbcProjectionSettings private (
managementTable == that.managementTable && useConnectionFactory == that.useConnectionFactory &&
timeWindow == that.timeWindow && keepNumberOfEntries == that.keepNumberOfEntries &&
evictInterval == that.evictInterval && deleteInterval == that.deleteInterval &&
- logDbCallsExceeding == that.logDbCallsExceeding
+ logDbCallsExceeding == that.logDbCallsExceeding &&
+ warnAboutFilteredEventsInFlow == that.warnAboutFilteredEventsInFlow
case _ => false
}
@@ -143,7 +148,8 @@ final class R2dbcProjectionSettings private (
keepNumberOfEntries,
evictInterval,
deleteInterval,
- logDbCallsExceeding
+ logDbCallsExceeding,
+ warnAboutFilteredEventsInFlow
)
val h = values.foldLeft(MurmurHash3.productSeed) { case (h, value) =>
MurmurHash3.mix(h, value.##)
@@ -162,7 +168,8 @@ final class R2dbcProjectionSettings private (
keepNumberOfEntries: Int = keepNumberOfEntries,
evictInterval: JDuration = evictInterval,
deleteInterval: JDuration = deleteInterval,
- logDbCallsExceeding: FiniteDuration = logDbCallsExceeding
+ logDbCallsExceeding: FiniteDuration = logDbCallsExceeding,
+ warnAboutFilteredEventsInFlow: Boolean = warnAboutFilteredEventsInFlow
): R2dbcProjectionSettings =
new R2dbcProjectionSettings(
dialect,
@@ -175,7 +182,8 @@ final class R2dbcProjectionSettings private (
keepNumberOfEntries,
evictInterval,
deleteInterval,
- logDbCallsExceeding
+ logDbCallsExceeding,
+ warnAboutFilteredEventsInFlow
)
def withDialect(dialect: Dialect): R2dbcProjectionSettings =
@@ -211,6 +219,9 @@ final class R2dbcProjectionSettings private (
def withLogDbCallsExceeding(logDbCallsExceeding: FiniteDuration): R2dbcProjectionSettings =
copy(logDbCallsExceeding = logDbCallsExceeding)
+ def withWarnAboutFilteredEventsInFlow(warnAboutFilteredEventsInFlow: Boolean): R2dbcProjectionSettings =
+ copy(warnAboutFilteredEventsInFlow = warnAboutFilteredEventsInFlow)
+
val offsetTableWithSchema: String = schema.map(_ + ".").getOrElse("") + offsetTable
val timestampOffsetTableWithSchema: String = schema.map(_ + ".").getOrElse("") + timestampOffsetTable
val managementTableWithSchema: String = schema.map(_ + ".").getOrElse("") + managementTable
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/BySliceSourceProviderAdapter.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/BySliceSourceProviderAdapter.scala
index c1741c3..96854fa 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/BySliceSourceProviderAdapter.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/BySliceSourceProviderAdapter.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc.internal
@@ -17,8 +17,7 @@ import java.time.Instant
import java.util.Optional
import java.util.concurrent.CompletionStage
import java.util.function.Supplier
-
-import scala.concurrent.{ ExecutionContext, Future }
+import scala.concurrent.Future
import scala.jdk.FutureConverters._
import scala.jdk.OptionConverters._
@@ -28,9 +27,9 @@ import pekko.annotation.InternalApi
import pekko.persistence.query.typed.EventEnvelope
import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
import pekko.persistence.query.typed.scaladsl.LoadEventQuery
+import pekko.projection.BySlicesSourceProvider
import pekko.projection.javadsl
import pekko.projection.scaladsl
-import pekko.projection.BySlicesSourceProvider
import pekko.stream.scaladsl.Source
/**
@@ -46,7 +45,7 @@ import pekko.stream.scaladsl.Source
def source(offset: () => Future[Option[Offset]]): Future[Source[Envelope, NotUsed]] = {
// the parasitic context is used to convert the Optional to Option and a java streams Source to a scala Source,
// it _should_ not be used for the blocking operation of getting offsets themselves
- val ec = ExecutionContext.parasitic
+ val ec = scala.concurrent.ExecutionContext.parasitic
val offsetAdapter = new Supplier[CompletionStage[Optional[Offset]]] {
override def get(): CompletionStage[Optional[Offset]] = offset().map(_.toJava)(ec).asJava
}
@@ -66,7 +65,8 @@ import pekko.stream.scaladsl.Source
override def timestampOf(persistenceId: String, sequenceNr: Long): Future[Option[Instant]] =
delegate match {
case timestampQuery: pekko.persistence.query.typed.javadsl.EventTimestampQuery =>
- timestampQuery.timestampOf(persistenceId, sequenceNr).asScala.map(_.toScala)(ExecutionContext.parasitic)
+ timestampQuery.timestampOf(persistenceId, sequenceNr).asScala.map(_.toScala)(
+ scala.concurrent.ExecutionContext.parasitic)
case _ =>
Future.failed(
new IllegalArgumentException(
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/OffsetPidSeqNr.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/OffsetPidSeqNr.scala
index 68edcca..31b01d7 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/OffsetPidSeqNr.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/OffsetPidSeqNr.scala
@@ -13,7 +13,8 @@
package org.apache.pekko.projection.r2dbc.internal
-import org.apache.pekko.annotation.InternalApi
+import org.apache.pekko
+import pekko.annotation.InternalApi
/**
* INTERNAL API
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcHandlerAdapter.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcHandlerAdapter.scala
index c64a1ed..1028e2d 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcHandlerAdapter.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcHandlerAdapter.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc.internal
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcOffsetStore.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcOffsetStore.scala
index dc8ec16..7f7c2e5 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcOffsetStore.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcOffsetStore.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021-2022 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc.internal
@@ -22,7 +22,8 @@ import java.util.concurrent.atomic.AtomicReference
import scala.annotation.tailrec
import scala.collection.immutable
-import scala.concurrent.{ ExecutionContext, Future }
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
import org.apache.pekko
import pekko.Done
@@ -32,14 +33,13 @@ import pekko.annotation.InternalApi
import pekko.persistence.Persistence
import pekko.persistence.query.DeletedDurableState
import pekko.persistence.query.DurableStateChange
-import pekko.persistence.query.Offset
import pekko.persistence.query.TimestampOffset
import pekko.persistence.query.UpdatedDurableState
import pekko.persistence.query.typed.EventEnvelope
import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
-import pekko.persistence.r2dbc.Dialect
import pekko.persistence.r2dbc.internal.EnvelopeOrigin
import pekko.persistence.r2dbc.internal.R2dbcExecutor
+import pekko.persistence.r2dbc.Dialect
import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
import pekko.persistence.typed.PersistenceId
import pekko.projection.BySlicesSourceProvider
@@ -55,11 +55,15 @@ import io.r2dbc.spi.Connection
import io.r2dbc.spi.Statement
import org.slf4j.LoggerFactory
-object R2dbcOffsetStore {
+/**
+ * INTERNAL API
+ */
+@InternalApi
+private[projection] object R2dbcOffsetStore {
type SeqNr = Long
type Pid = String
- final case class Record(pid: Pid, seqNr: SeqNr, timestamp: Instant)
+ final case class Record(slice: Int, pid: Pid, seqNr: SeqNr, timestamp: Instant)
final case class RecordWithOffset(
record: Record,
offset: TimestampOffset,
@@ -68,7 +72,7 @@ object R2dbcOffsetStore {
fromPubSub: Boolean)
object State {
- val empty: State = State(Map.empty, Vector.empty, Instant.EPOCH)
+ val empty: State = State(Map.empty, Vector.empty, Instant.EPOCH, 0)
def apply(records: immutable.IndexedSeq[Record]): State = {
if (records.isEmpty) empty
@@ -76,7 +80,12 @@ object R2dbcOffsetStore {
}
}
- final case class State(byPid: Map[Pid, Record], latest: immutable.IndexedSeq[Record], oldestTimestamp: Instant) {
+ final case class State(
+ byPid: Map[Pid, Record],
+ latest: immutable.IndexedSeq[Record],
+ oldestTimestamp: Instant,
+ sizeAfterEvict: Int) {
+
def size: Int = byPid.size
def latestTimestamp: Instant =
@@ -91,44 +100,45 @@ object R2dbcOffsetStore {
}
def add(records: immutable.IndexedSeq[Record]): State = {
- records.foldLeft(this) { case (acc, r) =>
- val newByPid =
- acc.byPid.get(r.pid) match {
- case Some(existingRecord) =>
- if (r.seqNr > existingRecord.seqNr)
- acc.byPid.updated(r.pid, r)
- else
- acc.byPid // older or same seqNr
- case None =>
- acc.byPid.updated(r.pid, r)
- }
-
- val latestTimestamp = acc.latestTimestamp
- val newLatest =
- if (r.timestamp.isAfter(latestTimestamp)) {
- Vector(r)
- } else if (r.timestamp == latestTimestamp) {
- acc.latest.find(_.pid == r.pid) match {
- case None => acc.latest :+ r
+ records.foldLeft(this) {
+ case (acc, r) =>
+ val newByPid =
+ acc.byPid.get(r.pid) match {
case Some(existingRecord) =>
- // keep highest seqNr
- if (r.seqNr >= existingRecord.seqNr)
- acc.latest.filterNot(_.pid == r.pid) :+ r
+ if (r.seqNr > existingRecord.seqNr)
+ acc.byPid.updated(r.pid, r)
else
- acc.latest
+ acc.byPid // older or same seqNr
+ case None =>
+ acc.byPid.updated(r.pid, r)
}
- } else {
- acc.latest // older than existing latest, keep existing latest
- }
- val newOldestTimestamp =
- if (acc.oldestTimestamp == Instant.EPOCH)
- r.timestamp // first record
- else if (r.timestamp.isBefore(acc.oldestTimestamp))
- r.timestamp
- else
- acc.oldestTimestamp // this is the normal case
-
- acc.copy(byPid = newByPid, latest = newLatest, oldestTimestamp = newOldestTimestamp)
+
+ val latestTimestamp = acc.latestTimestamp
+ val newLatest =
+ if (r.timestamp.isAfter(latestTimestamp)) {
+ Vector(r)
+ } else if (r.timestamp == latestTimestamp) {
+ acc.latest.find(_.pid == r.pid) match {
+ case None => acc.latest :+ r
+ case Some(existingRecord) =>
+ // keep highest seqNr
+ if (r.seqNr >= existingRecord.seqNr)
+ acc.latest.filterNot(_.pid == r.pid) :+ r
+ else
+ acc.latest
+ }
+ } else {
+ acc.latest // older than existing latest, keep existing latest
+ }
+ val newOldestTimestamp =
+ if (acc.oldestTimestamp == Instant.EPOCH)
+ r.timestamp // first record
+ else if (r.timestamp.isBefore(acc.oldestTimestamp))
+ r.timestamp
+ else
+ acc.oldestTimestamp // this is the normal case
+
+ acc.copy(byPid = newByPid, latest = newLatest, oldestTimestamp = newOldestTimestamp)
}
}
@@ -142,21 +152,31 @@ object R2dbcOffsetStore {
def window: JDuration =
JDuration.between(oldestTimestamp, latestTimestamp)
+ private lazy val sortedByTimestamp: Vector[Record] = byPid.valuesIterator.toVector.sortBy(_.timestamp)
+
+ lazy val latestBySlice: Vector[Record] = {
+ val builder = scala.collection.mutable.Map[Int, Record]()
+ sortedByTimestamp.reverseIterator.foreach { record =>
+ if (!builder.contains(record.slice))
+ builder.update(record.slice, record)
+ }
+ builder.values.toVector
+ }
+
def evict(until: Instant, keepNumberOfEntries: Int): State = {
if (oldestTimestamp.isBefore(until) && size > keepNumberOfEntries) {
- val sorted = byPid.valuesIterator.toVector.sortBy(_.timestamp)
- State(
- sorted
+ val newState = State(
+ sortedByTimestamp
.take(size - keepNumberOfEntries)
- .filterNot(_.timestamp.isBefore(until)) ++ sorted.takeRight(keepNumberOfEntries))
+ .filterNot(_.timestamp.isBefore(until)) ++
+ sortedByTimestamp
+ .takeRight(keepNumberOfEntries) ++ latestBySlice)
+ newState.copy(sizeAfterEvict = newState.size)
} else
this
}
- }
- val FutureDone: Future[Done] = Future.successful(Done)
- val FutureTrue: Future[Boolean] = Future.successful(true)
- val FutureFalse: Future[Boolean] = Future.successful(false)
+ }
def fromConfig(
projectionId: ProjectionId,
@@ -165,14 +185,31 @@ object R2dbcOffsetStore {
settings: R2dbcProjectionSettings,
r2dbcExecutor: R2dbcExecutor,
clock: Clock = Clock.systemUTC()
- ): R2dbcOffsetStore = {
+ ): R2dbcOffsetStore =
settings.dialect match {
case Dialect.Postgres | Dialect.Yugabyte =>
new R2dbcOffsetStore(projectionId, sourceProvider, system, settings, r2dbcExecutor, clock)
case Dialect.MySQL =>
new MySQLR2dbcOffsetStore(projectionId, sourceProvider, system, settings, r2dbcExecutor, clock)
}
+
+ final class RejectedEnvelope(message: String) extends IllegalStateException(message)
+
+ sealed trait Validation
+
+ object Validation {
+ case object Accepted extends Validation
+ case object Duplicate extends Validation
+ case object RejectedSeqNr extends Validation
+ case object RejectedBacktrackingSeqNr extends Validation
+
+ val FutureAccepted: Future[Validation] = Future.successful(Accepted)
+ val FutureDuplicate: Future[Validation] = Future.successful(Duplicate)
+ val FutureRejectedSeqNr: Future[Validation] = Future.successful(RejectedSeqNr)
+ val FutureRejectedBacktrackingSeqNr: Future[Validation] = Future.successful(RejectedBacktrackingSeqNr)
}
+
+ val FutureDone: Future[Done] = Future.successful(Done)
}
/**
@@ -195,23 +232,22 @@ private[projection] class R2dbcOffsetStore(
// FIXME include projectionId in all log messages
private val logger = LoggerFactory.getLogger(this.getClass)
+ private val persistenceExt = Persistence(system)
+
private val evictWindow = settings.timeWindow.plus(settings.evictInterval)
- private val evictKeepNumberOfEntriesThreshold = (settings.keepNumberOfEntries * 1.1).toInt
private val offsetSerialization = new OffsetSerialization(system)
import offsetSerialization.fromStorageRepresentation
import offsetSerialization.toStorageRepresentation
private val timestampOffsetTable = settings.timestampOffsetTableWithSchema
- protected val offsetTable = settings.offsetTableWithSchema
- protected val managementTable = settings.managementTableWithSchema
+ protected val offsetTable: String = settings.offsetTableWithSchema
+ protected val managementTable: String = settings.managementTableWithSchema
private[projection] implicit val executionContext: ExecutionContext = system.executionContext
- private val persistenceExt = Persistence(system)
-
private val selectTimestampOffsetSql: String = sql"""
- SELECT persistence_id, seq_nr, timestamp_offset
+ SELECT slice, persistence_id, seq_nr, timestamp_offset
FROM $timestampOffsetTable WHERE slice BETWEEN ? AND ? AND projection_name = ?"""
private val insertTimestampOffsetSql: String = sql"""
@@ -220,8 +256,9 @@ private[projection] class R2dbcOffsetStore(
VALUES (?,?,?,?,?,?, $timestampSql)"""
// delete less than a timestamp
- private val deleteOldTimestampOffsetSql: String =
- sql"DELETE FROM $timestampOffsetTable WHERE slice BETWEEN ? AND ? AND projection_name = ? AND timestamp_offset < ?"
+ private val deleteOldTimestampOffsetSql: String = sql"""
+ DELETE FROM $timestampOffsetTable WHERE slice BETWEEN ? AND ? AND projection_name = ? AND timestamp_offset < ?
+ AND NOT (persistence_id || '-' || seq_nr) = ANY (?)"""
// delete greater than or equal a timestamp
private val deleteNewTimestampOffsetSql: String =
@@ -285,7 +322,7 @@ private[projection] class R2dbcOffsetStore(
case Some(provider) => provider
case None =>
throw new IllegalArgumentException(
- "Expected BySlicesSourceProvider to be defined when TimestampOffset is used.")
+ s"Expected BySlicesSourceProvider to be defined when TimestampOffset is used.")
}
private def timestampOf(persistenceId: String, sequenceNr: Long): Future[Option[Instant]] = {
@@ -298,7 +335,7 @@ private[projection] class R2dbcOffsetStore(
timestampQuery.timestampOf(persistenceId, sequenceNr).asScala.map(_.toScala)
case _ =>
throw new IllegalArgumentException(
- "Expected BySlicesSourceProvider to implement EventTimestampQuery when TimestampOffset is used.")
+ s"Expected BySlicesSourceProvider to implement EventTimestampQuery when TimestampOffset is used.")
}
}
@@ -308,20 +345,20 @@ private[projection] class R2dbcOffsetStore(
def getInflight(): Map[Pid, SeqNr] =
inflight.get()
- def getOffset[Off](): Future[Option[Off]] = {
+ def getOffset[Offset](): Future[Option[Offset]] = {
getState().latestOffset match {
- case Some(t) => Future.successful(Some(t.asInstanceOf[Off]))
+ case Some(t) => Future.successful(Some(t.asInstanceOf[Offset]))
case None => readOffset()
}
}
- def readOffset[Off](): Future[Option[Off]] = {
- // look for TimestampOffset first since that is used by pekko-persistence-r2dbc,
+ def readOffset[Offset](): Future[Option[Offset]] = {
+ // look for TimestampOffset first since that is used by akka-persistence-r2dbc,
// and then fall back to the other more primitive offset types
sourceProvider match {
- case Some(provider) =>
+ case Some(_) =>
readTimestampOffset().flatMap {
- case Some(t) => Future.successful(Some(t.asInstanceOf[Off]))
+ case Some(t) => Future.successful(Some(t.asInstanceOf[Offset]))
case None => readPrimitiveOffset()
}
case None =>
@@ -350,16 +387,17 @@ private[projection] class R2dbcOffsetStore(
.bind(2, projectionId.name)
},
row => {
+ val slice = row.get("slice", classOf[java.lang.Integer])
val pid = row.get("persistence_id", classOf[String])
val seqNr = row.get("seq_nr", classOf[java.lang.Long])
val timestamp = row.get("timestamp_offset", classOf[Instant])
- Record(pid, seqNr, timestamp)
+ Record(slice, pid, seqNr, timestamp)
})
recordsFut.map { records =>
val newState = State(records)
- logger.debug(
+ logger.debugN(
"readTimestampOffset state with [{}] persistenceIds, oldest [{}], latest [{}]",
- newState.byPid.size: java.lang.Integer,
+ newState.byPid.size,
newState.oldestTimestamp,
newState.latestTimestamp)
if (!state.compareAndSet(oldState, newState))
@@ -373,7 +411,7 @@ private[projection] class R2dbcOffsetStore(
}
}
- private def readPrimitiveOffset[Off](): Future[Option[Off]] = {
+ private def readPrimitiveOffset[Offset](): Future[Option[Offset]] = {
if (settings.isOffsetTableDefined) {
val singleOffsets = r2dbcExecutor.select("read offset")(
conn => {
@@ -385,7 +423,7 @@ private[projection] class R2dbcOffsetStore(
row => {
val offsetStr = row.get("current_offset", classOf[String])
val manifest = row.get("manifest", classOf[String])
- val mergeable = row.get[java.lang.Boolean]("mergeable", classOf[java.lang.Boolean])
+ val mergeable = row.get("mergeable", classOf[java.lang.Boolean])
val key = row.get("projection_key", classOf[String])
val adaptedProjectionId = ProjectionId(projectionId.name, key)
@@ -398,9 +436,9 @@ private[projection] class R2dbcOffsetStore(
else if (offsets.forall(_.mergeable)) {
Some(
fromStorageRepresentation[MergeableOffset[_], Offset](MultipleOffsets(offsets.toList))
- .asInstanceOf[Off])
+ .asInstanceOf[Offset])
} else {
- offsets.find(_.id == projectionId).map(fromStorageRepresentation[Off, Off])
+ offsets.find(_.id == projectionId).map(fromStorageRepresentation[Offset, Offset])
}
logger.trace2("found offset [{}] for [{}]", result, projectionId)
@@ -429,7 +467,8 @@ private[projection] class R2dbcOffsetStore(
def saveOffsetInTx(conn: Connection, offset: OffsetPidSeqNr): Future[Done] = {
offset match {
case OffsetPidSeqNr(t: TimestampOffset, Some((pid, seqNr))) =>
- val record = Record(pid, seqNr, t.timestamp)
+ val slice = persistenceExt.sliceForPersistenceId(pid)
+ val record = Record(slice, pid, seqNr, t.timestamp)
saveTimestampOffsetInTx(conn, Vector(record))
case OffsetPidSeqNr(_: TimestampOffset, None) =>
throw new IllegalArgumentException("Required EventEnvelope or DurableStateChange for TimestampOffset.")
@@ -452,12 +491,13 @@ private[projection] class R2dbcOffsetStore(
else if (offsets.head.offset.isInstanceOf[TimestampOffset]) {
val records = offsets.map {
case OffsetPidSeqNr(t: TimestampOffset, Some((pid, seqNr))) =>
- Record(pid, seqNr, t.timestamp)
+ val slice = persistenceExt.sliceForPersistenceId(pid)
+ Record(slice, pid, seqNr, t.timestamp)
case OffsetPidSeqNr(_: TimestampOffset, None) =>
throw new IllegalArgumentException("Required EventEnvelope or DurableStateChange for TimestampOffset.")
case _ =>
throw new IllegalArgumentException(
- "Mix of TimestampOffset and other offset type in same transaction is not supported")
+ "Mix of TimestampOffset and other offset type in same transaction isnot supported")
}
saveTimestampOffsetInTx(conn, records)
} else {
@@ -465,7 +505,7 @@ private[projection] class R2dbcOffsetStore(
}
}
- private def saveTimestampOffsetInTx[Off](conn: Connection, records: immutable.IndexedSeq[Record]): Future[Done] = {
+ private def saveTimestampOffsetInTx(conn: Connection, records: immutable.IndexedSeq[Record]): Future[Done] = {
idle.set(false)
val oldState = state.get()
val filteredRecords = {
@@ -487,16 +527,21 @@ private[projection] class R2dbcOffsetStore(
} else {
val newState = oldState.add(filteredRecords)
- // accumulate some more than the timeWindow before evicting
+ // accumulate some more than the timeWindow before evicting, and at least 10% increase of size
+ // for testing keepNumberOfEntries = 0 is used
+ val evictThresholdReached =
+ if (settings.keepNumberOfEntries == 0) true else newState.size > (newState.sizeAfterEvict * 1.1).toInt
val evictedNewState =
- if (newState.size > evictKeepNumberOfEntriesThreshold && newState.window.compareTo(evictWindow) > 0) {
+ if (newState.size > settings.keepNumberOfEntries && evictThresholdReached &&
+ newState.window
+ .compareTo(evictWindow) > 0) {
val evictUntil = newState.latestTimestamp.minus(settings.timeWindow)
val s = newState.evict(evictUntil, settings.keepNumberOfEntries)
- logger.debug(
+ logger.debugN(
"Evicted [{}] records until [{}], keeping [{}] records. Latest [{}].",
- newState.size - s.size: java.lang.Integer,
+ newState.size - s.size,
evictUntil,
- s.size: java.lang.Integer,
+ s.size,
newState.latestTimestamp)
s
} else
@@ -535,8 +580,7 @@ private[projection] class R2dbcOffsetStore(
require(records.nonEmpty)
- // FIXME change to trace
- logger.debug("saving timestamp offset [{}], {}", records.last.timestamp: Any, records: Any)
+ logger.trace2("saving timestamp offset [{}], {}", records.last.timestamp, records)
val statement = conn.createStatement(insertTimestampOffsetSql)
@@ -546,10 +590,8 @@ private[projection] class R2dbcOffsetStore(
} else {
// TODO Try Batch without bind parameters for better performance. Risk of sql injection for these parameters is low.
val boundStatement =
- records.zipWithIndex.foldLeft(statement) { case (stmt, (rec, idx)) =>
- if (idx != 0) {
- stmt.add()
- }
+ records.foldLeft(statement) { (stmt, rec) =>
+ stmt.add()
bindRecord(stmt, rec)
}
R2dbcExecutor.updateBatchInTx(boundStatement)
@@ -565,7 +607,6 @@ private[projection] class R2dbcOffsetStore(
case Some(r) => r.seqNr < inflightSeqNr
case None => true
}
- case null => true
}
if (newInflight.size >= 10000) {
throw new IllegalStateException(
@@ -582,7 +623,7 @@ private[projection] class R2dbcOffsetStore(
clearInflight() // CAS retry, concurrent update of inflight
}
- private def savePrimitiveOffsetInTx[Off](conn: Connection, offset: Off): Future[Done] = {
+ private def savePrimitiveOffsetInTx[Offset](conn: Connection, offset: Offset): Future[Done] = {
logger.trace("saving offset [{}]", offset)
if (!settings.isOffsetTableDefined)
@@ -615,98 +656,111 @@ private[projection] class R2dbcOffsetStore(
R2dbcExecutor.updateInTx(statements).map(_ => Done)(ExecutionContext.parasitic)
}
- def isDuplicate(record: Record): Boolean =
- getState().isDuplicate(record)
+ /**
+ * The stored sequence number for a persistenceId, or 0 if unknown persistenceId.
+ */
+ def storedSeqNr(pid: Pid): SeqNr =
+ getState().byPid.get(pid) match {
+ case Some(record) => record.seqNr
+ case None => 0L
+ }
- def filterAccepted[Envelope](envelopes: immutable.Seq[Envelope]): Future[immutable.Seq[Envelope]] = {
+ def validateAll[Envelope](envelopes: immutable.Seq[Envelope]): Future[immutable.Seq[(Envelope, Validation)]] = {
+ import Validation._
envelopes
- .foldLeft(Future.successful((getInflight(), Vector.empty[Envelope]))) { (acc, envelope) =>
- acc.flatMap { case (inflight, filteredEnvelopes) =>
- createRecordWithOffset(envelope) match {
- case Some(recordWithOffset) =>
- isAccepted(recordWithOffset, inflight).map {
- case true =>
- (
- inflight.updated(recordWithOffset.record.pid, recordWithOffset.record.seqNr),
- filteredEnvelopes :+ envelope)
- case false =>
- (inflight, filteredEnvelopes)
- }
- case None =>
- Future.successful((inflight, filteredEnvelopes :+ envelope))
- }
+ .foldLeft(Future.successful((getInflight(), Vector.empty[(Envelope, Validation)]))) { (acc, envelope) =>
+ acc.flatMap {
+ case (inflight, filteredEnvelopes) =>
+ createRecordWithOffset(envelope) match {
+ case Some(recordWithOffset) =>
+ validate(recordWithOffset, inflight).map {
+ case Accepted =>
+ (
+ inflight.updated(recordWithOffset.record.pid, recordWithOffset.record.seqNr),
+ filteredEnvelopes :+ (envelope -> Accepted))
+ case rejected =>
+ (inflight, filteredEnvelopes :+ (envelope -> rejected))
+ }
+ case None =>
+ Future.successful((inflight, filteredEnvelopes :+ (envelope -> Accepted)))
+ }
}
}
- .map { case (_, filteredEnvelopes) =>
- filteredEnvelopes
+ .map {
+ case (_, filteredEnvelopes) =>
+ filteredEnvelopes
}
}
- def isAccepted[Envelope](envelope: Envelope): Future[Boolean] = {
+ /**
+ * Validate if the sequence number of the envelope is the next expected, or if the envelope
+ * is a duplicate that has already been processed, or there is a gap in sequence numbers that
+ * should be rejected.
+ */
+ def validate[Envelope](envelope: Envelope): Future[Validation] = {
createRecordWithOffset(envelope) match {
- case Some(recordWithOffset) => isAccepted(recordWithOffset, getInflight())
- case None => FutureTrue
+ case Some(recordWithOffset) => validate(recordWithOffset, getInflight())
+ case None => Validation.FutureAccepted
}
}
- private def isAccepted[Envelope](
- recordWithOffset: RecordWithOffset,
- currentInflight: Map[Pid, SeqNr]): Future[Boolean] = {
+ private def validate(recordWithOffset: RecordWithOffset, currentInflight: Map[Pid, SeqNr]): Future[Validation] = {
+ import Validation._
val pid = recordWithOffset.record.pid
val seqNr = recordWithOffset.record.seqNr
val currentState = getState()
- val duplicate = isDuplicate(recordWithOffset.record)
+ val duplicate = getState().isDuplicate(recordWithOffset.record)
if (duplicate) {
logger.trace("Filtering out duplicate sequence number [{}] for pid [{}]", seqNr, pid)
- FutureFalse
+ FutureDuplicate
} else if (recordWithOffset.strictSeqNr) {
// strictSeqNr == true is for event sourced
val prevSeqNr = currentInflight.getOrElse(pid, currentState.byPid.get(pid).map(_.seqNr).getOrElse(0L))
def logUnexpected(): Unit = {
if (recordWithOffset.fromPubSub)
- logger.debug(
+ logger.debugN(
"Rejecting pub-sub envelope, unexpected sequence number [{}] for pid [{}], previous sequence number [{}]. Offset: {}",
- seqNr: java.lang.Long,
+ seqNr,
pid,
- prevSeqNr: java.lang.Long,
+ prevSeqNr,
recordWithOffset.offset)
else if (!recordWithOffset.fromBacktracking)
- logger.debug(
+ logger.debugN(
"Rejecting unexpected sequence number [{}] for pid [{}], previous sequence number [{}]. Offset: {}",
- seqNr: java.lang.Long,
+ seqNr,
pid,
- prevSeqNr: java.lang.Long,
+ prevSeqNr,
recordWithOffset.offset)
else
- logger.warn(
+ logger.warnN(
"Rejecting unexpected sequence number [{}] for pid [{}], previous sequence number [{}]. Offset: {}",
- seqNr: java.lang.Long,
+ seqNr,
pid,
- prevSeqNr: java.lang.Long,
+ prevSeqNr,
recordWithOffset.offset)
}
def logUnknown(): Unit = {
if (recordWithOffset.fromPubSub) {
- logger.debug(
+ logger.debugN(
"Rejecting pub-sub envelope, unknown sequence number [{}] for pid [{}] (might be accepted later): {}",
- seqNr: java.lang.Long,
+ seqNr,
pid,
recordWithOffset.offset)
} else if (!recordWithOffset.fromBacktracking) {
// This may happen rather frequently when using `publish-events`, after reconnecting and such.
- logger.debug(
+ logger.debugN(
"Rejecting unknown sequence number [{}] for pid [{}] (might be accepted later): {}",
- seqNr: java.lang.Long,
+ seqNr,
pid,
recordWithOffset.offset)
} else {
- logger.warn(
+ logger.warnN(
"Rejecting unknown sequence number [{}] for pid [{}]. Offset: {}",
- seqNr: java.lang.Long,
+ seqNr,
pid,
recordWithOffset.offset)
}
@@ -716,26 +770,22 @@ private[projection] class R2dbcOffsetStore(
// expecting seqNr to be +1 of previously known
val ok = seqNr == prevSeqNr + 1
if (ok) {
- FutureTrue
+ FutureAccepted
} else if (seqNr <= currentInflight.getOrElse(pid, 0L)) {
// currentInFlight contains those that have been processed or about to be processed in Flow,
// but offset not saved yet => ok to handle as duplicate
- FutureFalse
+ FutureDuplicate
} else if (!recordWithOffset.fromBacktracking) {
logUnexpected()
- FutureFalse
+ FutureRejectedSeqNr
} else {
logUnexpected()
// This will result in projection restart (with normal configuration)
- Future.failed(
- new IllegalStateException(
- s"Rejected envelope from backtracking, persistenceId [$pid], seqNr [$seqNr] " +
- "due to unexpected sequence number. " +
- "Please report this issue at https://github.com/apache/pekko-persistence-r2dbc"))
+ FutureRejectedBacktrackingSeqNr
}
} else if (seqNr == 1) {
// always accept first event if no other event for that pid has been seen
- FutureTrue
+ FutureAccepted
} else {
// Haven't see seen this pid within the time window. Since events can be missed
// when read at the tail we will only accept it if the event with previous seqNr has timestamp
@@ -745,28 +795,25 @@ private[projection] class R2dbcOffsetStore(
case Some(previousTimestamp) =>
val before = currentState.latestTimestamp.minus(settings.timeWindow)
if (previousTimestamp.isBefore(before)) {
- logger.debug(
+ logger.debugN(
"Accepting envelope with pid [{}], seqNr [{}], where previous event timestamp [{}] " +
"is before time window [{}].",
pid,
- seqNr: java.lang.Long,
+ seqNr,
previousTimestamp,
before)
- true
+ Accepted
} else if (!recordWithOffset.fromBacktracking) {
logUnknown()
- false
+ RejectedSeqNr
} else {
logUnknown()
// This will result in projection restart (with normal configuration)
- throw new IllegalStateException(
- s"Rejected envelope from backtracking, persistenceId [$pid], seqNr [$seqNr], " +
- "due to unknown sequence number. " +
- "Please report this issue at https://github.com/apache/pekko-persistence-r2dbc")
+ RejectedBacktrackingSeqNr
}
case None =>
// previous not found, could have been deleted
- true
+ Accepted
}
}
} else {
@@ -775,12 +822,10 @@ private[projection] class R2dbcOffsetStore(
val ok = seqNr > prevSeqNr
if (ok) {
- FutureTrue
+ FutureAccepted
} else {
- logger.trace("Filtering out earlier revision [{}] for pid [{}], previous revision [{}]", seqNr: java.lang.Long,
- pid,
- prevSeqNr: java.lang.Long)
- FutureFalse
+ logger.traceN("Filtering out earlier revision [{}] for pid [{}], previous revision [{}]", seqNr, pid, prevSeqNr)
+ FutureDuplicate
}
}
}
@@ -798,8 +843,9 @@ private[projection] class R2dbcOffsetStore(
@tailrec final def addInflights[Envelope](envelopes: immutable.Seq[Envelope]): Unit = {
val currentInflight = getInflight()
- val entries = envelopes.iterator.map(createRecordWithOffset).collect { case Some(r) =>
- r.record.pid -> r.record.seqNr
+ val entries = envelopes.iterator.map(createRecordWithOffset).collect {
+ case Some(r) =>
+ r.record.pid -> r.record.seqNr
}
val newInflight = currentInflight ++ entries
if (!inflight.compareAndSet(currentInflight, newInflight))
@@ -832,6 +878,13 @@ private[projection] class R2dbcOffsetStore(
val until = currentState.latestTimestamp.minus(settings.timeWindow)
val minSlice = timestampOffsetBySlicesSourceProvider.minSlice
val maxSlice = timestampOffsetBySlicesSourceProvider.maxSlice
+ val notInLatestBySlice = currentState.latestBySlice.collect {
+ case record if record.timestamp.isBefore(until) =>
+ // note that deleteOldTimestampOffsetSql already has `AND timestamp_offset < ?`
+ // and that's why timestamp >= until don't have to be included here
+ s"${record.pid}-${record.seqNr}"
+ }.toArray
+
val result = r2dbcExecutor.updateOne("delete old timestamp offset") { conn =>
conn
.createStatement(deleteOldTimestampOffsetSql)
@@ -839,11 +892,9 @@ private[projection] class R2dbcOffsetStore(
.bind(1, maxSlice)
.bind(2, projectionId.name)
.bind(3, until)
+ .bind(4, notInLatestBySlice)
}
- // FIXME would it be good to keep at least one record per slice that can be used as the
- // starting point for the slice if the slice ranges are changed?
-
result.failed.foreach { exc =>
idle.set(false) // try again next tick
logger.warn(
@@ -854,9 +905,9 @@ private[projection] class R2dbcOffsetStore(
}
if (logger.isDebugEnabled)
result.foreach { rows =>
- logger.debug(
+ logger.debugN(
"Deleted [{}] timestamp offset rows until [{}] for projection [{}].",
- rows: java.lang.Long,
+ rows,
until,
projectionId.id)
}
@@ -870,18 +921,24 @@ private[projection] class R2dbcOffsetStore(
* Resetting an offset. Deletes newer offsets. Used from ProjectionManagement. Doesn't update in-memory state because
* the projection is supposed to be stopped/started for this operation.
*/
- def managementSetOffset[Off](offset: Off): Future[Done] = {
+ def managementSetOffset[Offset](offset: Offset): Future[Done] = {
offset match {
case t: TimestampOffset =>
r2dbcExecutor
.withConnection("set offset") { conn =>
deleteNewTimestampOffsetsInTx(conn, t.timestamp).flatMap { _ =>
val records =
- if (t.seen.isEmpty)
+ if (t.seen.isEmpty) {
// we need some persistenceId to be able to store the new offset timestamp
- Vector(Record(PersistenceId("mgmt", UUID.randomUUID().toString).id, seqNr = 1L, t.timestamp))
- else
- t.seen.iterator.map { case (pid, seqNr) => Record(pid, seqNr, t.timestamp) }.toVector
+ val pid = PersistenceId("mgmt", UUID.randomUUID().toString).id
+ val slice = persistenceExt.sliceForPersistenceId(pid)
+ Vector(Record(slice, pid, seqNr = 1L, t.timestamp))
+ } else
+ t.seen.iterator.map {
+ case (pid, seqNr) =>
+ val slice = persistenceExt.sliceForPersistenceId(pid)
+ Record(slice, pid, seqNr, t.timestamp)
+ }.toVector
insertTimestampOffsetInTx(conn, records)
}
}
@@ -912,14 +969,11 @@ private[projection] class R2dbcOffsetStore(
.bind(2, projectionId.name)
.bind(3, timestamp))
- // FIXME would it be good to keep at least one record per slice that can be used as the
- // starting point for the slice if the slice ranges are changed?
-
if (logger.isDebugEnabled)
result.foreach { rows =>
- logger.debug(
+ logger.debugN(
"Deleted [{}] timestamp offset rows >= [{}] for projection [{}].",
- rows: java.lang.Long,
+ rows,
timestamp,
projectionId.id)
}
@@ -1002,34 +1056,41 @@ private[projection] class R2dbcOffsetStore(
.bind(2, paused)
.bind(3, Instant.now(clock).toEpochMilli)
}
- .map(_ => Done)(ExecutionContext.parasitic)
+ .flatMap {
+ case i if i == 1 => Future.successful(Done)
+ case _ =>
+ Future.failed(new RuntimeException(s"Failed to update management table for $projectionId"))
+ }
}
private def createRecordWithOffset[Envelope](envelope: Envelope): Option[RecordWithOffset] = {
envelope match {
case eventEnvelope: EventEnvelope[_] if eventEnvelope.offset.isInstanceOf[TimestampOffset] =>
val timestampOffset = eventEnvelope.offset.asInstanceOf[TimestampOffset]
+ val slice = persistenceExt.sliceForPersistenceId(eventEnvelope.persistenceId)
Some(
RecordWithOffset(
- Record(eventEnvelope.persistenceId, eventEnvelope.sequenceNr, timestampOffset.timestamp),
+ Record(slice, eventEnvelope.persistenceId, eventEnvelope.sequenceNr, timestampOffset.timestamp),
timestampOffset,
strictSeqNr = true,
fromBacktracking = EnvelopeOrigin.fromBacktracking(eventEnvelope),
fromPubSub = EnvelopeOrigin.fromPubSub(eventEnvelope)))
case change: UpdatedDurableState[_] if change.offset.isInstanceOf[TimestampOffset] =>
val timestampOffset = change.offset.asInstanceOf[TimestampOffset]
+ val slice = persistenceExt.sliceForPersistenceId(change.persistenceId)
Some(
RecordWithOffset(
- Record(change.persistenceId, change.revision, timestampOffset.timestamp),
+ Record(slice, change.persistenceId, change.revision, timestampOffset.timestamp),
timestampOffset,
strictSeqNr = false,
- fromBacktracking = change.value == null,
+ fromBacktracking = false,
fromPubSub = false))
case change: DeletedDurableState[_] if change.offset.isInstanceOf[TimestampOffset] =>
val timestampOffset = change.offset.asInstanceOf[TimestampOffset]
+ val slice = persistenceExt.sliceForPersistenceId(change.persistenceId)
Some(
RecordWithOffset(
- Record(change.persistenceId, change.revision, timestampOffset.timestamp),
+ Record(slice, change.persistenceId, change.revision, timestampOffset.timestamp),
timestampOffset,
strictSeqNr = false,
fromBacktracking = false,
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcProjectionImpl.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcProjectionImpl.scala
index b63b612..ab8cf1a 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcProjectionImpl.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcProjectionImpl.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021-2022 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc.internal
@@ -25,6 +25,7 @@ import scala.util.control.NonFatal
import org.apache.pekko
import pekko.Done
import pekko.actor.typed.ActorSystem
+import pekko.actor.typed.scaladsl.LoggerOps
import pekko.annotation.InternalApi
import pekko.event.Logging
import pekko.event.LoggingAdapter
@@ -32,7 +33,6 @@ import pekko.persistence.query.DeletedDurableState
import pekko.persistence.query.UpdatedDurableState
import pekko.persistence.query.typed.EventEnvelope
import pekko.persistence.query.typed.scaladsl.LoadEventQuery
-import pekko.persistence.r2dbc.internal.EnvelopeOrigin
import pekko.persistence.r2dbc.internal.R2dbcExecutor
import pekko.persistence.state.scaladsl.DurableStateStore
import pekko.persistence.state.scaladsl.GetObjectResult
@@ -49,6 +49,7 @@ import pekko.projection.StatusObserver
import pekko.projection.internal.ActorHandlerInit
import pekko.projection.internal.AtLeastOnce
import pekko.projection.internal.AtMostOnce
+import pekko.projection.internal.CanTriggerReplay
import pekko.projection.internal.ExactlyOnce
import pekko.projection.internal.GroupedHandlerStrategy
import pekko.projection.internal.HandlerStrategy
@@ -61,6 +62,8 @@ import pekko.projection.internal.ProjectionSettings
import pekko.projection.internal.SettingsImpl
import pekko.projection.javadsl
import pekko.projection.r2dbc.R2dbcProjectionSettings
+import pekko.projection.r2dbc.internal.R2dbcOffsetStore.RejectedEnvelope
+import pekko.projection.r2dbc.internal.R2dbcProjectionImpl.extractOffsetPidSeqNr
import pekko.projection.r2dbc.scaladsl.R2dbcHandler
import pekko.projection.r2dbc.scaladsl.R2dbcSession
import pekko.projection.scaladsl
@@ -78,7 +81,9 @@ import org.slf4j.LoggerFactory
*/
@InternalApi
private[projection] object R2dbcProjectionImpl {
- import EnvelopeOrigin.{ fromBacktracking, isFilteredEvent }
+ import pekko.persistence.r2dbc.internal.EnvelopeOrigin.fromBacktracking
+ import pekko.persistence.r2dbc.internal.EnvelopeOrigin.isFilteredEvent
+
val log: Logger = LoggerFactory.getLogger(classOf[R2dbcProjectionImpl[_, _]])
private val FutureDone: Future[Done] = Future.successful(Done)
@@ -95,7 +100,8 @@ private[projection] object R2dbcProjectionImpl {
private val loadEnvelopeCounter = new AtomicLong
- def loadEnvelope[Envelope](env: Envelope, sourceProvider: SourceProvider[_, Envelope])(implicit
+ def loadEnvelope[Envelope](env: Envelope, sourceProvider: SourceProvider[_, Envelope])(
+ implicit
ec: ExecutionContext): Future[Envelope] = {
env match {
case eventEnvelope: EventEnvelope[_]
@@ -115,12 +121,9 @@ private[projection] object R2dbcProjectionImpl {
}).map { loadedEnv =>
val count = loadEnvelopeCounter.incrementAndGet()
if (count % 1000 == 0)
- log.info("Loaded event lazily, persistenceId [{}], seqNr [{}]. Load count [{}]", pid, seqNr: java.lang.Long,
- count: java.lang.Long)
+ log.infoN("Loaded event lazily, persistenceId [{}], seqNr [{}]. Load count [{}]", pid, seqNr, count)
else
- log.debug("Loaded event lazily, persistenceId [{}], seqNr [{}]. Load count [{}]", pid,
- seqNr: java.lang.Long,
- count: java.lang.Long)
+ log.debugN("Loaded event lazily, persistenceId [{}], seqNr [{}]. Load count [{}]", pid, seqNr, count)
loadedEnv.asInstanceOf[Envelope]
}
@@ -132,21 +135,23 @@ private[projection] object R2dbcProjectionImpl {
case store: pekko.persistence.state.javadsl.DurableStateStore[_] =>
import scala.jdk.FutureConverters._
store.getObject(pid).asScala.map(_.toScala)
+ case unknown =>
+ throw new IllegalArgumentException(s"Unsupported source provider type '${unknown.getClass}'")
}).map {
case GetObjectResult(Some(loadedValue), loadedRevision) =>
val count = loadEnvelopeCounter.incrementAndGet()
if (count % 1000 == 0)
- log.info(
+ log.infoN(
"Loaded durable state lazily, persistenceId [{}], revision [{}]. Load count [{}]",
pid,
- loadedRevision: java.lang.Long,
- count: java.lang.Long)
+ loadedRevision,
+ count)
else
- log.debug(
+ log.debugN(
"Loaded durable state lazily, persistenceId [{}], revision [{}]. Load count [{}]",
pid,
- loadedRevision: java.lang.Long,
- count: java.lang.Long)
+ loadedRevision,
+ count)
new UpdatedDurableState(pid, loadedRevision, loadedValue, upd.offset, upd.timestamp)
.asInstanceOf[Envelope]
case GetObjectResult(None, loadedRevision) =>
@@ -182,28 +187,39 @@ private[projection] object R2dbcProjectionImpl {
() =>
new AdaptedR2dbcHandler(handlerFactory()) {
override def process(envelope: Envelope): Future[Done] = {
- offsetStore.isAccepted(envelope).flatMap {
- case true =>
- if (isFilteredEvent(envelope)) {
- val offset = extractOffsetPidSeqNr(sourceProvider, envelope)
- offsetStore.saveOffset(offset)
- } else {
- loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
- val offset = extractOffsetPidSeqNr(sourceProvider, loadedEnvelope)
- r2dbcExecutor.withConnection("exactly-once handler") { conn =>
- // run users handler
- val session = new R2dbcSession(conn)
- delegate
- .process(session, loadedEnvelope)
- .flatMap { _ =>
- offsetStore.saveOffsetInTx(conn, offset)
- }
+ import R2dbcOffsetStore.Validation._
+ offsetStore
+ .validate(envelope)
+ .flatMap {
+ case Accepted =>
+ if (isFilteredEvent(envelope)) {
+ val offset = extractOffsetPidSeqNr(sourceProvider, envelope)
+ offsetStore.saveOffset(offset)
+ } else {
+ loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
+ val offset = extractOffsetPidSeqNr(sourceProvider, loadedEnvelope)
+ r2dbcExecutor.withConnection("exactly-once handler") { conn =>
+ // run users handler
+ val session = new R2dbcSession(conn)
+ delegate
+ .process(session, loadedEnvelope)
+ .flatMap { _ =>
+ offsetStore.saveOffsetInTx(conn, offset)
+ }
+ }
}
}
- }
- case false =>
- FutureDone
- }
+ case Duplicate =>
+ FutureDone
+ case RejectedSeqNr =>
+ triggerReplayIfPossible(sourceProvider, offsetStore, envelope)
+ FutureDone
+ case RejectedBacktrackingSeqNr =>
+ if (triggerReplayIfPossible(sourceProvider, offsetStore, envelope))
+ FutureDone
+ else
+ throwRejectedEnvelope(sourceProvider, envelope)
+ }
}
}
}
@@ -218,7 +234,23 @@ private[projection] object R2dbcProjectionImpl {
system: ActorSystem[_]): () => Handler[immutable.Seq[Envelope]] = { () =>
new AdaptedR2dbcHandler(handlerFactory()) {
override def process(envelopes: immutable.Seq[Envelope]): Future[Done] = {
- offsetStore.filterAccepted(envelopes).flatMap { acceptedEnvelopes =>
+ import R2dbcOffsetStore.Validation._
+ offsetStore.validateAll(envelopes).flatMap { isAcceptedEnvelopes =>
+ isAcceptedEnvelopes.foreach {
+ case (env, RejectedSeqNr) =>
+ triggerReplayIfPossible(sourceProvider, offsetStore, env)
+ case (env, RejectedBacktrackingSeqNr) =>
+ if (triggerReplayIfPossible(sourceProvider, offsetStore, env))
+ FutureDone
+ else
+ throwRejectedEnvelope(sourceProvider, env)
+ case _ =>
+ }
+
+ val acceptedEnvelopes = isAcceptedEnvelopes.collect {
+ case (env, Accepted) => env
+ }
+
if (acceptedEnvelopes.isEmpty) {
FutureDone
} else {
@@ -252,28 +284,39 @@ private[projection] object R2dbcProjectionImpl {
() =>
new AdaptedR2dbcHandler(handlerFactory()) {
override def process(envelope: Envelope): Future[Done] = {
- offsetStore.isAccepted(envelope).flatMap {
- case true =>
- if (isFilteredEvent(envelope)) {
- offsetStore.addInflight(envelope)
- FutureDone
- } else {
- loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
- r2dbcExecutor
- .withConnection("at-least-once handler") { conn =>
- // run users handler
- val session = new R2dbcSession(conn)
- delegate.process(session, loadedEnvelope)
- }
- .map { _ =>
- offsetStore.addInflight(loadedEnvelope)
- Done
- }
+ import R2dbcOffsetStore.Validation._
+ offsetStore
+ .validate(envelope)
+ .flatMap {
+ case Accepted =>
+ if (isFilteredEvent(envelope)) {
+ offsetStore.addInflight(envelope)
+ FutureDone
+ } else {
+ loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
+ r2dbcExecutor
+ .withConnection("at-least-once handler") { conn =>
+ // run users handler
+ val session = new R2dbcSession(conn)
+ delegate.process(session, loadedEnvelope)
+ }
+ .map { _ =>
+ offsetStore.addInflight(loadedEnvelope)
+ Done
+ }
+ }
}
- }
- case false =>
- FutureDone
- }
+ case Duplicate =>
+ FutureDone
+ case RejectedSeqNr =>
+ triggerReplayIfPossible(sourceProvider, offsetStore, envelope)
+ FutureDone
+ case RejectedBacktrackingSeqNr =>
+ if (triggerReplayIfPossible(sourceProvider, offsetStore, envelope))
+ FutureDone
+ else
+ throwRejectedEnvelope(sourceProvider, envelope)
+ }
}
}
}
@@ -285,24 +328,35 @@ private[projection] object R2dbcProjectionImpl {
() =>
new AdaptedHandler(handlerFactory()) {
override def process(envelope: Envelope): Future[Done] = {
- offsetStore.isAccepted(envelope).flatMap {
- case true =>
- if (isFilteredEvent(envelope)) {
- offsetStore.addInflight(envelope)
- FutureDone
- } else {
- loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
- delegate
- .process(loadedEnvelope)
- .map { _ =>
- offsetStore.addInflight(loadedEnvelope)
- Done
- }
+ import R2dbcOffsetStore.Validation._
+ offsetStore
+ .validate(envelope)
+ .flatMap {
+ case Accepted =>
+ if (isFilteredEvent(envelope)) {
+ offsetStore.addInflight(envelope)
+ FutureDone
+ } else {
+ loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
+ delegate
+ .process(loadedEnvelope)
+ .map { _ =>
+ offsetStore.addInflight(loadedEnvelope)
+ Done
+ }
+ }
}
- }
- case false =>
- FutureDone
- }
+ case Duplicate =>
+ FutureDone
+ case RejectedSeqNr =>
+ triggerReplayIfPossible(sourceProvider, offsetStore, envelope)
+ FutureDone
+ case RejectedBacktrackingSeqNr =>
+ if (triggerReplayIfPossible(sourceProvider, offsetStore, envelope))
+ FutureDone
+ else
+ throwRejectedEnvelope(sourceProvider, envelope)
+ }
}
}
}
@@ -316,7 +370,23 @@ private[projection] object R2dbcProjectionImpl {
system: ActorSystem[_]): () => Handler[immutable.Seq[Envelope]] = { () =>
new AdaptedHandler(handlerFactory()) {
override def process(envelopes: immutable.Seq[Envelope]): Future[Done] = {
- offsetStore.filterAccepted(envelopes).flatMap { acceptedEnvelopes =>
+ import R2dbcOffsetStore.Validation._
+ offsetStore.validateAll(envelopes).flatMap { isAcceptedEnvelopes =>
+ isAcceptedEnvelopes.foreach {
+ case (env, RejectedSeqNr) =>
+ triggerReplayIfPossible(sourceProvider, offsetStore, env)
+ case (env, RejectedBacktrackingSeqNr) =>
+ if (triggerReplayIfPossible(sourceProvider, offsetStore, env))
+ FutureDone
+ else
+ throwRejectedEnvelope(sourceProvider, env)
+ case _ =>
+ }
+
+ val acceptedEnvelopes = isAcceptedEnvelopes.collect {
+ case (env, Accepted) => env
+ }
+
if (acceptedEnvelopes.isEmpty) {
FutureDone
} else {
@@ -344,35 +414,75 @@ private[projection] object R2dbcProjectionImpl {
private[projection] def adaptedHandlerForFlow[Offset, Envelope](
sourceProvider: SourceProvider[Offset, Envelope],
handler: FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _],
- offsetStore: R2dbcOffsetStore)(
- implicit
- ec: ExecutionContext): FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _] = {
-
+ offsetStore: R2dbcOffsetStore,
+ settings: R2dbcProjectionSettings)(
+ implicit system: ActorSystem[_]): FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _] = {
+ import R2dbcOffsetStore.Validation._
+ implicit val ec: ExecutionContext = system.executionContext
FlowWithContext[Envelope, ProjectionContext]
.mapAsync(1) { env =>
offsetStore
- .isAccepted(env)
- .flatMap { ok =>
- if (ok) {
- if (isFilteredEvent(env)) {
+ .validate(env)
+ .flatMap {
+ case Accepted =>
+ if (isFilteredEvent(env) && settings.warnAboutFilteredEventsInFlow) {
log.info("atLeastOnceFlow doesn't support of skipping envelopes. Envelope [{}] still emitted.", env)
}
loadEnvelope(env, sourceProvider).map { loadedEnvelope =>
offsetStore.addInflight(loadedEnvelope)
Some(loadedEnvelope)
}
- } else {
+ case Duplicate =>
Future.successful(None)
- }
+ case RejectedSeqNr =>
+ triggerReplayIfPossible(sourceProvider, offsetStore, env)
+ Future.successful(None)
+ case RejectedBacktrackingSeqNr =>
+ if (triggerReplayIfPossible(sourceProvider, offsetStore, env))
+ Future.successful(None)
+ else
+ throwRejectedEnvelope(sourceProvider, env)
}
}
- .collect { case Some(env) =>
- env
+ .collect {
+ case Some(env) =>
+ env
}
.via(handler)
}
- @nowarn // implicit parameters are used in the adapted handlers
+ private def triggerReplayIfPossible[Offset, Envelope](
+ sourceProvider: SourceProvider[Offset, Envelope],
+ offsetStore: R2dbcOffsetStore,
+ envelope: Envelope): Boolean = {
+ envelope match {
+ case env: EventEnvelope[Any @unchecked] if env.sequenceNr > 1 =>
+ sourceProvider match {
+ case provider: CanTriggerReplay =>
+ val fromSeqNr = offsetStore.storedSeqNr(env.persistenceId) + 1
+ provider.triggerReplay(env.persistenceId, fromSeqNr)
+ true
+ case _ =>
+ false // no replay support for other source providers
+ }
+ case _ =>
+ false // no replay support for non typed envelopes
+ }
+ }
+
+ private def throwRejectedEnvelope[Offset, Envelope](
+ sourceProvider: SourceProvider[Offset, Envelope],
+ envelope: Envelope): Nothing = {
+ extractOffsetPidSeqNr(sourceProvider, envelope) match {
+ case OffsetPidSeqNr(_, Some((pid, seqNr))) =>
+ throw new RejectedEnvelope(
+ s"Rejected envelope from backtracking, persistenceId [$pid], seqNr [$seqNr] due to unexpected sequence number.")
+ case OffsetPidSeqNr(_, None) =>
+ throw new RejectedEnvelope(s"Rejected envelope from backtracking.")
+ }
+ }
+
+ @nowarn("msg=never used")
abstract class AdaptedR2dbcHandler[E](val delegate: R2dbcHandler[E])(
implicit
ec: ExecutionContext,
@@ -386,7 +496,7 @@ private[projection] object R2dbcProjectionImpl {
delegate.stop()
}
- @nowarn // implicit parameters are used in the adapted handlers
+ @nowarn("msg=never used")
abstract class AdaptedHandler[E](val delegate: Handler[E])(implicit ec: ExecutionContext, system: ActorSystem[_])
extends Handler[E] {
@@ -514,7 +624,7 @@ private[projection] class R2dbcProjectionImpl[Offset, Envelope](
settings) {
implicit val executionContext: ExecutionContext = system.executionContext
- override val logger: LoggingAdapter = Logging(system.classicSystem, classOf[R2dbcInternalProjectionState])
+ override val logger: LoggingAdapter = Logging(system.classicSystem, classOf[R2dbcProjectionImpl[_, _]])
private val isExactlyOnceWithSkip: Boolean =
offsetStrategy match {
@@ -531,7 +641,7 @@ private[projection] class R2dbcProjectionImpl[Offset, Envelope](
// Called from InternalProjectionState.saveOffsetAndReport
override def saveOffset(projectionId: ProjectionId, offset: Offset): Future[Done] = {
// need the envelope to be able to call offsetStore.saveOffset
- // FIXME maybe we can cleanup this mess when moving R2dbcProjection to the Pekko Projections repository?
+ // FIXME maybe we can cleanup this mess when moving R2dbcProjection to the Pekko Projections repository? This is all internal api.
throw new IllegalStateException(
"Unexpected call to saveOffset. It should have called saveOffsetAndReport. Please report bug at https://github.com/apache/pekko-persistence-r2dbc/issues")
}
@@ -544,7 +654,7 @@ private[projection] class R2dbcProjectionImpl[Offset, Envelope](
val envelope = projectionContext.envelope
if (offsetStore.isInflight(envelope) || isExactlyOnceWithSkip) {
- val offset = R2dbcProjectionImpl.extractOffsetPidSeqNr(projectionContext.offset, envelope)
+ val offset = extractOffsetPidSeqNr(projectionContext.offset, envelope)
offsetStore
.saveOffset(offset)
.map { done =>
@@ -556,6 +666,7 @@ private[projection] class R2dbcProjectionImpl[Offset, Envelope](
getTelemetry().onOffsetStored(batchSize)
done
}
+
} else {
FutureDone
}
@@ -579,7 +690,7 @@ private[projection] class R2dbcProjectionImpl[Offset, Envelope](
if (acceptedContexts.isEmpty) {
FutureDone
} else {
- val offsets = acceptedContexts.map(ctx => R2dbcProjectionImpl.extractOffsetPidSeqNr(ctx.offset, ctx.envelope))
+ val offsets = acceptedContexts.map(ctx => extractOffsetPidSeqNr(ctx.offset, ctx.envelope))
offsetStore
.saveOffsets(offsets)
.map { done =>
@@ -600,7 +711,8 @@ private[projection] class R2dbcProjectionImpl[Offset, Envelope](
new R2dbcRunningProjection(RunningProjection.withBackoff(() => this.mappedSource(), settings), this)
}
- private class R2dbcRunningProjection(source: Source[Done, _], projectionState: R2dbcInternalProjectionState)(implicit
+ private class R2dbcRunningProjection(source: Source[Done, _], projectionState: R2dbcInternalProjectionState)(
+ implicit
system: ActorSystem[_])
extends RunningProjection
with RunningProjectionManagement[Offset] {
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcHandler.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcHandler.scala
index 6657061..3ed0416 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcHandler.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcHandler.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc.javadsl
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcProjection.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcProjection.scala
index b70441c..cd2aa97 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcProjection.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcProjection.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc.javadsl
@@ -181,9 +181,9 @@ object R2dbcProjection {
* processing semantics.
*
* The flow should emit a `Done` element for each completed envelope. The offset of the envelope is carried in the
- * context of the `FlowWithContext` and is stored in Cassandra when corresponding `Done` is emitted. Since the offset
- * is stored after processing the envelope it means that if the projection is restarted from previously stored offset
- * then some envelopes may be processed more than once.
+ * context of the `FlowWithContext` and is stored in the database when corresponding `Done` is emitted. Since the
+ * offset is stored after processing the envelope it means that if the projection is restarted from previously stored
+ * offset then some envelopes may be processed more than once.
*
* If the flow filters out envelopes the corresponding offset will not be stored, and such envelope will be processed
* again if the projection is restarted and no later offset was stored.
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcReplication.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcReplication.scala
new file mode 100644
index 0000000..075589d
--- /dev/null
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcReplication.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * license agreements; and to You under the Apache License, version 2.0:
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This file is part of the Apache Pekko project, which was derived from Akka.
+ */
+
+/*
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
+ */
+
+package org.apache.pekko.projection.r2dbc.javadsl
+
+import org.apache.pekko
+import pekko.actor.typed.ActorSystem
+import pekko.projection.grpc.replication.javadsl.ReplicationProjectionProvider
+import pekko.projection.r2dbc.R2dbcProjectionSettings
+
+import java.util.Optional
+
+object R2dbcReplication {
+
+ /**
+ * Creates a projection provider for using R2dbc as backend for the Pekko Projection gRPC transport for Replicated
+ * Event Sourcing.
+ */
+ def create(system: ActorSystem[_]): ReplicationProjectionProvider =
+ R2dbcProjection.atLeastOnceFlow(
+ _,
+ Optional.of(R2dbcProjectionSettings(system).withWarnAboutFilteredEventsInFlow(false)),
+ _,
+ _,
+ _)
+
+ /**
+ * Creates a projection provider for using R2dbc as backend for the Pekko Projection gRPC transport for Replicated
+ * Event Sourcing.
+ */
+ def create(settings: R2dbcProjectionSettings): ReplicationProjectionProvider =
+ R2dbcProjection.atLeastOnceFlow(_, Optional.of(settings.withWarnAboutFilteredEventsInFlow(false)), _, _, _)
+
+}
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcHandler.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcHandler.scala
index 3657cbd..3fa4902 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcHandler.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcHandler.scala
@@ -8,7 +8,7 @@
*/
/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
*/
package org.apache.pekko.projection.r2dbc.scaladsl
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcProjection.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcProjection.scala
index 6d8eea1..5e48a76 100644
--- a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcProjection.scala
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcProjection.scala
@@ -15,6 +15,7 @@ package org.apache.pekko.projection.r2dbc.scaladsl
import scala.collection.immutable
import scala.concurrent.duration.Duration
+
import org.apache.pekko
import pekko.Done
import pekko.actor.typed.ActorSystem
@@ -371,7 +372,7 @@ object R2dbcProjection {
connFactory)
val adaptedHandler =
- R2dbcProjectionImpl.adaptedHandlerForFlow(sourceProvider, handler, offsetStore)(system.executionContext)
+ R2dbcProjectionImpl.adaptedHandlerForFlow(sourceProvider, handler, offsetStore, r2dbcSettings)(system)
new R2dbcProjectionImpl(
projectionId,
diff --git a/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcReplication.scala b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcReplication.scala
new file mode 100644
index 0000000..8bd71c2
--- /dev/null
+++ b/r2dbc/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcReplication.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * license agreements; and to You under the Apache License, version 2.0:
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This file is part of the Apache Pekko project, which was derived from Akka.
+ */
+
+/*
+ * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
+ */
+
+package org.apache.pekko.projection.r2dbc.scaladsl
+
+import org.apache.pekko
+import pekko.actor.typed.ActorSystem
+import pekko.projection.grpc.replication.scaladsl.ReplicationProjectionProvider
+import pekko.projection.r2dbc.R2dbcProjectionSettings
+
+object R2dbcReplication {
+
+ /**
+ * Creates a projection provider for using R2dbc as backend for the Pekko Projection gRPC transport for Replicated
+ * Event Sourcing.
+ */
+ def apply()(implicit system: ActorSystem[_]): ReplicationProjectionProvider =
+ R2dbcProjection.atLeastOnceFlow(
+ _,
+ Some(R2dbcProjectionSettings(system).withWarnAboutFilteredEventsInFlow(false)),
+ _,
+ _)(_)
+
+ /**
+ * Creates a projection provider for using R2dbc as backend for the Pekko Projection gRPC transport for Replicated
+ * Event Sourcing.
+ */
+ def apply(settings: R2dbcProjectionSettings): ReplicationProjectionProvider =
+ R2dbcProjection.atLeastOnceFlow(_, Some(settings.withWarnAboutFilteredEventsInFlow(false)), _, _)(_)
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pekko.apache.org
For additional commands, e-mail: commits-help@pekko.apache.org