You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pekko.apache.org by fa...@apache.org on 2026/05/22 22:23:48 UTC
(pekko-persistence-r2dbc) branch main updated: remove pekko-projection-r2dbc (#386)
This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pekko-persistence-r2dbc.git
The following commit(s) were added to refs/heads/main by this push:
new 591d78d remove pekko-projection-r2dbc (#386)
591d78d is described below
commit 591d78dabb5d192bfa7ec82b750223a5c45b8cd7
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Fri May 22 23:23:41 2026 +0100
remove pekko-projection-r2dbc (#386)
* remove pekko-projection-r2dbc
* Update Dependencies.scala
* Update Dependencies.scala
---
.github/workflows/build-test.yml | 2 +-
CONTRIBUTING.md | 2 +-
build.sbt | 14 +-
ddl-scripts/create_tables_mysql.sql | 36 -
ddl-scripts/create_tables_postgres.sql | 36 -
ddl-scripts/create_tables_postgres_jsonb.sql | 36 -
docs/src/main/paradox/projection.md | 246 +---
.../home/projection/R2dbcProjectionDocExample.java | 262 ----
.../projection/R2dbcProjectionDocExample.scala | 233 ----
.../RuntimePluginConfigCompatibilityExample.scala | 79 --
project/Dependencies.scala | 31 +-
project/project-info.conf | 4 -
.../r2dbcprojectionsettings.excludes | 51 -
.../1.0.x.backwards.excludes/r2dbcsession.excludes | 20 -
.../envelope-origin.excludes | 25 -
projection/src/main/resources/reference.conf | 52 -
.../projection/r2dbc/R2dbcProjectionSettings.scala | 202 ---
.../internal/BySliceSourceProviderAdapter.scala | 88 --
.../projection/r2dbc/internal/OffsetPidSeqNr.scala | 32 -
.../r2dbc/internal/R2dbcHandlerAdapter.scala | 64 -
.../r2dbc/internal/R2dbcOffsetStore.scala | 1045 --------------
.../r2dbc/internal/R2dbcProjectionImpl.scala | 643 ---------
.../internal/mysql/MySQLR2dbcOffsetStore.scala | 66 -
.../projection/r2dbc/javadsl/R2dbcHandler.scala | 86 --
.../projection/r2dbc/javadsl/R2dbcProjection.scala | 221 ---
.../projection/r2dbc/javadsl/R2dbcSession.scala | 51 -
.../projection/r2dbc/scaladsl/R2dbcHandler.scala | 62 -
.../r2dbc/scaladsl/R2dbcProjection.scala | 401 ------
.../projection/r2dbc/scaladsl/R2dbcSession.scala | 44 -
projection/src/test/resources/logback-test.xml | 26 -
.../pekko/projection/TestStatusObserver.scala | 88 --
.../r2dbc/DurableStateEndToEndSpec.scala | 261 ----
.../projection/r2dbc/EventSourcedChaosSpec.scala | 316 -----
.../r2dbc/EventSourcedEndToEndSpec.scala | 342 -----
.../projection/r2dbc/EventSourcedPubSubSpec.scala | 257 ----
.../projection/r2dbc/R2dbcOffsetStoreSpec.scala | 273 ----
.../r2dbc/R2dbcOffsetStoreStateSpec.scala | 109 --
.../projection/r2dbc/R2dbcProjectionSpec.scala | 1408 -------------------
.../r2dbc/R2dbcTimestampOffsetProjectionSpec.scala | 1460 --------------------
.../r2dbc/R2dbcTimestampOffsetStoreSpec.scala | 863 ------------
.../projection/r2dbc/RuntimePluginConfigSpec.scala | 277 ----
.../apache/pekko/projection/r2dbc/TestClock.scala | 61 -
.../apache/pekko/projection/r2dbc/TestConfig.scala | 78 --
.../apache/pekko/projection/r2dbc/TestData.scala | 39 -
.../pekko/projection/r2dbc/TestDbLifecycle.scala | 81 --
.../r2dbc/TestSourceProviderWithInput.scala | 110 --
46 files changed, 7 insertions(+), 10176 deletions(-)
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 56a54cd..3c1419d 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -185,7 +185,7 @@ jobs:
docker exec -i yb-tserver-n1 /home/yugabyte/bin/ysqlsh -h yb-tserver-n1 -t -d database2 < ddl-scripts/create_tables_yugabyte.sql
- name: test
- run: sbt -Dpekko.persistence.r2dbc.dialect=yugabyte -Dpekko.projection.r2dbc.dialect=yugabyte ++${{ matrix.SCALA_VERSION }} test
+ run: sbt -Dpekko.persistence.r2dbc.dialect=yugabyte ++${{ matrix.SCALA_VERSION }} test
test-mysql:
name: Run tests with MySQL
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9b1d7a3..6446294 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -61,7 +61,7 @@ docker exec -i yb-tserver-n1 /home/yugabyte/bin/ysqlsh -h yb-tserver-n1 -t < ddl
Run tests with:
```
-sbt -Dpekko.persistence.r2dbc.dialect=yugabyte sbt -Dpekko.projection.r2dbc.dialect=yugabyte
+sbt -Dpekko.persistence.r2dbc.dialect=yugabyte
```
### Connect with ysqlsh
diff --git a/build.sbt b/build.sbt
index 4005ac6..835ed7b 100644
--- a/build.sbt
+++ b/build.sbt
@@ -46,7 +46,7 @@ lazy val root = (project in file("."))
.settings(dontPublish)
.settings(
name := "pekko-persistence-r2dbc-root")
- .aggregate(core, projection, migration, docs)
+ .aggregate(core, migration, docs)
def suffixFileFilter(suffix: String): FileFilter = new SimpleFileFilter(f => f.getAbsolutePath.endsWith(suffix))
@@ -56,14 +56,6 @@ lazy val core = (project in file("core"))
name := "pekko-persistence-r2dbc",
libraryDependencies ++= Dependencies.core)
-lazy val projection = (project in file("projection"))
- .dependsOn(core % "compile->compile;test->test")
- .enablePlugins(ReproducibleBuildsPlugin)
- .settings(
- name := "pekko-projection-r2dbc",
- libraryDependencies ++= Dependencies.projection,
- dependencyOverrides ++= Dependencies.pekkoTestDependencyOverrides)
-
lazy val migration = (project in file("migration"))
.enablePlugins(ReproducibleBuildsPlugin)
.settings(
@@ -87,7 +79,7 @@ lazy val docs = project
.in(file("docs"))
.enablePlugins(PekkoParadoxPlugin, ParadoxSitePlugin, ScalaUnidocPlugin)
.disablePlugins(MimaPlugin)
- .dependsOn(core, projection, migration)
+ .dependsOn(core, migration)
.settings(dontPublish)
.settings(
name := "Apache Pekko Persistence R2DBC",
@@ -111,8 +103,6 @@ lazy val docs = project
"scaladoc.scala.base_url" -> "https://www.scala-lang.org/api/current/",
"scaladoc.org.apache.pekko.persistence.r2dbc.base_url" ->
s"https://pekko.apache.org/api/pekko-persistence-r2dbc/${Dependencies.PekkoPersistenceR2dbcVersionInDocs}",
- "scaladoc.org.apache.pekko.projection.r2dbc.base_url" ->
- s"https://pekko.apache.org/api/pekko-persistence-r2dbc/${Dependencies.PekkoPersistenceR2dbcVersionInDocs}",
"scaladoc.org.apache.pekko.projection.base_url" ->
s"https://pekko.apache.org/api/pekko-projection/${Dependencies.PekkoProjectionVersionInDocs}",
"scaladoc.org.apache.pekko.base_url" -> s"https://pekko.apache.org/api/pekko/${Dependencies.PekkoVersionInDocs}",
diff --git a/ddl-scripts/create_tables_mysql.sql b/ddl-scripts/create_tables_mysql.sql
index e006404..3cb0aee 100644
--- a/ddl-scripts/create_tables_mysql.sql
+++ b/ddl-scripts/create_tables_mysql.sql
@@ -74,39 +74,3 @@ CREATE TABLE IF NOT EXISTS durable_state (
-- `durable_state_slice_idx` is only needed if the slice based queries are used
CREATE INDEX durable_state_slice_idx ON durable_state(slice, entity_type, db_timestamp, revision);
-
--- Primitive offset types are stored in this table.
--- If only timestamp based offsets are used this table is optional.
--- Configure pekko.projection.r2dbc.offset-store.offset-table="" if the table is not created.
-CREATE TABLE IF NOT EXISTS projection_offset_store (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- current_offset VARCHAR(255) NOT NULL,
- manifest VARCHAR(32) NOT NULL,
- mergeable BOOLEAN NOT NULL,
- last_updated BIGINT NOT NULL,
- PRIMARY KEY(projection_name, projection_key)
-);
-
--- Timestamp based offsets are stored in this table.
-CREATE TABLE IF NOT EXISTS projection_timestamp_offset_store (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- slice INT NOT NULL,
- persistence_id VARCHAR(255) NOT NULL,
- seq_nr BIGINT NOT NULL,
- -- timestamp_offset is the db_timestamp of the original event
- timestamp_offset TIMESTAMP(6) NOT NULL,
- -- timestamp_consumed is when the offset was stored
- -- the consumer lag is timestamp_consumed - timestamp_offset
- timestamp_consumed TIMESTAMP(6) NOT NULL,
- PRIMARY KEY(slice, projection_name, timestamp_offset, persistence_id, seq_nr)
-);
-
-CREATE TABLE IF NOT EXISTS projection_management (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- paused BOOLEAN NOT NULL,
- last_updated BIGINT NOT NULL,
- PRIMARY KEY(projection_name, projection_key)
-);
diff --git a/ddl-scripts/create_tables_postgres.sql b/ddl-scripts/create_tables_postgres.sql
index ebb91c6..2ded2f4 100644
--- a/ddl-scripts/create_tables_postgres.sql
+++ b/ddl-scripts/create_tables_postgres.sql
@@ -57,39 +57,3 @@ CREATE TABLE IF NOT EXISTS durable_state (
-- `durable_state_slice_idx` is only needed if the slice based queries are used
CREATE INDEX IF NOT EXISTS durable_state_slice_idx ON durable_state(slice, entity_type, db_timestamp, revision);
-
--- Primitive offset types are stored in this table.
--- If only timestamp based offsets are used this table is optional.
--- Configure pekko.projection.r2dbc.offset-store.offset-table="" if the table is not created.
-CREATE TABLE IF NOT EXISTS projection_offset_store (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- current_offset VARCHAR(255) NOT NULL,
- manifest VARCHAR(32) NOT NULL,
- mergeable BOOLEAN NOT NULL,
- last_updated BIGINT NOT NULL,
- PRIMARY KEY(projection_name, projection_key)
-);
-
--- Timestamp based offsets are stored in this table.
-CREATE TABLE IF NOT EXISTS projection_timestamp_offset_store (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- slice INT NOT NULL,
- persistence_id VARCHAR(255) NOT NULL,
- seq_nr BIGINT NOT NULL,
- -- timestamp_offset is the db_timestamp of the original event
- timestamp_offset timestamp with time zone NOT NULL,
- -- timestamp_consumed is when the offset was stored
- -- the consumer lag is timestamp_consumed - timestamp_offset
- timestamp_consumed timestamp with time zone NOT NULL,
- PRIMARY KEY(slice, projection_name, timestamp_offset, persistence_id, seq_nr)
-);
-
-CREATE TABLE IF NOT EXISTS projection_management (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- paused BOOLEAN NOT NULL,
- last_updated BIGINT NOT NULL,
- PRIMARY KEY(projection_name, projection_key)
-);
diff --git a/ddl-scripts/create_tables_postgres_jsonb.sql b/ddl-scripts/create_tables_postgres_jsonb.sql
index 6c2e36a..00e93d7 100644
--- a/ddl-scripts/create_tables_postgres_jsonb.sql
+++ b/ddl-scripts/create_tables_postgres_jsonb.sql
@@ -59,39 +59,3 @@ CREATE TABLE IF NOT EXISTS durable_state (
-- `durable_state_slice_idx` is only needed if the slice based queries are used
CREATE INDEX IF NOT EXISTS durable_state_slice_idx ON durable_state(slice, entity_type, db_timestamp, revision);
-
--- Primitive offset types are stored in this table.
--- If only timestamp based offsets are used this table is optional.
--- Configure pekko.projection.r2dbc.offset-store.offset-table="" if the table is not created.
-CREATE TABLE IF NOT EXISTS pekko_projection_offset_store (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- current_offset VARCHAR(255) NOT NULL,
- manifest VARCHAR(32) NOT NULL,
- mergeable BOOLEAN NOT NULL,
- last_updated BIGINT NOT NULL,
- PRIMARY KEY(projection_name, projection_key)
-);
-
--- Timestamp based offsets are stored in this table.
-CREATE TABLE IF NOT EXISTS pekko_projection_timestamp_offset_store (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- slice INT NOT NULL,
- persistence_id VARCHAR(255) NOT NULL,
- seq_nr BIGINT NOT NULL,
- -- timestamp_offset is the db_timestamp of the original event
- timestamp_offset timestamp with time zone NOT NULL,
- -- timestamp_consumed is when the offset was stored
- -- the consumer lag is timestamp_consumed - timestamp_offset
- timestamp_consumed timestamp with time zone NOT NULL,
- PRIMARY KEY(slice, projection_name, timestamp_offset, persistence_id, seq_nr)
-);
-
-CREATE TABLE IF NOT EXISTS pekko_projection_management (
- projection_name VARCHAR(255) NOT NULL,
- projection_key VARCHAR(255) NOT NULL,
- paused BOOLEAN NOT NULL,
- last_updated BIGINT NOT NULL,
- PRIMARY KEY(projection_name, projection_key)
-);
diff --git a/docs/src/main/paradox/projection.md b/docs/src/main/paradox/projection.md
index b9eb5e6..5bc619e 100644
--- a/docs/src/main/paradox/projection.md
+++ b/docs/src/main/paradox/projection.md
@@ -1,247 +1,3 @@
# Projection
-The @apidoc[R2dbcProjection$] has support for storing the offset in a relational database using R2DBC.
-
-The source of the envelopes is from a `SourceProvider`, which can be:
-
-* events from Event Sourced entities via the @extref:[SourceProvider for eventsBySlices](pekko-projection:eventsourced.html#sourceprovider-for-eventsbyslices) with the @ref:[eventsBySlices query](query.md#eventsbyslices)
-* state changes for Durable State entities via the @extref:[SourceProvider for changesBySlices](pekko-projection:durable-state.html#sourceprovider-for-changesbyslices) with the @ref:[changesBySlices query](query.md#changesbyslices)
-* any other `SourceProvider` with supported @ref:[offset types](#offset-types)
-
-A @apidoc[R2dbcHandler] receives a @apidoc[org.apache.pekko.projection.r2dbc.*.R2dbcSession] instance and an envelope. The
-`R2dbcSession` provides the means to access an open R2DBC connection that can be used to process the envelope.
-The target database operations can be run in the same transaction as the storage of the offset, which means
-that @ref:[exactly-once](#exactly-once) processing semantics is supported. It also offers
-@ref:[at-least-once](#at-least-once) semantics.
-
-## Dependencies
-
-To use the R2DBC module of Pekko Projections add the following dependency in your project:
-
-@@dependency [Maven,sbt,Gradle] {
- group=org.apache.pekko
- artifact=pekko-projection-r2dbc_$scala.binary.version$
- version=$project.version$
-}
-
-Pekko Projections R2DBC depends on Pekko $pekko.version$ or later, and note that it is important that all `pekko-*`
-dependencies are in the same version, so it is recommended to depend on them explicitly to avoid problems
-with transient dependencies causing an unlucky mix of versions.
-
-@@project-info{ projectId="projection" }
-
-
-### Transitive dependencies
-
-The table below shows `pekko-projection-r2dbc`'s direct dependencies, and the second tab shows all libraries it depends on transitively.
-
-@@dependencies{ projectId="projection" }
-
-## Schema
-
-The `projection_offset_store`, `projection_timestamp_offset_store` and `projection_management` tables
-need to be created in the configured database, see schema definition in @ref:[Creating the schema](getting-started.md#schema).
-
-## Configuration
-
-By default, `pekko-projection-r2dbc` uses the same connection pool and `dialect` as `pekko-persistence-r2dbc`, see
-@ref:[Connection configuration](config.md#connection-configuration).
-
-### Reference configuration
-
-The following can be overridden in your `application.conf` for the Projection specific settings:
-
-@@snip [reference.conf](/projection/src/main/resources/reference.conf) {#projection-config}
-
-## Running with Sharded Daemon Process
-
-The Sharded Daemon Process can be used to distribute `n` instances of a given Projection across the cluster.
-Therefore, it's important that each Projection instance consumes a subset of the stream of envelopes.
-
-When using `eventsBySlices` the initialization code looks like this:
-
-Scala
-: @@snip [R2dbcProjectionDocExample.scala](/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala) { #initProjections }
-
-Java
-: @@snip [R2dbcProjectionDocExample.java](/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java) { #initProjections }
-
-The @ref:[`ShoppingCartHandler` is shown below](#handler).
-
-There are alternative ways of running the `ProjectionBehavior` as described in @extref:[Running a Projection](pekko-projection:running.html), but note that when using the R2DBC plugin as `SourceProvider` it is recommended to use `eventsBySlices` and not `eventsByTag`.
-
-## Slices
-
-The `SourceProvider` for Event Sourced actors has historically been using `eventsByTag` but the R2DBC plugin is
-instead providing `eventsBySlices` as an improved solution.
-
-The usage of `eventsByTag` for Projections has the drawback that the number of tags must be decided
-up-front and can't easily be changed afterwards. Starting with too many tags means much overhead since
-many projection instances would be running on each node in a small Pekko Cluster. Each projection instance
-polling the database periodically. Starting with too few tags means that it can't be scaled later to more
-Pekko nodes.
-
-With `eventsBySlices` more Projection instances can be added when needed and still reuse the offsets
-for the previous slice distributions.
-
-A slice is deterministically defined based on the persistence id. The purpose is to evenly distribute all
-persistence ids over the slices. The `eventsBySlices` query is for a range of the slices. For example if
-using 1024 slices and running 4 Projection instances the slice ranges would be 0-255, 256-511, 512-767, 768-1023.
-Changing to 8 slice ranges means that the ranges would be 0-127, 128-255, 256-383, ..., 768-895, 896-1023.
-
-However, when changing the number of slices the projections with the old slice distribution must be
-stopped before starting new projections. That can be done with a full shutdown before deploying the
-new slice distribution or pause (stop) the projections with @extref:[the management API](pekko-projection:management.html).
-
-When using `R2dbcProjection` together with the `EventSourcedProvider.eventsBySlices` the events will be delivered in
-sequence number order without duplicates.
-
-When using `R2dbcProjection` together with `DurableStateSourceProvider.changesBySlices` the changes will be delivered
-in revision number order without duplicates.
-
-## exactly-once
-
-The offset is stored in the same transaction used for the user defined `handler`, which means exactly-once
-processing semantics if the projection is restarted from previously stored offset.
-
-Scala
-: @@snip [R2dbcProjectionDocExample.scala](/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala) { #exactlyOnce }
-
-Java
-: @@snip [R2dbcProjectionDocExample.java](/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java) { #exactlyOnce }
-
-The @ref:[`ShoppingCartHandler` is shown below](#handler).
-
-## at-least-once
-
-The offset is stored after the envelope has been processed and giving at-least-once processing semantics.
-This means that if the projection is restarted from a previously stored offset some elements may be processed more
-than once. Therefore, the @ref:[Handler](#handler) code must be idempotent.
-
-Scala
-: @@snip [R2dbcProjectionDocExample.scala](/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala) { #atLeastOnce }
-
-Java
-: @@snip [R2dbcProjectionDocExample.java](/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java) { #atLeastOnce }
-
-The offset is stored after a time window, or limited by a number of envelopes, whatever happens first.
-This window can be defined with `withSaveOffset` of the returned `AtLeastOnceProjection`.
-The default settings for the window is defined in configuration section `pekko.projection.at-least-once`.
-There is a performance benefit of not storing the offset too often, but the drawback is that there can be more
-duplicates when the projection that will be processed again when the projection is restarted.
-
-The @ref:[`ShoppingCartHandler` is shown below](#handler).
-
-## groupedWithin
-
-The envelopes can be grouped before processing, which can be useful for batch updates.
-
-Scala
-: @@snip [R2dbcProjectionDocExample.scala](/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala) { #grouped }
-
-Java
-: @@snip [R2dbcProjectionDocExample.java](/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java) { #grouped }
-
-The envelopes are grouped within a time window, or limited by a number of envelopes, whatever happens first.
-This window can be defined with `withGroup` of the returned `GroupedProjection`. The default settings for
-the window is defined in configuration section `pekko.projection.grouped`.
-
-When using `groupedWithin` the handler is a @scala[`R2dbcHandler[immutable.Seq[EventEnvelope[ShoppingCart.Event]]]`]@java[`R2dbcHandler<List<EventEnvelope<ShoppingCart.Event>>>`].
-The @ref:[`GroupedShoppingCartHandler` is shown below](#grouped-handler).
-
-The offset is stored in the same transaction used for the user defined `handler`, which means exactly-once
-processing semantics if the projection is restarted from previously stored offset.
-
-## Handler
-
-It's in the @apidoc[R2dbcHandler] that you implement the processing of each envelope. It's essentially a consumer function
-from `(R2dbcSession, Envelope)` to @scala[`Future[Done]`]@java[`CompletionStage<Done>`].
-
-A handler that is consuming `ShoppingCart.Event` from `eventsBySlices` can look like this:
-
-Scala
-: @@snip [R2dbcProjectionDocExample.scala](/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala) { #handler }
-
-Java
-: @@snip [R2dbcProjectionDocExample.java](/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java) { #handler }
-
-@@@ note { title=Hint }
-Such simple handlers can also be defined as plain functions via the helper @scala[`R2dbcHandler.apply`]@java[`R2dbcHandler.fromFunction`] factory method.
-@@@
-
-### Grouped handler
-
-When using @ref:[`R2dbcProjection.groupedWithin`](#groupedwithin) the handler is processing a @scala[`Seq`]@java[`List`] of envelopes.
-
-Scala
-: @@snip [R2dbcProjectionDocExample.scala](/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala) { #grouped-handler }
-
-Java
-: @@snip [R2dbcProjectionDocExample.java](/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java) { #grouped-handler }
-
-### Stateful handler
-
-The @apidoc[R2dbcHandler] can be stateful, with variables and mutable data structures. It is invoked by the `Projection` machinery
-one envelope at a time and visibility guarantees between the invocations are handled automatically, i.e. no volatile
-or other concurrency primitives are needed for managing the state as long as it's not accessed by other threads
-than the one that called `process`.
-
-@@@ note
-
-It is important that the `Handler` instance is not shared between several `Projection` instances,
-because then it would be invoked concurrently, which is not how it is intended to be used. Each `Projection`
-instance should use a new `Handler` instance.
-
-@@@
-
-### Async handler
-
-The @apidoc[Handler] can be used with `R2dbcProjection.atLeastOnceAsync` and
-`R2dbcProjection.groupedWithinAsync` if the handler is not storing the projection result in the database.
-The handler could send to a Kafka topic or integrate with something else.
-
-There are several examples of such `Handler` in the @extref:[documentation for Cassandra Projections](pekko-projection:cassandra.html#handler).
-Same type of handlers can be used with `R2dbcProjection` instead of `CassandraProjection`.
-
-### Actor handler
-
-A good alternative for advanced state management is to implement the handler as an
-@extref:[actor](pekko:typed/actors.html) which is described in
-@extref:[Processing with Actor](pekko-projection:actor.html).
-
-### Flow handler
-
-A Pekko Streams `FlowWithContext` can be used instead of a handler for processing the envelopes,
-which is described in @extref:[Processing with Pekko Streams](pekko-projection:flow.html).
-
-### Handler lifecycle
-
-You can override the `start` and `stop` methods of the `R2dbcHandler` to implement initialization
-before first envelope is processed and resource cleanup when the projection is stopped.
-Those methods are also called when the `Projection` is restarted after failure.
-
-See also @extref:[error handling](pekko-projection:error.html).
-
-## Offset types
-
-The supported offset types of the `R2dbcProjection` are:
-
-* @apidoc[pekko.persistence.query.TimestampOffset] that is used for @extref:[SourceProvider for eventsBySlices](pekko-projection:eventsourced.html#sourceprovider-for-eventsbyslices) and @extref:[SourceProvider for changesBySlices](pekko-projection:durable-state.html#sourceprovider-for-changesbyslices)
-* other @apidoc[pekko.persistence.query.Offset] types
-* @apidoc[MergeableOffset] that is used for @extref:[messages from Kafka](pekko-projection:kafka.html#mergeable-offset)
-* `String`
-* `Int`
-* `Long`
-* Any other type that has a configured Pekko Serializer is stored with base64 encoding of the serialized bytes.
-
-## Publish events for lower latency
-
-See @ref:[eventsBySlices documentation](query.md#publish-events-for-lower-latency-of-eventsbyslices).
-
-## Compatibility with plugin configuration at runtime
-
-`R2dbcProjection` is compatible with @ref:[plugin configuration at runtime](config.md#plugin-configuration-at-runtime).
-
-The following example demonstrates how an `atLeastOnce` projection of `EventSourcedBehavior` events, which where persisted using plugin configuration at runtime, can be set up:
-
-@@snip [RuntimePluginConfigCompatibilityExample.scala](/docs/src/test/scala/docs/home/projection/RuntimePluginConfigCompatibilityExample.scala) { #runtime-plugin-config-compatibility }
\ No newline at end of file
+This lib has moved to the @extref:[Apache Pekko Projection](pekko-projection:). This was part of the 2.0.0 refactor. The pekko-projection-r2dbc lib still works the same way but it will be released at the same time as other Pekko Projection libs instead of being released with pekko-persistence-r2dbc.
diff --git a/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java b/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java
deleted file mode 100644
index c16ae43..0000000
--- a/docs/src/test/java/jdocs/home/projection/R2dbcProjectionDocExample.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package jdocs.home.projection;
-
-import org.apache.pekko.Done;
-import org.apache.pekko.actor.typed.ActorSystem;
-import org.apache.pekko.actor.typed.javadsl.Behaviors;
-import org.apache.pekko.cluster.sharding.typed.javadsl.EntityTypeKey;
-import org.apache.pekko.japi.Pair;
-import org.apache.pekko.persistence.query.Offset;
-import org.apache.pekko.persistence.r2dbc.query.javadsl.R2dbcReadJournal;
-import org.apache.pekko.projection.eventsourced.javadsl.EventSourcedProvider;
-import org.apache.pekko.projection.javadsl.SourceProvider;
-
-import docs.home.CborSerializable;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-// #handler
-// #grouped-handler
-import org.apache.pekko.projection.r2dbc.javadsl.R2dbcHandler;
-import org.apache.pekko.projection.r2dbc.javadsl.R2dbcSession;
-import io.r2dbc.spi.Statement;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionStage;
-
-// #grouped-handler
-// #handler
-
-// #initProjections
-import org.apache.pekko.cluster.sharding.typed.javadsl.ShardedDaemonProcess;
-import org.apache.pekko.projection.ProjectionBehavior;
-import org.apache.pekko.persistence.query.typed.EventEnvelope;
-import org.apache.pekko.projection.Projection;
-// #initProjections
-
-// #exactlyOnce
-// #atLeastOnce
-// #grouped
-// #initProjections
-import org.apache.pekko.projection.ProjectionId;
-import org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings;
-import org.apache.pekko.projection.r2dbc.javadsl.R2dbcProjection;
-
-// #initProjections
-// #grouped
-// #atLeastOnce
-// #exactlyOnce
-
-@SuppressWarnings({"unused", "InnerClassMayBeStatic"})
-class R2dbcProjectionDocExample {
-
- static class ShoppingCart {
- public static EntityTypeKey<Command> ENTITY_TYPE_KEY =
- EntityTypeKey.create(Command.class, "ShoppingCart");
-
- interface Command extends CborSerializable {}
-
- interface Event {
- String getCartId();
- }
-
- public static class CheckedOut implements Event {
-
- public final String cartId;
- public final Instant eventTime;
-
- public CheckedOut(String cartId, Instant eventTime) {
- this.cartId = cartId;
- this.eventTime = eventTime;
- }
-
- public String getCartId() {
- return cartId;
- }
-
- @Override
- public String toString() {
- return "CheckedOut(" + cartId + "," + eventTime + ")";
- }
- }
- }
-
- // #handler
- public class ShoppingCartHandler extends R2dbcHandler<EventEnvelope<ShoppingCart.Event>> {
- private final Logger logger = LoggerFactory.getLogger(getClass());
-
- @Override
- public CompletionStage<Done> process(
- R2dbcSession session, EventEnvelope<ShoppingCart.Event> envelope) {
- ShoppingCart.Event event = envelope.event();
- if (event instanceof ShoppingCart.CheckedOut) {
- ShoppingCart.CheckedOut checkedOut = (ShoppingCart.CheckedOut) event;
- logger.info(
- "Shopping cart {} was checked out at {}", checkedOut.cartId, checkedOut.eventTime);
-
- Statement stmt =
- session
- .createStatement("INSERT into order (id, time) VALUES ($1, $2)")
- .bind(0, checkedOut.cartId)
- .bind(1, checkedOut.eventTime);
- return session.updateOne(stmt).thenApply(rowsUpdated -> Done.getInstance());
-
- } else {
- logger.debug("Shopping cart {} changed by {}", event.getCartId(), event);
- return CompletableFuture.completedFuture(Done.getInstance());
- }
- }
- }
- // #handler
-
- // #grouped-handler
- public class GroupedShoppingCartHandler
- extends R2dbcHandler<List<EventEnvelope<ShoppingCart.Event>>> {
- private final Logger logger = LoggerFactory.getLogger(getClass());
-
- @Override
- public CompletionStage<Done> process(
- R2dbcSession session, List<EventEnvelope<ShoppingCart.Event>> envelopes) {
- List<Statement> stmts = new ArrayList<>();
- for (EventEnvelope<ShoppingCart.Event> envelope : envelopes) {
- ShoppingCart.Event event = envelope.event();
- if (event instanceof ShoppingCart.CheckedOut) {
- ShoppingCart.CheckedOut checkedOut = (ShoppingCart.CheckedOut) event;
- logger.info(
- "Shopping cart {} was checked out at {}", checkedOut.cartId, checkedOut.eventTime);
-
- Statement stmt =
- session
- .createStatement("INSERT into order (id, time) VALUES ($1, $2)")
- .bind(0, checkedOut.cartId)
- .bind(1, checkedOut.eventTime);
- stmts.add(stmt);
- } else {
- logger.debug("Shopping cart {} changed by {}", event.getCartId(), event);
- }
- }
-
- return session.update(stmts).thenApply(rowsUpdated -> Done.getInstance());
- }
- }
- // #grouped-handler
-
- ActorSystem<Void> system = ActorSystem.create(Behaviors.empty(), "Example");
-
- // #initProjections
- void initProjections() {
- // Split the slices into 4 ranges
- int numberOfSliceRanges = 4;
- List<Pair<Integer, Integer>> sliceRanges =
- EventSourcedProvider.sliceRanges(
- system, R2dbcReadJournal.Identifier(), numberOfSliceRanges);
-
- ShardedDaemonProcess.get(system)
- .init(
- ProjectionBehavior.Command.class,
- "ShoppingCartProjection",
- sliceRanges.size(),
- i -> ProjectionBehavior.create(createProjection(sliceRanges.get(i))),
- ProjectionBehavior.stopMessage());
- }
-
- Projection<EventEnvelope<ShoppingCart.Event>> createProjection(
- Pair<Integer, Integer> sliceRange) {
- int minSlice = sliceRange.first();
- int maxSlice = sliceRange.second();
-
- String entityType = ShoppingCart.ENTITY_TYPE_KEY.name();
-
- SourceProvider<Offset, EventEnvelope<ShoppingCart.Event>> sourceProvider =
- EventSourcedProvider.eventsBySlices(
- system, R2dbcReadJournal.Identifier(), entityType, minSlice, maxSlice);
-
- ProjectionId projectionId =
- ProjectionId.of("ShoppingCarts", "carts-" + minSlice + "-" + maxSlice);
- Optional<R2dbcProjectionSettings> settings = Optional.empty();
-
- return R2dbcProjection.exactlyOnce(
- projectionId, settings, sourceProvider, ShoppingCartHandler::new, system);
- }
- // #initProjections
-
- // #sourceProvider
- // Split the slices into 4 ranges
- int numberOfSliceRanges = 4;
- List<Pair<Integer, Integer>> sliceRanges =
- EventSourcedProvider.sliceRanges(system, R2dbcReadJournal.Identifier(), numberOfSliceRanges);
-
- // Example of using the first slice range
- int minSlice = sliceRanges.get(0).first();
- int maxSlice = sliceRanges.get(0).second();
- String entityType = ShoppingCart.ENTITY_TYPE_KEY.name();
-
- SourceProvider<Offset, EventEnvelope<ShoppingCart.Event>> sourceProvider =
- EventSourcedProvider.eventsBySlices(
- system, R2dbcReadJournal.Identifier(), entityType, minSlice, maxSlice);
- // #sourceProvider
-
- {
- // #exactlyOnce
- ProjectionId projectionId =
- ProjectionId.of("ShoppingCarts", "carts-" + minSlice + "-" + maxSlice);
-
- Optional<R2dbcProjectionSettings> settings = Optional.empty();
-
- Projection<EventEnvelope<ShoppingCart.Event>> projection =
- R2dbcProjection.exactlyOnce(
- projectionId, settings, sourceProvider, ShoppingCartHandler::new, system);
- // #exactlyOnce
- }
-
- {
- // #atLeastOnce
- ProjectionId projectionId =
- ProjectionId.of("ShoppingCarts", "carts-" + minSlice + "-" + maxSlice);
-
- Optional<R2dbcProjectionSettings> settings = Optional.empty();
-
- int saveOffsetAfterEnvelopes = 100;
- Duration saveOffsetAfterDuration = Duration.ofMillis(500);
-
- Projection<EventEnvelope<ShoppingCart.Event>> projection =
- R2dbcProjection.atLeastOnce(
- projectionId, settings, sourceProvider, ShoppingCartHandler::new, system)
- .withSaveOffset(saveOffsetAfterEnvelopes, saveOffsetAfterDuration);
- // #atLeastOnce
- }
-
- {
- // #grouped
- ProjectionId projectionId =
- ProjectionId.of("ShoppingCarts", "carts-" + minSlice + "-" + maxSlice);
-
- Optional<R2dbcProjectionSettings> settings = Optional.empty();
-
- int saveOffsetAfterEnvelopes = 100;
- Duration saveOffsetAfterDuration = Duration.ofMillis(500);
-
- Projection<EventEnvelope<ShoppingCart.Event>> projection =
- R2dbcProjection.groupedWithin(
- projectionId, settings, sourceProvider, GroupedShoppingCartHandler::new, system)
- .withGroup(saveOffsetAfterEnvelopes, saveOffsetAfterDuration);
- // #grouped
- }
-}
diff --git a/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala b/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala
deleted file mode 100644
index c8150e4..0000000
--- a/docs/src/test/scala/docs/home/projection/R2dbcProjectionDocExample.scala
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package docs.home.projection
-
-import java.time.Instant
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import scala.concurrent.duration._
-
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.typed.ActorSystem
-import pekko.actor.typed.scaladsl.Behaviors
-import pekko.cluster.sharding.typed.scaladsl.EntityTypeKey
-import pekko.persistence.query.Offset
-import docs.home.CborSerializable
-import org.slf4j.LoggerFactory
-
-//#handler
-//#grouped-handler
-import org.apache.pekko
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import pekko.persistence.query.typed.EventEnvelope
-
-//#grouped-handler
-//#handler
-
-object R2dbcProjectionDocExample {
-
- object ShoppingCart {
- val EntityKey: EntityTypeKey[Command] = EntityTypeKey[Command]("ShoppingCart")
-
- sealed trait Command extends CborSerializable
-
- sealed trait Event extends CborSerializable {
- def cartId: String
- }
-
- final case class ItemAdded(cartId: String, itemId: String, quantity: Int) extends Event
- final case class ItemRemoved(cartId: String, itemId: String) extends Event
- final case class ItemQuantityAdjusted(cartId: String, itemId: String, newQuantity: Int) extends Event
- final case class CheckedOut(cartId: String, eventTime: Instant) extends Event
- }
-
- // #handler
- class ShoppingCartHandler()(implicit ec: ExecutionContext) extends R2dbcHandler[EventEnvelope[ShoppingCart.Event]] {
- private val logger = LoggerFactory.getLogger(getClass)
-
- override def process(session: R2dbcSession, envelope: EventEnvelope[ShoppingCart.Event]): Future[Done] = {
- envelope.event match {
- case ShoppingCart.CheckedOut(cartId, time) =>
- logger.info(s"Shopping cart $cartId was checked out at $time")
- val stmt = session
- .createStatement("INSERT into order (id, time) VALUES ($1, $2)")
- .bind(0, cartId)
- .bind(1, time)
- session
- .updateOne(stmt)
- .map(_ => Done)
-
- case otherEvent =>
- logger.debug(s"Shopping cart ${otherEvent.cartId} changed by $otherEvent")
- Future.successful(Done)
- }
- }
- }
- // #handler
-
- // #grouped-handler
- import scala.collection.immutable
-
- class GroupedShoppingCartHandler()(implicit ec: ExecutionContext)
- extends R2dbcHandler[immutable.Seq[EventEnvelope[ShoppingCart.Event]]] {
- private val logger = LoggerFactory.getLogger(getClass)
-
- override def process(
- session: R2dbcSession,
- envelopes: immutable.Seq[EventEnvelope[ShoppingCart.Event]]): Future[Done] = {
-
- // save all events in DB
- val stmts = envelopes
- .map(_.event)
- .collect { case ShoppingCart.CheckedOut(cartId, time) =>
- logger.info(s"Shopping cart $cartId was checked out at $time")
-
- session
- .createStatement("INSERT into order (id, time) VALUES ($1, $2)")
- .bind(0, cartId)
- .bind(1, time)
-
- }
- .toVector
-
- session.update(stmts).map(_ => Done)
- }
- }
- // #grouped-handler
-
- implicit val system: ActorSystem[Nothing] = ActorSystem[Nothing](Behaviors.empty, "Example")
- implicit val ec: ExecutionContext = system.executionContext
-
- object IllustrateInit {
- // #initProjections
- import org.apache.pekko
- import pekko.cluster.sharding.typed.scaladsl.ShardedDaemonProcess
- import pekko.projection.r2dbc.scaladsl.R2dbcProjection
- import pekko.persistence.r2dbc.query.scaladsl.R2dbcReadJournal
- import pekko.projection.ProjectionId
- import pekko.projection.eventsourced.scaladsl.EventSourcedProvider
- import pekko.projection.Projection
- import pekko.projection.ProjectionBehavior
- import pekko.projection.scaladsl.SourceProvider
- import pekko.persistence.query.typed.EventEnvelope
-
- def initProjections(): Unit = {
- def sourceProvider(sliceRange: Range): SourceProvider[Offset, EventEnvelope[ShoppingCart.Event]] =
- EventSourcedProvider
- .eventsBySlices[ShoppingCart.Event](
- system,
- readJournalPluginId = R2dbcReadJournal.Identifier,
- entityType,
- sliceRange.min,
- sliceRange.max)
-
- def projection(sliceRange: Range): Projection[EventEnvelope[ShoppingCart.Event]] = {
- val minSlice = sliceRange.min
- val maxSlice = sliceRange.max
- val projectionId = ProjectionId("ShoppingCarts", s"carts-$minSlice-$maxSlice")
-
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- settings = None,
- sourceProvider(sliceRange),
- handler = () => new ShoppingCartHandler)
- }
-
- // Split the slices into 4 ranges
- val numberOfSliceRanges: Int = 4
- val sliceRanges = EventSourcedProvider.sliceRanges(system, R2dbcReadJournal.Identifier, numberOfSliceRanges)
-
- ShardedDaemonProcess(system).init(
- name = "ShoppingCartProjection",
- numberOfInstances = sliceRanges.size,
- behaviorFactory = i => ProjectionBehavior(projection(sliceRanges(i))),
- stopMessage = ProjectionBehavior.Stop)
- }
- // #initProjections
- }
-
- // #sourceProvider
- import org.apache.pekko
- import pekko.projection.eventsourced.scaladsl.EventSourcedProvider
- import pekko.persistence.r2dbc.query.scaladsl.R2dbcReadJournal
- import pekko.projection.scaladsl.SourceProvider
-
- // Slit the slices into 4 ranges
- val numberOfSliceRanges: Int = 4
- val sliceRanges = EventSourcedProvider.sliceRanges(system, R2dbcReadJournal.Identifier, numberOfSliceRanges)
-
- // Example of using the first slice range
- val minSlice: Int = sliceRanges.head.min
- val maxSlice: Int = sliceRanges.head.max
- val entityType: String = ShoppingCart.EntityKey.name
-
- val sourceProvider: SourceProvider[Offset, EventEnvelope[ShoppingCart.Event]] =
- EventSourcedProvider
- .eventsBySlices[ShoppingCart.Event](
- system,
- readJournalPluginId = R2dbcReadJournal.Identifier,
- entityType,
- minSlice,
- maxSlice)
- // #sourceProvider
-
- object IllustrateExactlyOnce {
- // #exactlyOnce
- import org.apache.pekko
- import pekko.projection.r2dbc.scaladsl.R2dbcProjection
- import pekko.projection.ProjectionId
-
- val projectionId = ProjectionId("ShoppingCarts", s"carts-$minSlice-$maxSlice")
-
- val projection =
- R2dbcProjection
- .exactlyOnce(projectionId, settings = None, sourceProvider, handler = () => new ShoppingCartHandler)
- // #exactlyOnce
- }
-
- object IllustrateAtLeastOnce {
- // #atLeastOnce
- import org.apache.pekko
- import pekko.projection.r2dbc.scaladsl.R2dbcProjection
- import pekko.projection.ProjectionId
-
- val projectionId = ProjectionId("ShoppingCarts", s"carts-$minSlice-$maxSlice")
-
- val projection =
- R2dbcProjection
- .atLeastOnce(projectionId, settings = None, sourceProvider, handler = () => new ShoppingCartHandler)
- .withSaveOffset(afterEnvelopes = 100, afterDuration = 500.millis)
- // #atLeastOnce
- }
-
- object IllustrateGrouped {
- // #grouped
- import org.apache.pekko
- import pekko.projection.r2dbc.scaladsl.R2dbcProjection
- import pekko.projection.ProjectionId
-
- val projectionId = ProjectionId("ShoppingCarts", s"carts-$minSlice-$maxSlice")
-
- val projection =
- R2dbcProjection
- .groupedWithin(projectionId, settings = None, sourceProvider, handler = () => new GroupedShoppingCartHandler)
- .withGroup(groupAfterEnvelopes = 20, groupAfterDuration = 500.millis)
- // #grouped
- }
-
-}
diff --git a/docs/src/test/scala/docs/home/projection/RuntimePluginConfigCompatibilityExample.scala b/docs/src/test/scala/docs/home/projection/RuntimePluginConfigCompatibilityExample.scala
deleted file mode 100644
index bcc5fc1..0000000
--- a/docs/src/test/scala/docs/home/projection/RuntimePluginConfigCompatibilityExample.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package docs.home.projection
-
-import org.apache.pekko
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.query.Offset
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.projection.ProjectionId
-import pekko.projection.eventsourced.scaladsl.EventSourcedProvider
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.scaladsl.AtLeastOnceProjection
-import com.typesafe.config.ConfigFactory
-import docs.home.RuntimePluginConfigExample.EventSourcedBehaviorForDatabase
-import docs.home.RuntimePluginConfigExample._
-
-object RuntimePluginConfigCompatibilityExample {
-
- implicit val system: ActorSystem[_] = ???
- val entityType: String = ???
- val projectionId: ProjectionId = ???
- val sliceRange: Range = ???
-
- // #runtime-plugin-config-compatibility
- def atLeastOnceProjection(
- eventSourcedBehavior: EventSourcedBehaviorForDatabase,
- sliceRange: Range
- ): AtLeastOnceProjection[Offset, EventEnvelope[Event]] = {
- val sourceProvider = EventSourcedProvider
- .eventsBySlices[Event](
- system = system,
- readJournalPluginId = s"${eventSourcedBehavior.configKey}.query",
- readJournalConfig = eventSourcedBehavior.config,
- entityType = entityType,
- minSlice = sliceRange.min,
- maxSlice = sliceRange.max)
- val projectionConfig = ConfigFactory
- .load(
- eventSourcedBehavior.config
- .withFallback(
- ConfigFactory
- .parseString(
- s"""
- pekko.projection.r2dbc.use-connection-factory = "${eventSourcedBehavior.configKey}.connection-factory"
- """
- )
- )
- )
- R2dbcProjection
- .atLeastOnce(
- projectionId = projectionId,
- config = projectionConfig,
- settings = None,
- sourceProvider = sourceProvider,
- handler = () => ???
- )
- }
-
- atLeastOnceProjection(eventSourcedBehaviorForDatabase1, sliceRange)
- atLeastOnceProjection(eventSourcedBehaviorForDatabase2, sliceRange)
- // #runtime-plugin-config-compatibility
-}
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 9af221c..ecf61ae 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -19,9 +19,8 @@ object Dependencies {
val PekkoVersion = PekkoCoreDependency.version
val PekkoVersionInDocs = PekkoCoreDependency.default.link
val PekkoPersistenceJdbcVersion = PekkoPersistenceJdbcDependency.version
- val PekkoPersistenceR2dbcVersionInDocs = "1.0"
- val PekkoProjectionVersion = PekkoProjectionDependency.version
- val PekkoProjectionVersionInDocs = PekkoProjectionDependency.default.link
+ val PekkoPersistenceR2dbcVersionInDocs = "current"
+ val PekkoProjectionVersionInDocs = "current"
object Compile {
val pekkoActorTyped = "org.apache.pekko" %% "pekko-actor-typed" % PekkoVersion
@@ -29,8 +28,6 @@ object Dependencies {
val pekkoPersistence = "org.apache.pekko" %% "pekko-persistence-typed" % PekkoVersion
val pekkoPersistenceQuery = "org.apache.pekko" %% "pekko-persistence-query" % PekkoVersion
- val pekkoProjectionCore = "org.apache.pekko" %% "pekko-projection-core" % PekkoProjectionVersion
-
val r2dbcSpi = "io.r2dbc" % "r2dbc-spi" % "1.0.0.RELEASE"
val r2dbcPool = "io.r2dbc" % "r2dbc-pool" % "1.0.2.RELEASE"
val r2dbcPostgres = "org.postgresql" % "r2dbc-postgresql" % "1.1.1.RELEASE"
@@ -54,12 +51,6 @@ object Dependencies {
val pekkoStreamTyped = "org.apache.pekko" %% "pekko-stream-typed" % PekkoVersion % Test
val pekkoTestkit = "org.apache.pekko" %% "pekko-testkit" % PekkoVersion % Test
- val pekkoProjectionEventSourced =
- "org.apache.pekko" %% "pekko-projection-eventsourced" % PekkoProjectionVersion % Test
- val pekkoProjectionDurableState =
- "org.apache.pekko" %% "pekko-projection-durable-state" % PekkoProjectionVersion % Test
- val pekkoProjectionTestKit = "org.apache.pekko" %% "pekko-projection-testkit" % PekkoProjectionVersion % Test
-
val postgresql = "org.postgresql" % "postgresql" % "42.7.11" % Test
val logback = "ch.qos.logback" % "logback-classic" % "1.5.32" % Test
@@ -85,22 +76,6 @@ object Dependencies {
TestDeps.logback,
TestDeps.scalaTest)
- val projection = Seq(
- pekkoPersistenceQuery,
- r2dbcSpi,
- r2dbcPool,
- r2dbcPostgres % "provided,test",
- r2dbcMysql % "provided,test",
- pekkoProjectionCore,
- TestDeps.pekkoProjectionEventSourced,
- TestDeps.pekkoProjectionDurableState,
- TestDeps.pekkoProjectionTestKit,
- TestDeps.pekkoActorTestkitTyped,
- TestDeps.pekkoJackson,
- TestDeps.pekkoStreamTestkit,
- TestDeps.logback,
- TestDeps.scalaTest)
-
val migration = Seq(
"org.apache.pekko" %% "pekko-persistence-jdbc" % PekkoPersistenceJdbcVersion % Test,
TestDeps.postgresql,
@@ -109,8 +84,6 @@ object Dependencies {
val docs = Seq(
TestDeps.pekkoPersistenceTyped,
- TestDeps.pekkoProjectionEventSourced,
- TestDeps.pekkoProjectionDurableState,
TestDeps.pekkoShardingTyped)
val pekkoTestDependencyOverrides = Seq(
diff --git a/project/project-info.conf b/project/project-info.conf
index 8b35737..a116cbc 100644
--- a/project/project-info.conf
+++ b/project/project-info.conf
@@ -29,8 +29,4 @@ project-info {
title: "Pekko Persistence R2DBC"
jpms-name: "pekko.persistence.r2dbc"
}
- projection: ${project-info.shared-info} {
- title: "Pekko Projections with R2DBC"
- jpms-name: "pekko.projection.r2dbc"
- }
}
diff --git a/projection/src/main/mima-filters/1.0.x.backwards.excludes/r2dbcprojectionsettings.excludes b/projection/src/main/mima-filters/1.0.x.backwards.excludes/r2dbcprojectionsettings.excludes
deleted file mode 100644
index 8b28128..0000000
--- a/projection/src/main/mima-filters/1.0.x.backwards.excludes/r2dbcprojectionsettings.excludes
+++ /dev/null
@@ -1,51 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Converting case class to class
-ProblemFilters.exclude[MissingTypesProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.this")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.canEqual")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.productArity")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.productPrefix")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.productElement")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.productElementName")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.productElementNames")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.productIterator")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$1")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$2")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$3")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$4")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$5")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$6")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$7")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$8")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$9")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.copy$default$10")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._1")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._2")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._3")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._4")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._5")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._6")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._7")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._8")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._9")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings._10")
-ProblemFilters.exclude[MissingTypesProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings$")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.unapply")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.R2dbcProjectionSettings.fromProduct")
diff --git a/projection/src/main/mima-filters/1.0.x.backwards.excludes/r2dbcsession.excludes b/projection/src/main/mima-filters/1.0.x.backwards.excludes/r2dbcsession.excludes
deleted file mode 100644
index de0fbe3..0000000
--- a/projection/src/main/mima-filters/1.0.x.backwards.excludes/r2dbcsession.excludes
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Caused by https://github.com/apache/pekko-persistence-r2dbc/pull/101
-ProblemFilters.exclude[IncompatibleSignatureProblem]("org.apache.pekko.projection.r2dbc.javadsl.R2dbcSession.updateOne")
-ProblemFilters.exclude[IncompatibleSignatureProblem]("org.apache.pekko.projection.r2dbc.javadsl.R2dbcSession.update")
diff --git a/projection/src/main/mima-filters/2.0.x.backwards.excludes/envelope-origin.excludes b/projection/src/main/mima-filters/2.0.x.backwards.excludes/envelope-origin.excludes
deleted file mode 100644
index b8338e1..0000000
--- a/projection/src/main/mima-filters/2.0.x.backwards.excludes/envelope-origin.excludes
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Caused by porting https://github.com/akka/akka-persistence-r2dbc/pull/348
-ProblemFilters.exclude[MissingTypesProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore$RecordWithOffset$")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#RecordWithOffset.copy")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#RecordWithOffset.this")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#RecordWithOffset.apply")
-ProblemFilters.exclude[IncompatibleSignatureProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#RecordWithOffset.unapply")
-ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#RecordWithOffset.copy*")
-ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.pekko.projection.r2dbc.internal.R2dbcOffsetStore#RecordWithOffset.envelopeLoaded")
diff --git a/projection/src/main/resources/reference.conf b/projection/src/main/resources/reference.conf
deleted file mode 100644
index 67ffe1d..0000000
--- a/projection/src/main/resources/reference.conf
+++ /dev/null
@@ -1,52 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-# This defines the default configuration for pekko-projection-r2dbc.
-# Make your edits/overrides in your application.conf.
-
-//#projection-config
-pekko.projection.r2dbc {
- # postgres, yugabyte or mysql
- dialect = ${pekko.persistence.r2dbc.dialect}
-
- offset-store {
- # set this to your database schema if applicable, empty by default
- schema = ""
- # the database table name for the offset store,
- # can be set to "" if only timestamp offsets are used and table is not created
- offset-table = "projection_offset_store"
-
- # the database table name for the offset store
- timestamp-offset-table = "projection_timestamp_offset_store"
-
- # the database table name for the projection manangement data
- management-table = "projection_management"
-
- # The offset store will keep track of persistence ids and sequence numbers
- # within this time window from latest offset.
- time-window = 5 minutes
-
- # Keep this number of entries. Don't evict old entries until this threshold
- # has been reached.
- keep-number-of-entries = 10000
-
- # Remove old entries outside the time-window from the offset store memory
- # with this frequency.
- evict-interval = 10 seconds
-
- # Remove old entries outside the time-window from the offset store database
- # with this frequency.
- delete-interval = 1 minute
- }
-
- # By default it shares connection-factory with pekko-persistence-r2dbc (write side),
- # i.e. same connection pool. To use a separate pool for projections this can be
- # set to another config path that defines the same kind of config as
- # pekko.persistence.r2dbc.connection-factory.
- use-connection-factory = "pekko.persistence.r2dbc.connection-factory"
-
- # Logs database calls that take longer than this duration at INFO level.
- # Set to "off" to disable this logging.
- # Set to 0 to log all calls.
- log-db-calls-exceeding = 300 ms
-}
-//#projection-config
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSettings.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSettings.scala
deleted file mode 100644
index 173005a..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSettings.scala
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.time.{ Duration => JDuration }
-import java.util.Locale
-
-import scala.concurrent.duration._
-import scala.jdk.DurationConverters._
-import scala.util.hashing.MurmurHash3
-
-import com.typesafe.config.Config
-import org.apache.pekko
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.r2dbc.Dialect
-
-object R2dbcProjectionSettings {
-
- val DefaultConfigPath = "pekko.projection.r2dbc"
-
- def apply(config: Config): R2dbcProjectionSettings = {
- val logDbCallsExceeding: FiniteDuration =
- config.getString("log-db-calls-exceeding").toLowerCase(Locale.ROOT) match {
- case "off" => -1.millis
- case _ => config.getDuration("log-db-calls-exceeding").toScala
- }
-
- new R2dbcProjectionSettings(
- dialect = Dialect.fromString(config.getString("dialect")),
- schema = Option(config.getString("offset-store.schema")).filterNot(_.trim.isEmpty),
- offsetTable = config.getString("offset-store.offset-table"),
- timestampOffsetTable = config.getString("offset-store.timestamp-offset-table"),
- managementTable = config.getString("offset-store.management-table"),
- useConnectionFactory = config.getString("use-connection-factory"),
- timeWindow = config.getDuration("offset-store.time-window"),
- keepNumberOfEntries = config.getInt("offset-store.keep-number-of-entries"),
- evictInterval = config.getDuration("offset-store.evict-interval"),
- deleteInterval = config.getDuration("offset-store.delete-interval"),
- logDbCallsExceeding
- )
- }
-
- def apply(system: ActorSystem[_]): R2dbcProjectionSettings =
- apply(system.settings.config.getConfig(DefaultConfigPath))
-
- /** @since 2.0.0 */
- def apply(config: Config, system: ActorSystem[_]): R2dbcProjectionSettings =
- apply(config.withFallback(system.settings.config.getConfig(DefaultConfigPath)))
-
- def apply(
- schema: Option[String],
- offsetTable: String,
- timestampOffsetTable: String,
- managementTable: String,
- useConnectionFactory: String,
- timeWindow: JDuration,
- keepNumberOfEntries: Int,
- evictInterval: JDuration,
- deleteInterval: JDuration,
- logDbCallsExceeding: FiniteDuration
- ): R2dbcProjectionSettings = new R2dbcProjectionSettings(
- Dialect.Postgres,
- schema,
- offsetTable,
- timestampOffsetTable,
- managementTable,
- useConnectionFactory,
- timeWindow,
- keepNumberOfEntries,
- evictInterval,
- deleteInterval,
- logDbCallsExceeding
- )
-}
-
-final class R2dbcProjectionSettings private (
- val dialect: Dialect,
- val schema: Option[String],
- val offsetTable: String,
- val timestampOffsetTable: String,
- val managementTable: String,
- val useConnectionFactory: String,
- val timeWindow: JDuration,
- val keepNumberOfEntries: Int,
- val evictInterval: JDuration,
- val deleteInterval: JDuration,
- val logDbCallsExceeding: FiniteDuration
-) extends Serializable {
-
- override def toString: String =
- s"R2dbcProjectionSettings($dialect, $schema, $offsetTable, $timestampOffsetTable, $managementTable, " +
- s"$useConnectionFactory, $timeWindow, $keepNumberOfEntries, $evictInterval, $deleteInterval, $logDbCallsExceeding)"
-
- override def equals(other: Any): Boolean =
- other match {
- case that: R2dbcProjectionSettings =>
- dialect == that.dialect && schema == that.schema &&
- offsetTable == that.offsetTable && timestampOffsetTable == that.timestampOffsetTable &&
- managementTable == that.managementTable && useConnectionFactory == that.useConnectionFactory &&
- timeWindow == that.timeWindow && keepNumberOfEntries == that.keepNumberOfEntries &&
- evictInterval == that.evictInterval && deleteInterval == that.deleteInterval &&
- logDbCallsExceeding == that.logDbCallsExceeding
- case _ => false
- }
-
- override def hashCode(): Int = {
- val values = Seq(
- dialect,
- schema,
- offsetTable,
- timestampOffsetTable,
- managementTable,
- useConnectionFactory,
- timeWindow,
- keepNumberOfEntries,
- evictInterval,
- deleteInterval,
- logDbCallsExceeding
- )
- val h = values.foldLeft(MurmurHash3.productSeed) { case (h, value) =>
- MurmurHash3.mix(h, value.##)
- }
- MurmurHash3.finalizeHash(h, values.size)
- }
-
- private[this] def copy(
- dialect: Dialect = dialect,
- schema: Option[String] = schema,
- offsetTable: String = offsetTable,
- timestampOffsetTable: String = timestampOffsetTable,
- managementTable: String = managementTable,
- useConnectionFactory: String = useConnectionFactory,
- timeWindow: JDuration = timeWindow,
- keepNumberOfEntries: Int = keepNumberOfEntries,
- evictInterval: JDuration = evictInterval,
- deleteInterval: JDuration = deleteInterval,
- logDbCallsExceeding: FiniteDuration = logDbCallsExceeding
- ): R2dbcProjectionSettings =
- new R2dbcProjectionSettings(
- dialect,
- schema,
- offsetTable,
- timestampOffsetTable,
- managementTable,
- useConnectionFactory,
- timeWindow,
- keepNumberOfEntries,
- evictInterval,
- deleteInterval,
- logDbCallsExceeding
- )
-
- def withDialect(dialect: Dialect): R2dbcProjectionSettings =
- copy(dialect = dialect)
-
- def withSchema(schema: Option[String]): R2dbcProjectionSettings =
- copy(schema = schema)
-
- def withOffsetTable(offsetTable: String): R2dbcProjectionSettings =
- copy(offsetTable = offsetTable)
-
- def withTimestampOffsetTable(timestampOffsetTable: String): R2dbcProjectionSettings =
- copy(timestampOffsetTable = timestampOffsetTable)
-
- def withManagementTable(managementTable: String): R2dbcProjectionSettings =
- copy(managementTable = managementTable)
-
- def withUseConnectionFactory(useConnectionFactory: String): R2dbcProjectionSettings =
- copy(useConnectionFactory = useConnectionFactory)
-
- def withTimeWindow(timeWindow: JDuration): R2dbcProjectionSettings =
- copy(timeWindow = timeWindow)
-
- def withKeepNumberOfEntries(keepNumberOfEntries: Int): R2dbcProjectionSettings =
- copy(keepNumberOfEntries = keepNumberOfEntries)
-
- def withEvictInterval(evictInterval: JDuration): R2dbcProjectionSettings =
- copy(evictInterval = evictInterval)
-
- def withDeleteInterval(deleteInterval: JDuration): R2dbcProjectionSettings =
- copy(deleteInterval = deleteInterval)
-
- def withLogDbCallsExceeding(logDbCallsExceeding: FiniteDuration): R2dbcProjectionSettings =
- copy(logDbCallsExceeding = logDbCallsExceeding)
-
- val offsetTableWithSchema: String = schema.map(_ + ".").getOrElse("") + offsetTable
- val timestampOffsetTableWithSchema: String = schema.map(_ + ".").getOrElse("") + timestampOffsetTable
- val managementTableWithSchema: String = schema.map(_ + ".").getOrElse("") + managementTable
-
- def isOffsetTableDefined: Boolean = offsetTable.nonEmpty
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/BySliceSourceProviderAdapter.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/BySliceSourceProviderAdapter.scala
deleted file mode 100644
index d0b9383..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/BySliceSourceProviderAdapter.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.internal
-
-import java.time.Instant
-import java.util.Optional
-import java.util.concurrent.CompletionStage
-import java.util.function.Supplier
-
-import scala.concurrent.{ ExecutionContext, Future }
-import scala.jdk.CollectionConverters._
-import scala.jdk.FutureConverters._
-import scala.jdk.OptionConverters._
-
-import org.apache.pekko
-import pekko.NotUsed
-import pekko.annotation.InternalApi
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
-import pekko.persistence.query.typed.scaladsl.LoadEventQuery
-import pekko.projection.javadsl
-import pekko.projection.scaladsl
-import pekko.projection.BySlicesSourceProvider
-import pekko.stream.scaladsl.Source
-
-/**
- * INTERNAL API: Adapter from javadsl.SourceProvider to scaladsl.SourceProvider
- */
-@InternalApi private[projection] class BySliceSourceProviderAdapter[Offset, Envelope](
- delegate: javadsl.SourceProvider[Offset, Envelope])
- extends scaladsl.SourceProvider[Offset, Envelope]
- with BySlicesSourceProvider
- with EventTimestampQuery
- with LoadEventQuery {
-
- def source(offset: () => Future[Option[Offset]]): Future[Source[Envelope, NotUsed]] = {
- // the parasitic context is used to convert the Optional to Option and a java streams Source to a scala Source,
- // it _should_ not be used for the blocking operation of getting offsets themselves
- val ec = ExecutionContext.parasitic
- val offsetAdapter = new Supplier[CompletionStage[Optional[Offset]]] {
- override def get(): CompletionStage[Optional[Offset]] = offset().map(_.toJava)(ec).asJava
- }
- delegate.source(offsetAdapter).asScala.map(_.asScala)(ec)
- }
-
- def extractOffset(envelope: Envelope): Offset = delegate.extractOffset(envelope)
-
- def extractCreationTime(envelope: Envelope): Long = delegate.extractCreationTime(envelope)
-
- override def minSlice: Int =
- delegate.asInstanceOf[BySlicesSourceProvider].minSlice
-
- override def maxSlice: Int =
- delegate.asInstanceOf[BySlicesSourceProvider].maxSlice
-
- override def timestampOf(persistenceId: String, sequenceNr: Long): Future[Option[Instant]] =
- delegate match {
- case timestampQuery: pekko.persistence.query.typed.javadsl.EventTimestampQuery =>
- timestampQuery.timestampOf(persistenceId, sequenceNr).asScala.map(_.toScala)(ExecutionContext.parasitic)
- case _ =>
- Future.failed(
- new IllegalArgumentException(
- s"Expected SourceProvider [${delegate.getClass.getName}] to implement " +
- "EventTimestampQuery when TimestampOffset is used."))
- }
-
- override def loadEnvelope[Event](persistenceId: String, sequenceNr: Long): Future[EventEnvelope[Event]] =
- delegate match {
- case timestampQuery: pekko.persistence.query.typed.javadsl.LoadEventQuery =>
- timestampQuery.loadEnvelope[Event](persistenceId, sequenceNr).asScala
- case _ =>
- Future.failed(
- new IllegalArgumentException(
- s"Expected SourceProvider [${delegate.getClass.getName}] to implement " +
- "EventTimestampQuery when LoadEventQuery is used."))
- }
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/OffsetPidSeqNr.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/OffsetPidSeqNr.scala
deleted file mode 100644
index 68edcca..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/OffsetPidSeqNr.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2022 - 2023 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.internal
-
-import org.apache.pekko.annotation.InternalApi
-
-/**
- * INTERNAL API
- */
-@InternalApi private[pekko] object OffsetPidSeqNr {
- def apply(offset: Any, pid: String, seqNr: Long): OffsetPidSeqNr =
- new OffsetPidSeqNr(offset, Some(pid -> seqNr))
-
- def apply(offset: Any): OffsetPidSeqNr =
- new OffsetPidSeqNr(offset, None)
-}
-
-/**
- * INTERNAL API
- */
-@InternalApi private[pekko] final case class OffsetPidSeqNr(offset: Any, pidSeqNr: Option[(String, Long)])
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcHandlerAdapter.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcHandlerAdapter.scala
deleted file mode 100644
index c64a1ed..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcHandlerAdapter.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.internal
-
-import scala.collection.immutable
-import scala.concurrent.Future
-import scala.jdk.CollectionConverters._
-import scala.jdk.FutureConverters._
-
-import org.apache.pekko
-import pekko.Done
-import pekko.annotation.InternalApi
-import pekko.projection.r2dbc.javadsl
-import pekko.projection.r2dbc.javadsl.R2dbcSession
-import pekko.projection.r2dbc.scaladsl
-
-/**
- * INTERNAL API: Adapter from javadsl.R2dbcHandler to scaladsl.R2dbcHandler
- */
-@InternalApi private[projection] class R2dbcHandlerAdapter[Envelope](delegate: javadsl.R2dbcHandler[Envelope])
- extends scaladsl.R2dbcHandler[Envelope] {
-
- override def process(session: scaladsl.R2dbcSession, envelope: Envelope): Future[Done] = {
- delegate.process(new R2dbcSession(session.connection)(session.ec, session.system), envelope).asScala
- }
-
- override def start(): Future[Done] =
- delegate.start().asScala
-
- override def stop(): Future[Done] =
- delegate.stop().asScala
-
-}
-
-/**
- * INTERNAL API: Adapter from `javadsl.R2dbcHandler[java.util.List[Envelope]]` to
- * `scaladsl.R2dbcHandler[immutable.Seq[Envelope]]`
- */
-@InternalApi private[projection] class R2dbcGroupedHandlerAdapter[Envelope](
- delegate: javadsl.R2dbcHandler[java.util.List[Envelope]])
- extends scaladsl.R2dbcHandler[immutable.Seq[Envelope]] {
-
- override def process(session: scaladsl.R2dbcSession, envelopes: immutable.Seq[Envelope]): Future[Done] = {
- delegate.process(new R2dbcSession(session.connection)(session.ec, session.system), envelopes.asJava).asScala
- }
-
- override def start(): Future[Done] =
- delegate.start().asScala
-
- override def stop(): Future[Done] =
- delegate.stop().asScala
-
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcOffsetStore.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcOffsetStore.scala
deleted file mode 100644
index 061b9da..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcOffsetStore.scala
+++ /dev/null
@@ -1,1045 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021-2022 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.internal
-
-import java.time.Clock
-import java.time.Instant
-import java.time.{ Duration => JDuration }
-import java.util.UUID
-import java.util.concurrent.atomic.AtomicBoolean
-import java.util.concurrent.atomic.AtomicReference
-
-import scala.annotation.tailrec
-import scala.collection.immutable
-import scala.concurrent.{ ExecutionContext, Future }
-
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.typed.ActorSystem
-import pekko.actor.typed.scaladsl.LoggerOps
-import pekko.annotation.InternalApi
-import pekko.persistence.Persistence
-import pekko.persistence.query.DeletedDurableState
-import pekko.persistence.query.DurableStateChange
-import pekko.persistence.query.Offset
-import pekko.persistence.query.TimestampOffset
-import pekko.persistence.query.UpdatedDurableState
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
-import pekko.persistence.r2dbc.Dialect
-import pekko.persistence.r2dbc.internal.EnvelopeOrigin
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
-import pekko.persistence.typed.PersistenceId
-import pekko.projection.BySlicesSourceProvider
-import pekko.projection.MergeableOffset
-import pekko.projection.ProjectionId
-import pekko.projection.internal.ManagementState
-import pekko.projection.internal.OffsetSerialization
-import pekko.projection.internal.OffsetSerialization.MultipleOffsets
-import pekko.projection.internal.OffsetSerialization.SingleOffset
-import pekko.projection.r2dbc.R2dbcProjectionSettings
-import pekko.projection.r2dbc.internal.mysql.MySQLR2dbcOffsetStore
-import io.r2dbc.spi.Connection
-import io.r2dbc.spi.Statement
-import org.slf4j.LoggerFactory
-
-object R2dbcOffsetStore {
- type SeqNr = Long
- type Pid = String
-
- final case class Record(pid: Pid, seqNr: SeqNr, timestamp: Instant)
- final case class RecordWithOffset(
- record: Record,
- offset: TimestampOffset,
- strictSeqNr: Boolean,
- fromBacktracking: Boolean,
- fromPubSub: Boolean)
-
- object State {
- val empty: State = State(Map.empty, Vector.empty, Instant.EPOCH)
-
- def apply(records: immutable.IndexedSeq[Record]): State = {
- if (records.isEmpty) empty
- else empty.add(records)
- }
- }
-
- final case class State(byPid: Map[Pid, Record], latest: immutable.IndexedSeq[Record], oldestTimestamp: Instant) {
- def size: Int = byPid.size
-
- def latestTimestamp: Instant =
- if (latest.isEmpty) Instant.EPOCH
- else latest.head.timestamp
-
- def latestOffset: Option[TimestampOffset] = {
- if (latest.isEmpty)
- None
- else
- Some(TimestampOffset(latestTimestamp, latest.map(r => r.pid -> r.seqNr).toMap))
- }
-
- def add(records: immutable.IndexedSeq[Record]): State = {
- records.foldLeft(this) { case (acc, r) =>
- val newByPid =
- acc.byPid.get(r.pid) match {
- case Some(existingRecord) =>
- if (r.seqNr > existingRecord.seqNr)
- acc.byPid.updated(r.pid, r)
- else
- acc.byPid // older or same seqNr
- case None =>
- acc.byPid.updated(r.pid, r)
- }
-
- val latestTimestamp = acc.latestTimestamp
- val newLatest =
- if (r.timestamp.isAfter(latestTimestamp)) {
- Vector(r)
- } else if (r.timestamp == latestTimestamp) {
- acc.latest.find(_.pid == r.pid) match {
- case None => acc.latest :+ r
- case Some(existingRecord) =>
- // keep highest seqNr
- if (r.seqNr >= existingRecord.seqNr)
- acc.latest.filterNot(_.pid == r.pid) :+ r
- else
- acc.latest
- }
- } else {
- acc.latest // older than existing latest, keep existing latest
- }
- val newOldestTimestamp =
- if (acc.oldestTimestamp == Instant.EPOCH)
- r.timestamp // first record
- else if (r.timestamp.isBefore(acc.oldestTimestamp))
- r.timestamp
- else
- acc.oldestTimestamp // this is the normal case
-
- acc.copy(byPid = newByPid, latest = newLatest, oldestTimestamp = newOldestTimestamp)
- }
- }
-
- def isDuplicate(record: Record): Boolean = {
- byPid.get(record.pid) match {
- case Some(existingRecord) => record.seqNr <= existingRecord.seqNr
- case None => false
- }
- }
-
- def window: JDuration =
- JDuration.between(oldestTimestamp, latestTimestamp)
-
- def evict(until: Instant, keepNumberOfEntries: Int): State = {
- if (oldestTimestamp.isBefore(until) && size > keepNumberOfEntries) {
- val sorted = byPid.valuesIterator.toVector.sortBy(_.timestamp)
- State(
- sorted
- .take(size - keepNumberOfEntries)
- .filterNot(_.timestamp.isBefore(until)) ++ sorted.takeRight(keepNumberOfEntries))
- } else
- this
- }
- }
-
- val FutureDone: Future[Done] = Future.successful(Done)
- val FutureTrue: Future[Boolean] = Future.successful(true)
- val FutureFalse: Future[Boolean] = Future.successful(false)
-
- def fromConfig(
- projectionId: ProjectionId,
- sourceProvider: Option[BySlicesSourceProvider],
- system: ActorSystem[_],
- settings: R2dbcProjectionSettings,
- r2dbcExecutor: R2dbcExecutor,
- clock: Clock = Clock.systemUTC()
- ): R2dbcOffsetStore = {
- settings.dialect match {
- case Dialect.Postgres | Dialect.Yugabyte =>
- new R2dbcOffsetStore(projectionId, sourceProvider, system, settings, r2dbcExecutor, clock)
- case Dialect.MySQL =>
- new MySQLR2dbcOffsetStore(projectionId, sourceProvider, system, settings, r2dbcExecutor, clock)
- }
- }
-}
-
-/**
- * INTERNAL API
- */
-@InternalApi
-private[projection] class R2dbcOffsetStore(
- projectionId: ProjectionId,
- sourceProvider: Option[BySlicesSourceProvider],
- system: ActorSystem[_],
- settings: R2dbcProjectionSettings,
- r2dbcExecutor: R2dbcExecutor,
- clock: Clock = Clock.systemUTC()) {
-
- import R2dbcOffsetStore._
-
- implicit protected val dialect: Dialect = settings.dialect
- protected lazy val timestampSql: String = "transaction_timestamp()"
-
- // FIXME include projectionId in all log messages
- private val logger = LoggerFactory.getLogger(this.getClass)
-
- private val evictWindow = settings.timeWindow.plus(settings.evictInterval)
- private val evictKeepNumberOfEntriesThreshold = (settings.keepNumberOfEntries * 1.1).toInt
-
- private val offsetSerialization = new OffsetSerialization(system)
- import offsetSerialization.fromStorageRepresentation
- import offsetSerialization.toStorageRepresentation
-
- private val timestampOffsetTable = settings.timestampOffsetTableWithSchema
- protected val offsetTable = settings.offsetTableWithSchema
- protected val managementTable = settings.managementTableWithSchema
-
- private[projection] implicit val executionContext: ExecutionContext = system.executionContext
-
- private val persistenceExt = Persistence(system)
-
- private val selectTimestampOffsetSql: String = sql"""
- SELECT persistence_id, seq_nr, timestamp_offset
- FROM $timestampOffsetTable WHERE slice BETWEEN ? AND ? AND projection_name = ?"""
-
- private val insertTimestampOffsetSql: String = sql"""
- INSERT INTO $timestampOffsetTable
- (projection_name, projection_key, slice, persistence_id, seq_nr, timestamp_offset, timestamp_consumed)
- VALUES (?,?,?,?,?,?, $timestampSql)"""
-
- // delete less than a timestamp
- private val deleteOldTimestampOffsetSql: String =
- sql"DELETE FROM $timestampOffsetTable WHERE slice BETWEEN ? AND ? AND projection_name = ? AND timestamp_offset < ?"
-
- // delete greater than or equal a timestamp
- private val deleteNewTimestampOffsetSql: String =
- sql"DELETE FROM $timestampOffsetTable WHERE slice BETWEEN ? AND ? AND projection_name = ? AND timestamp_offset >= ?"
-
- private val clearTimestampOffsetSql: String =
- sql"DELETE FROM $timestampOffsetTable WHERE slice BETWEEN ? AND ? AND projection_name = ?"
-
- private val selectOffsetSql: String =
- sql"SELECT projection_key, current_offset, manifest, mergeable FROM $offsetTable WHERE projection_name = ?"
-
- protected val upsertOffsetSql: String = sql"""
- INSERT INTO $offsetTable
- (projection_name, projection_key, current_offset, manifest, mergeable, last_updated)
- VALUES (?,?,?,?,?,?)
- ON CONFLICT (projection_name, projection_key)
- DO UPDATE SET
- current_offset = excluded.current_offset,
- manifest = excluded.manifest,
- mergeable = excluded.mergeable,
- last_updated = excluded.last_updated"""
-
- private val clearOffsetSql: String =
- sql"DELETE FROM $offsetTable WHERE projection_name = ? AND projection_key = ?"
-
- private val readManagementStateSql = sql"""
- SELECT paused FROM $managementTable WHERE
- projection_name = ? AND
- projection_key = ? """
-
- val updateManagementStateSql: String = sql"""
- INSERT INTO $managementTable
- (projection_name, projection_key, paused, last_updated)
- VALUES (?,?,?,?)
- ON CONFLICT (projection_name, projection_key)
- DO UPDATE SET
- paused = excluded.paused,
- last_updated = excluded.last_updated"""
-
- // The OffsetStore instance is used by a single projectionId and there shouldn't be any concurrent
- // calls to methods that access the `state`. To detect any violations of that concurrency assumption
- // we use AtomicReference and fail if the CAS fails.
- private val state = new AtomicReference(State.empty)
-
- // Transient state of inflight pid -> seqNr (before they have been stored and included in `state`), which is
- // needed for at-least-once or other projections where the offset is saved afterwards. Not needed for exactly-once.
- // This can be updated concurrently with CAS retries.
- private val inflight = new AtomicReference(Map.empty[Pid, SeqNr])
-
- // To avoid delete requests when no new offsets have been stored since previous delete
- private val idle = new AtomicBoolean(false)
-
- system.scheduler.scheduleWithFixedDelay(
- settings.deleteInterval,
- settings.deleteInterval,
- () => deleteOldTimestampOffsets(),
- system.executionContext)
-
- private def timestampOffsetBySlicesSourceProvider: BySlicesSourceProvider =
- sourceProvider match {
- case Some(provider) => provider
- case None =>
- throw new IllegalArgumentException(
- "Expected BySlicesSourceProvider to be defined when TimestampOffset is used.")
- }
-
- private def timestampOf(persistenceId: String, sequenceNr: Long): Future[Option[Instant]] = {
- timestampOffsetBySlicesSourceProvider match {
- case timestampQuery: EventTimestampQuery =>
- timestampQuery.timestampOf(persistenceId, sequenceNr)
- case timestampQuery: pekko.persistence.query.typed.javadsl.EventTimestampQuery =>
- import scala.jdk.FutureConverters._
- import scala.jdk.OptionConverters._
- timestampQuery.timestampOf(persistenceId, sequenceNr).asScala.map(_.toScala)
- case _ =>
- throw new IllegalArgumentException(
- "Expected BySlicesSourceProvider to implement EventTimestampQuery when TimestampOffset is used.")
- }
- }
-
- def getState(): State =
- state.get()
-
- def getInflight(): Map[Pid, SeqNr] =
- inflight.get()
-
- def getOffset[Offset](): Future[Option[Offset]] = {
- getState().latestOffset match {
- case Some(t) => Future.successful(Some(t.asInstanceOf[Offset]))
- case None => readOffset()
- }
- }
-
- def readOffset[Offset](): Future[Option[Offset]] = {
- // look for TimestampOffset first since that is used by pekko-persistence-r2dbc,
- // and then fall back to the other more primitive offset types
- sourceProvider match {
- case Some(provider) =>
- readTimestampOffset().flatMap {
- case Some(t) => Future.successful(Some(t.asInstanceOf[Offset]))
- case None => readPrimitiveOffset()
- }
- case None =>
- readPrimitiveOffset()
- }
- }
-
- private def readTimestampOffset(): Future[Option[TimestampOffset]] = {
- idle.set(false)
- val oldState = state.get()
-
- val (minSlice, maxSlice) = {
- sourceProvider match {
- case Some(provider) => (provider.minSlice, provider.maxSlice)
- case None => (0, persistenceExt.numberOfSlices - 1)
- }
- }
-
- val recordsFut = r2dbcExecutor.select("read timestamp offset")(
- conn => {
- logger.trace("reading timestamp offset for [{}]", projectionId)
- conn
- .createStatement(selectTimestampOffsetSql)
- .bind(0, minSlice)
- .bind(1, maxSlice)
- .bind(2, projectionId.name)
- },
- row => {
- val pid = row.get("persistence_id", classOf[String])
- val seqNr = row.get("seq_nr", classOf[java.lang.Long])
- val timestamp = row.get("timestamp_offset", classOf[Instant])
- Record(pid, seqNr, timestamp)
- })
- recordsFut.map { records =>
- val newState = State(records)
- logger.debug(
- "readTimestampOffset state with [{}] persistenceIds, oldest [{}], latest [{}]",
- newState.byPid.size: java.lang.Integer,
- newState.oldestTimestamp,
- newState.latestTimestamp)
- if (!state.compareAndSet(oldState, newState))
- throw new IllegalStateException("Unexpected concurrent modification of state from readOffset.")
- clearInflight()
- if (newState == State.empty) {
- None
- } else {
- newState.latestOffset
- }
- }
- }
-
- private def readPrimitiveOffset[Offset](): Future[Option[Offset]] = {
- if (settings.isOffsetTableDefined) {
- val singleOffsets = r2dbcExecutor.select("read offset")(
- conn => {
- logger.trace("reading offset for [{}]", projectionId)
- conn
- .createStatement(selectOffsetSql)
- .bind(0, projectionId.name)
- },
- row => {
- val offsetStr = row.get("current_offset", classOf[String])
- val manifest = row.get("manifest", classOf[String])
- val mergeable = row.get[java.lang.Boolean]("mergeable", classOf[java.lang.Boolean])
- val key = row.get("projection_key", classOf[String])
-
- val adaptedProjectionId = ProjectionId(projectionId.name, key)
- SingleOffset(adaptedProjectionId, manifest, offsetStr, mergeable)
- })
-
- singleOffsets.map { offsets =>
- val result =
- if (offsets.isEmpty) None
- else if (offsets.forall(_.mergeable)) {
- Some(
- fromStorageRepresentation[MergeableOffset[_], Offset](MultipleOffsets(offsets.toList))
- .asInstanceOf[Offset])
- } else {
- offsets.find(_.id == projectionId).map(fromStorageRepresentation[Offset, Offset])
- }
-
- logger.trace2("found offset [{}] for [{}]", result, projectionId)
-
- result
- }
- } else {
- Future.successful(None)
- }
- }
-
- /**
- * Like saveOffsetInTx, but in own transaction. Used by atLeastOnce.
- */
- def saveOffset(offset: OffsetPidSeqNr): Future[Done] = {
- r2dbcExecutor
- .withConnection("save offset") { conn =>
- saveOffsetInTx(conn, offset)
- }
- .map(_ => Done)(ExecutionContext.parasitic)
- }
-
- /**
- * This method is used together with the users' handler code and run in same transaction.
- */
- def saveOffsetInTx(conn: Connection, offset: OffsetPidSeqNr): Future[Done] = {
- offset match {
- case OffsetPidSeqNr(t: TimestampOffset, Some((pid, seqNr))) =>
- val record = Record(pid, seqNr, t.timestamp)
- saveTimestampOffsetInTx(conn, Vector(record))
- case OffsetPidSeqNr(_: TimestampOffset, None) =>
- throw new IllegalArgumentException("Required EventEnvelope or DurableStateChange for TimestampOffset.")
- case _ =>
- savePrimitiveOffsetInTx(conn, offset.offset)
- }
- }
-
- def saveOffsets(offsets: immutable.IndexedSeq[OffsetPidSeqNr]): Future[Done] = {
- r2dbcExecutor
- .withConnection("save offsets") { conn =>
- saveOffsetsInTx(conn, offsets)
- }
- .map(_ => Done)(ExecutionContext.parasitic)
- }
-
- def saveOffsetsInTx(conn: Connection, offsets: immutable.IndexedSeq[OffsetPidSeqNr]): Future[Done] = {
- if (offsets.isEmpty)
- FutureDone
- else if (offsets.head.offset.isInstanceOf[TimestampOffset]) {
- val records = offsets.map {
- case OffsetPidSeqNr(t: TimestampOffset, Some((pid, seqNr))) =>
- Record(pid, seqNr, t.timestamp)
- case OffsetPidSeqNr(_: TimestampOffset, None) =>
- throw new IllegalArgumentException("Required EventEnvelope or DurableStateChange for TimestampOffset.")
- case _ =>
- throw new IllegalArgumentException(
- "Mix of TimestampOffset and other offset type in same transaction is not supported")
- }
- saveTimestampOffsetInTx(conn, records)
- } else {
- savePrimitiveOffsetInTx(conn, offsets.last.offset)
- }
- }
-
- private def saveTimestampOffsetInTx[Offset](conn: Connection, records: immutable.IndexedSeq[Record]): Future[Done] = {
- idle.set(false)
- val oldState = state.get()
- val filteredRecords = {
- if (records.size <= 1)
- records.filterNot(oldState.isDuplicate)
- else {
- // use last record for each pid
- records
- .groupBy(_.pid)
- .valuesIterator
- .collect {
- case recordsByPid if !oldState.isDuplicate(recordsByPid.last) => recordsByPid.last
- }
- .toVector
- }
- }
- if (filteredRecords.isEmpty) {
- FutureDone
- } else {
- val newState = oldState.add(filteredRecords)
-
- // accumulate some more than the timeWindow before evicting
- val evictedNewState =
- if (newState.size > evictKeepNumberOfEntriesThreshold && newState.window.compareTo(evictWindow) > 0) {
- val evictUntil = newState.latestTimestamp.minus(settings.timeWindow)
- val s = newState.evict(evictUntil, settings.keepNumberOfEntries)
- logger.debug(
- "Evicted [{}] records until [{}], keeping [{}] records. Latest [{}].",
- newState.size - s.size: java.lang.Integer,
- evictUntil,
- s.size: java.lang.Integer,
- newState.latestTimestamp)
- s
- } else
- newState
-
- val offsetInserts = insertTimestampOffsetInTx(conn, filteredRecords)
-
- offsetInserts.map { _ =>
- if (state.compareAndSet(oldState, evictedNewState))
- cleanupInflight(evictedNewState)
- else
- throw new IllegalStateException("Unexpected concurrent modification of state from saveOffset.")
- Done
- }
- }
- }
-
- private def insertTimestampOffsetInTx(conn: Connection, records: immutable.IndexedSeq[Record]): Future[Long] = {
- def bindRecord(stmt: Statement, record: Record): Statement = {
- val slice = persistenceExt.sliceForPersistenceId(record.pid)
- val minSlice = timestampOffsetBySlicesSourceProvider.minSlice
- val maxSlice = timestampOffsetBySlicesSourceProvider.maxSlice
- if (slice < minSlice || slice > maxSlice)
- throw new IllegalArgumentException(
- s"This offset store [$projectionId] manages slices " +
- "[$minSlice - $maxSlice] but received slice [$slice] for persistenceId [${record.pid}]")
-
- stmt
- .bind(0, projectionId.name)
- .bind(1, projectionId.key)
- .bind(2, slice)
- .bind(3, record.pid)
- .bind(4, record.seqNr)
- .bind(5, record.timestamp)
- }
-
- require(records.nonEmpty)
-
- // FIXME change to trace
- logger.debug("saving timestamp offset [{}], {}", records.last.timestamp: Any, records: Any)
-
- val statement = conn.createStatement(insertTimestampOffsetSql)
-
- if (records.size == 1) {
- val boundStatement = bindRecord(statement, records.head)
- R2dbcExecutor.updateOneInTx(boundStatement)
- } else {
- // TODO Try Batch without bind parameters for better performance. Risk of sql injection for these parameters is low.
- val boundStatement =
- records.zipWithIndex.foldLeft(statement) { case (stmt, (rec, idx)) =>
- if (idx != 0) {
- stmt.add()
- }
- bindRecord(stmt, rec)
- }
- R2dbcExecutor.updateBatchInTx(boundStatement)
- }
- }
-
- @tailrec private def cleanupInflight(newState: State): Unit = {
- val currentInflight = getInflight()
- val newInflight =
- currentInflight.filter {
- case (inflightPid, inflightSeqNr) =>
- newState.byPid.get(inflightPid) match {
- case Some(r) => r.seqNr < inflightSeqNr
- case None => true
- }
- case _ => true
- }
- if (newInflight.size >= 10000) {
- throw new IllegalStateException(
- s"Too many envelopes in-flight [${newInflight.size}]. " +
- "Please report this issue at https://github.com/apache/pekko-persistence-r2dbc")
- }
- if (!inflight.compareAndSet(currentInflight, newInflight))
- cleanupInflight(newState) // CAS retry, concurrent update of inflight
- }
-
- @tailrec private def clearInflight(): Unit = {
- val currentInflight = getInflight()
- if (!inflight.compareAndSet(currentInflight, Map.empty[Pid, SeqNr]))
- clearInflight() // CAS retry, concurrent update of inflight
- }
-
- private def savePrimitiveOffsetInTx[Offset](conn: Connection, offset: Offset): Future[Done] = {
- logger.trace("saving offset [{}]", offset)
-
- if (!settings.isOffsetTableDefined)
- Future.failed(
- new IllegalArgumentException(
- "Offset table has been disabled config 'pekko.projection.r2dbc.offset-store.offset-table', " +
- s"but trying to save a non-timestamp offset [$offset]"))
-
- val now = Instant.now(clock).toEpochMilli
-
- // FIXME can we move serialization outside the transaction?
- val storageReps = toStorageRepresentation(projectionId, offset)
-
- def upsertStmt(singleOffset: SingleOffset): Statement = {
- conn
- .createStatement(upsertOffsetSql)
- .bind(0, singleOffset.id.name)
- .bind(1, singleOffset.id.key)
- .bind(2, singleOffset.offsetStr)
- .bind(3, singleOffset.manifest)
- .bind(4, java.lang.Boolean.valueOf(singleOffset.mergeable))
- .bind(5, now)
- }
-
- val statements = storageReps match {
- case single: SingleOffset => Vector(upsertStmt(single))
- case MultipleOffsets(many) => many.map(upsertStmt).toVector
- }
-
- R2dbcExecutor.updateInTx(statements).map(_ => Done)(ExecutionContext.parasitic)
- }
-
- def isDuplicate(record: Record): Boolean =
- getState().isDuplicate(record)
-
- def filterAccepted[Envelope](envelopes: immutable.Seq[Envelope]): Future[immutable.Seq[Envelope]] = {
- envelopes
- .foldLeft(Future.successful(getInflight(), Vector.empty[Envelope])) { (acc, envelope) =>
- acc.flatMap { case (inflight, filteredEnvelopes) =>
- createRecordWithOffset(envelope) match {
- case Some(recordWithOffset) =>
- isAccepted(recordWithOffset, inflight).map {
- case true =>
- (
- inflight.updated(recordWithOffset.record.pid, recordWithOffset.record.seqNr),
- filteredEnvelopes :+ envelope)
- case false =>
- (inflight, filteredEnvelopes)
- }
- case None =>
- Future.successful((inflight, filteredEnvelopes :+ envelope))
- }
- }
- }
- .map { case (_, filteredEnvelopes) =>
- filteredEnvelopes
- }
- }
-
- def isAccepted[Envelope](envelope: Envelope): Future[Boolean] = {
- createRecordWithOffset(envelope) match {
- case Some(recordWithOffset) => isAccepted(recordWithOffset, getInflight())
- case None => FutureTrue
- }
- }
-
- private def isAccepted[Envelope](
- recordWithOffset: RecordWithOffset,
- currentInflight: Map[Pid, SeqNr]): Future[Boolean] = {
- val pid = recordWithOffset.record.pid
- val seqNr = recordWithOffset.record.seqNr
- val currentState = getState()
-
- val duplicate = isDuplicate(recordWithOffset.record)
-
- if (duplicate) {
- logger.trace("Filtering out duplicate sequence number [{}] for pid [{}]", seqNr, pid)
- FutureFalse
- } else if (recordWithOffset.strictSeqNr) {
- // strictSeqNr == true is for event sourced
- val prevSeqNr = currentInflight.getOrElse(pid, currentState.byPid.get(pid).map(_.seqNr).getOrElse(0L))
-
- def logUnexpected(): Unit = {
- if (recordWithOffset.fromPubSub)
- logger.debug(
- "Rejecting pub-sub envelope, unexpected sequence number [{}] for pid [{}], previous sequence number [{}]. Offset: {}",
- seqNr: java.lang.Long,
- pid,
- prevSeqNr: java.lang.Long,
- recordWithOffset.offset)
- else if (!recordWithOffset.fromBacktracking)
- logger.debug(
- "Rejecting unexpected sequence number [{}] for pid [{}], previous sequence number [{}]. Offset: {}",
- seqNr: java.lang.Long,
- pid,
- prevSeqNr: java.lang.Long,
- recordWithOffset.offset)
- else
- logger.warn(
- "Rejecting unexpected sequence number [{}] for pid [{}], previous sequence number [{}]. Offset: {}",
- seqNr: java.lang.Long,
- pid,
- prevSeqNr: java.lang.Long,
- recordWithOffset.offset)
- }
-
- def logUnknown(): Unit = {
- if (recordWithOffset.fromPubSub) {
- logger.debug(
- "Rejecting pub-sub envelope, unknown sequence number [{}] for pid [{}] (might be accepted later): {}",
- seqNr: java.lang.Long,
- pid,
- recordWithOffset.offset)
- } else if (!recordWithOffset.fromBacktracking) {
- // This may happen rather frequently when using `publish-events`, after reconnecting and such.
- logger.debug(
- "Rejecting unknown sequence number [{}] for pid [{}] (might be accepted later): {}",
- seqNr: java.lang.Long,
- pid,
- recordWithOffset.offset)
- } else {
- logger.warn(
- "Rejecting unknown sequence number [{}] for pid [{}]. Offset: {}",
- seqNr: java.lang.Long,
- pid,
- recordWithOffset.offset)
- }
- }
-
- if (prevSeqNr > 0) {
- // expecting seqNr to be +1 of previously known
- val ok = seqNr == prevSeqNr + 1
- if (ok) {
- FutureTrue
- } else if (seqNr <= currentInflight.getOrElse(pid, 0L)) {
- // currentInFlight contains those that have been processed or about to be processed in Flow,
- // but offset not saved yet => ok to handle as duplicate
- FutureFalse
- } else if (!recordWithOffset.fromBacktracking) {
- logUnexpected()
- FutureFalse
- } else {
- logUnexpected()
- // This will result in projection restart (with normal configuration)
- Future.failed(
- new IllegalStateException(
- s"Rejected envelope from backtracking, persistenceId [$pid], seqNr [$seqNr] " +
- "due to unexpected sequence number. " +
- "Please report this issue at https://github.com/apache/pekko-persistence-r2dbc"))
- }
- } else if (seqNr == 1) {
- // always accept first event if no other event for that pid has been seen
- FutureTrue
- } else {
- // Haven't see seen this pid within the time window. Since events can be missed
- // when read at the tail we will only accept it if the event with previous seqNr has timestamp
- // before the time window of the offset store.
- // Backtracking will emit missed event again.
- timestampOf(pid, seqNr - 1).map {
- case Some(previousTimestamp) =>
- val before = currentState.latestTimestamp.minus(settings.timeWindow)
- if (previousTimestamp.isBefore(before)) {
- logger.debug(
- "Accepting envelope with pid [{}], seqNr [{}], where previous event timestamp [{}] " +
- "is before time window [{}].",
- pid,
- seqNr: java.lang.Long,
- previousTimestamp,
- before)
- true
- } else if (!recordWithOffset.fromBacktracking) {
- logUnknown()
- false
- } else {
- logUnknown()
- // This will result in projection restart (with normal configuration)
- throw new IllegalStateException(
- s"Rejected envelope from backtracking, persistenceId [$pid], seqNr [$seqNr], " +
- "due to unknown sequence number. " +
- "Please report this issue at https://github.com/apache/pekko-persistence-r2dbc")
- }
- case None =>
- // previous not found, could have been deleted
- true
- }
- }
- } else {
- // strictSeqNr == false is for durable state where each revision might not be visible
- val prevSeqNr = currentInflight.getOrElse(pid, currentState.byPid.get(pid).map(_.seqNr).getOrElse(0L))
- val ok = seqNr > prevSeqNr
-
- if (ok) {
- FutureTrue
- } else {
- logger.trace("Filtering out earlier revision [{}] for pid [{}], previous revision [{}]", seqNr: java.lang.Long,
- pid,
- prevSeqNr: java.lang.Long)
- FutureFalse
- }
- }
- }
-
- @tailrec final def addInflight[Envelope](envelope: Envelope): Unit = {
- createRecordWithOffset(envelope) match {
- case Some(recordWithOffset) =>
- val currentInflight = getInflight()
- val newInflight = currentInflight.updated(recordWithOffset.record.pid, recordWithOffset.record.seqNr)
- if (!inflight.compareAndSet(currentInflight, newInflight))
- addInflight(envelope) // CAS retry, concurrent update of inflight
- case None =>
- }
- }
-
- @tailrec final def addInflights[Envelope](envelopes: immutable.Seq[Envelope]): Unit = {
- val currentInflight = getInflight()
- val entries = envelopes.iterator.map(createRecordWithOffset).collect { case Some(r) =>
- r.record.pid -> r.record.seqNr
- }
- val newInflight = currentInflight ++ entries
- if (!inflight.compareAndSet(currentInflight, newInflight))
- addInflights(envelopes) // CAS retry, concurrent update of inflight
- }
-
- def isInflight[Envelope](envelope: Envelope): Boolean = {
- createRecordWithOffset(envelope) match {
- case Some(recordWithOffset) =>
- val pid = recordWithOffset.record.pid
- val seqNr = recordWithOffset.record.seqNr
- getInflight().get(pid) match {
- case Some(`seqNr`) => true
- case _ => false
- }
- case None => true
- }
- }
-
- def deleteOldTimestampOffsets(): Future[Long] = {
- if (idle.getAndSet(true)) {
- // no new offsets stored since previous delete
- Future.successful(0)
- } else {
- val currentState = getState()
- if (currentState.size <= settings.keepNumberOfEntries || currentState.window.compareTo(settings.timeWindow) < 0) {
- // it hasn't filled up the window yet
- Future.successful(0)
- } else {
- val until = currentState.latestTimestamp.minus(settings.timeWindow)
- val minSlice = timestampOffsetBySlicesSourceProvider.minSlice
- val maxSlice = timestampOffsetBySlicesSourceProvider.maxSlice
- val result = r2dbcExecutor.updateOne("delete old timestamp offset") { conn =>
- conn
- .createStatement(deleteOldTimestampOffsetSql)
- .bind(0, minSlice)
- .bind(1, maxSlice)
- .bind(2, projectionId.name)
- .bind(3, until)
- }
-
- // FIXME would it be good to keep at least one record per slice that can be used as the
- // starting point for the slice if the slice ranges are changed?
-
- result.failed.foreach { exc =>
- idle.set(false) // try again next tick
- logger.warn(
- "Failed to delete timestamp offset until [{}] for projection [{}]: {}",
- until,
- projectionId.id,
- exc.toString)
- }
- if (logger.isDebugEnabled)
- result.foreach { rows =>
- logger.debug(
- "Deleted [{}] timestamp offset rows until [{}] for projection [{}].",
- rows: java.lang.Long,
- until,
- projectionId.id)
- }
-
- result
- }
- }
- }
-
- /**
- * Resetting an offset. Deletes newer offsets. Used from ProjectionManagement. Doesn't update in-memory state because
- * the projection is supposed to be stopped/started for this operation.
- */
- def managementSetOffset[Offset](offset: Offset): Future[Done] = {
- offset match {
- case t: TimestampOffset =>
- r2dbcExecutor
- .withConnection("set offset") { conn =>
- deleteNewTimestampOffsetsInTx(conn, t.timestamp).flatMap { _ =>
- val records =
- if (t.seen.isEmpty)
- // we need some persistenceId to be able to store the new offset timestamp
- Vector(Record(PersistenceId("mgmt", UUID.randomUUID().toString).id, seqNr = 1L, t.timestamp))
- else
- t.seen.iterator.map { case (pid, seqNr) => Record(pid, seqNr, t.timestamp) }.toVector
- insertTimestampOffsetInTx(conn, records)
- }
- }
- .map(_ => Done)(ExecutionContext.parasitic)
-
- case _ =>
- r2dbcExecutor
- .withConnection("set offset") { conn =>
- savePrimitiveOffsetInTx(conn, offset)
- }
- .map(_ => Done)(ExecutionContext.parasitic)
- }
- }
-
- private def deleteNewTimestampOffsetsInTx(conn: Connection, timestamp: Instant): Future[Long] = {
- val currentState = getState()
- if (timestamp.isAfter(currentState.latestTimestamp)) {
- // nothing to delete
- Future.successful(0)
- } else {
- val minSlice = timestampOffsetBySlicesSourceProvider.minSlice
- val maxSlice = timestampOffsetBySlicesSourceProvider.maxSlice
- val result = R2dbcExecutor.updateOneInTx(
- conn
- .createStatement(deleteNewTimestampOffsetSql)
- .bind(0, minSlice)
- .bind(1, maxSlice)
- .bind(2, projectionId.name)
- .bind(3, timestamp))
-
- // FIXME would it be good to keep at least one record per slice that can be used as the
- // starting point for the slice if the slice ranges are changed?
-
- if (logger.isDebugEnabled)
- result.foreach { rows =>
- logger.debug(
- "Deleted [{}] timestamp offset rows >= [{}] for projection [{}].",
- rows: java.lang.Long,
- timestamp,
- projectionId.id)
- }
-
- result
- }
- }
-
- /**
- * Deletes all offsets. Used from ProjectionManagement. Doesn't update in-memory state because the projection is
- * supposed to be stopped/started for this operation.
- */
- def managementClearOffset(): Future[Done] = {
- clearTimestampOffset().flatMap(_ => clearPrimitiveOffset())
- }
-
- private def clearTimestampOffset(): Future[Done] = {
- sourceProvider match {
- case Some(_) =>
- idle.set(false)
- r2dbcExecutor
- .updateOne("clear timestamp offset") { conn =>
- val minSlice = timestampOffsetBySlicesSourceProvider.minSlice
- val maxSlice = timestampOffsetBySlicesSourceProvider.maxSlice
- logger.debug("clearing timestamp offset for [{}]", projectionId)
- conn
- .createStatement(clearTimestampOffsetSql)
- .bind(0, minSlice)
- .bind(1, maxSlice)
- .bind(2, projectionId.name)
- }
- .map { n =>
- logger.debug(s"clearing timestamp offset for [{}] - executed statement returned [{}]", projectionId, n)
- Done
- }
- case None =>
- FutureDone
- }
- }
-
- private def clearPrimitiveOffset(): Future[Done] = {
- if (settings.isOffsetTableDefined) {
- r2dbcExecutor
- .updateOne("clear offset") { conn =>
- logger.debug("clearing offset for [{}]", projectionId)
- conn
- .createStatement(clearOffsetSql)
- .bind(0, projectionId.name)
- .bind(1, projectionId.key)
- }
- .map { n =>
- logger.debug(s"clearing offset for [{}] - executed statement returned [{}]", projectionId, n)
- Done
- }
- } else {
- FutureDone
- }
- }
-
- def readManagementState(): Future[Option[ManagementState]] = {
- def createStatement(connection: Connection) =
- connection
- .createStatement(readManagementStateSql)
- .bind(0, projectionId.name)
- .bind(1, projectionId.key)
-
- r2dbcExecutor
- .selectOne("read management state")(
- conn => createStatement(conn),
- row => ManagementState(row.get[java.lang.Boolean]("paused", classOf[java.lang.Boolean])))
- }
-
- def savePaused(paused: Boolean): Future[Done] = {
- r2dbcExecutor
- .updateOne("update management state") { conn =>
- conn
- .createStatement(updateManagementStateSql)
- .bind(0, projectionId.name)
- .bind(1, projectionId.key)
- .bind(2, paused)
- .bind(3, Instant.now(clock).toEpochMilli)
- }
- .map(_ => Done)(ExecutionContext.parasitic)
- }
-
- private def createRecordWithOffset[Envelope](envelope: Envelope): Option[RecordWithOffset] = {
- envelope match {
- case eventEnvelope: EventEnvelope[_] if eventEnvelope.offset.isInstanceOf[TimestampOffset] =>
- val timestampOffset = eventEnvelope.offset.asInstanceOf[TimestampOffset]
- Some(
- RecordWithOffset(
- Record(eventEnvelope.persistenceId, eventEnvelope.sequenceNr, timestampOffset.timestamp),
- timestampOffset,
- strictSeqNr = true,
- fromBacktracking = EnvelopeOrigin.fromBacktracking(eventEnvelope),
- fromPubSub = EnvelopeOrigin.fromPubSub(eventEnvelope)))
- case change: UpdatedDurableState[_] if change.offset.isInstanceOf[TimestampOffset] =>
- val timestampOffset = change.offset.asInstanceOf[TimestampOffset]
- Some(
- RecordWithOffset(
- Record(change.persistenceId, change.revision, timestampOffset.timestamp),
- timestampOffset,
- strictSeqNr = false,
- fromBacktracking = change.value == null,
- fromPubSub = false))
- case change: DeletedDurableState[_] if change.offset.isInstanceOf[TimestampOffset] =>
- val timestampOffset = change.offset.asInstanceOf[TimestampOffset]
- Some(
- RecordWithOffset(
- Record(change.persistenceId, change.revision, timestampOffset.timestamp),
- timestampOffset,
- strictSeqNr = false,
- fromBacktracking = false,
- fromPubSub = false))
- case change: DurableStateChange[_] if change.offset.isInstanceOf[TimestampOffset] =>
- // in case additional types are added
- throw new IllegalArgumentException(
- s"DurableStateChange [${change.getClass.getName}] not implemented yet. Please report bug at https://github.com/apache/pekko-persistence-r2dbc/issues")
- case _ => None
- }
- }
-
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcProjectionImpl.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcProjectionImpl.scala
deleted file mode 100644
index 3d7d136..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/R2dbcProjectionImpl.scala
+++ /dev/null
@@ -1,643 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021-2022 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.internal
-
-import java.util.concurrent.atomic.AtomicLong
-
-import scala.collection.immutable
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import scala.concurrent.duration.FiniteDuration
-import scala.util.control.NonFatal
-
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.typed.ActorSystem
-import pekko.annotation.InternalApi
-import pekko.event.Logging
-import pekko.event.LoggingAdapter
-import pekko.persistence.query.DeletedDurableState
-import pekko.persistence.query.DurableStateChange
-import pekko.persistence.query.UpdatedDurableState
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.query.typed.scaladsl.LoadEventQuery
-import pekko.persistence.r2dbc.internal.EnvelopeOrigin
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import pekko.persistence.state.scaladsl.DurableStateStore
-import pekko.persistence.state.scaladsl.GetObjectResult
-import pekko.projection.BySlicesSourceProvider
-import pekko.projection.HandlerRecoveryStrategy
-import pekko.projection.HandlerRecoveryStrategy.Internal.RetryAndSkip
-import pekko.projection.HandlerRecoveryStrategy.Internal.Skip
-import pekko.projection.ProjectionContext
-import pekko.projection.ProjectionId
-import pekko.projection.RunningProjection
-import pekko.projection.RunningProjection.AbortProjectionException
-import pekko.projection.RunningProjectionManagement
-import pekko.projection.StatusObserver
-import pekko.projection.internal.ActorHandlerInit
-import pekko.projection.internal.AtLeastOnce
-import pekko.projection.internal.AtMostOnce
-import pekko.projection.internal.ExactlyOnce
-import pekko.projection.internal.GroupedHandlerStrategy
-import pekko.projection.internal.HandlerStrategy
-import pekko.projection.internal.InternalProjection
-import pekko.projection.internal.InternalProjectionState
-import pekko.projection.internal.ManagementState
-import pekko.projection.internal.OffsetStrategy
-import pekko.projection.internal.ProjectionContextImpl
-import pekko.projection.internal.ProjectionSettings
-import pekko.projection.internal.SettingsImpl
-import pekko.projection.javadsl
-import pekko.projection.r2dbc.R2dbcProjectionSettings
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import pekko.projection.scaladsl
-import pekko.projection.scaladsl.Handler
-import pekko.projection.scaladsl.SourceProvider
-import pekko.stream.RestartSettings
-import pekko.stream.scaladsl.FlowWithContext
-import pekko.stream.scaladsl.Source
-import io.r2dbc.spi.ConnectionFactory
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
-
-/**
- * INTERNAL API
- */
-@InternalApi
-private[projection] object R2dbcProjectionImpl {
- import EnvelopeOrigin.{ fromBacktracking, isFilteredEvent }
- val log: Logger = LoggerFactory.getLogger(classOf[R2dbcProjectionImpl[_, _]])
-
- private val FutureDone: Future[Done] = Future.successful(Done)
-
- private[projection] def createOffsetStore(
- projectionId: ProjectionId,
- sourceProvider: Option[BySlicesSourceProvider],
- settings: R2dbcProjectionSettings,
- connectionFactory: ConnectionFactory)(implicit system: ActorSystem[_]) = {
- val r2dbcExecutor =
- new R2dbcExecutor(connectionFactory, log, settings.logDbCallsExceeding)(system.executionContext, system)
- R2dbcOffsetStore.fromConfig(projectionId, sourceProvider, system, settings, r2dbcExecutor)
- }
-
- private val loadEnvelopeCounter = new AtomicLong
-
- def loadEnvelope[Envelope](env: Envelope, sourceProvider: SourceProvider[_, Envelope])(implicit
- ec: ExecutionContext): Future[Envelope] = {
- env match {
- case eventEnvelope: EventEnvelope[_]
- if fromBacktracking(eventEnvelope) && eventEnvelope.eventOption.isEmpty && !eventEnvelope.filtered =>
- val pid = eventEnvelope.persistenceId
- val seqNr = eventEnvelope.sequenceNr
- (sourceProvider match {
- case loadEventQuery: LoadEventQuery =>
- loadEventQuery.loadEnvelope[Any](pid, seqNr)
- case loadEventQuery: pekko.persistence.query.typed.javadsl.LoadEventQuery =>
- import scala.jdk.FutureConverters._
- loadEventQuery.loadEnvelope[Any](pid, seqNr).asScala
- case _ =>
- throw new IllegalArgumentException(
- s"Expected sourceProvider [${sourceProvider.getClass.getName}] " +
- "to implement LoadEventQuery when used with eventsBySlices.")
- }).map { loadedEnv =>
- val count = loadEnvelopeCounter.incrementAndGet()
- if (count % 1000 == 0)
- log.info("Loaded event lazily, persistenceId [{}], seqNr [{}]. Load count [{}]", pid, seqNr: java.lang.Long,
- count: java.lang.Long)
- else
- log.debug("Loaded event lazily, persistenceId [{}], seqNr [{}]. Load count [{}]", pid,
- seqNr: java.lang.Long,
- count: java.lang.Long)
- loadedEnv.asInstanceOf[Envelope]
- }
-
- case upd: UpdatedDurableState[_] if upd.value == null =>
- val pid = upd.persistenceId
- (sourceProvider match {
- case store: DurableStateStore[_] =>
- store.getObject(pid)
- case store: pekko.persistence.state.javadsl.DurableStateStore[_] =>
- import scala.jdk.FutureConverters._
- store.getObject(pid).asScala.map(_.toScala)
- }).map {
- case GetObjectResult(Some(loadedValue), loadedRevision) =>
- val count = loadEnvelopeCounter.incrementAndGet()
- if (count % 1000 == 0)
- log.info(
- "Loaded durable state lazily, persistenceId [{}], revision [{}]. Load count [{}]",
- pid,
- loadedRevision: java.lang.Long,
- count: java.lang.Long)
- else
- log.debug(
- "Loaded durable state lazily, persistenceId [{}], revision [{}]. Load count [{}]",
- pid,
- loadedRevision: java.lang.Long,
- count: java.lang.Long)
- new UpdatedDurableState(pid, loadedRevision, loadedValue, upd.offset, upd.timestamp)
- .asInstanceOf[Envelope]
- case GetObjectResult(None, loadedRevision) =>
- new DeletedDurableState(pid, loadedRevision, upd.offset, upd.timestamp)
- .asInstanceOf[Envelope]
- }
-
- case _ =>
- Future.successful(env)
- }
- }
-
- private def extractOffsetPidSeqNr[Offset, Envelope](
- sourceProvider: SourceProvider[Offset, Envelope],
- envelope: Envelope): OffsetPidSeqNr =
- extractOffsetPidSeqNr(sourceProvider.extractOffset(envelope), envelope)
-
- private def extractOffsetPidSeqNr[Offset, Envelope](offset: Offset, envelope: Envelope): OffsetPidSeqNr = {
- // we could define a new trait for the SourceProvider to implement this in case other (custom) envelope types are needed
- envelope match {
- case env: EventEnvelope[_] => OffsetPidSeqNr(offset, env.persistenceId, env.sequenceNr)
- case chg: UpdatedDurableState[_] => OffsetPidSeqNr(offset, chg.persistenceId, chg.revision)
- case del: DeletedDurableState[_] => OffsetPidSeqNr(offset, del.persistenceId, del.revision)
- case change: DurableStateChange[_] =>
- // in case additional types are added
- throw new IllegalArgumentException(
- s"DurableStateChange [${change.getClass.getName}] not implemented yet. Please report bug at https://github.com/apache/pekko-persistence-r2dbc/issues")
- case _ => OffsetPidSeqNr(offset)
- }
- }
-
- private[projection] def adaptedHandlerForExactlyOnce[Offset, Envelope](
- sourceProvider: SourceProvider[Offset, Envelope],
- handlerFactory: () => R2dbcHandler[Envelope],
- offsetStore: R2dbcOffsetStore,
- r2dbcExecutor: R2dbcExecutor)(implicit ec: ExecutionContext, system: ActorSystem[_]): () => Handler[Envelope] = {
- () =>
- new AdaptedR2dbcHandler(handlerFactory()) {
- override def process(envelope: Envelope): Future[Done] = {
- offsetStore.isAccepted(envelope).flatMap {
- case true =>
- if (isFilteredEvent(envelope)) {
- val offset = extractOffsetPidSeqNr(sourceProvider, envelope)
- offsetStore.saveOffset(offset)
- } else {
- loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
- val offset = extractOffsetPidSeqNr(sourceProvider, loadedEnvelope)
- r2dbcExecutor.withConnection("exactly-once handler") { conn =>
- // run users handler
- val session = new R2dbcSession(conn)
- delegate
- .process(session, loadedEnvelope)
- .flatMap { _ =>
- offsetStore.saveOffsetInTx(conn, offset)
- }
- }
- }
- }
- case false =>
- FutureDone
- }
- }
- }
- }
-
- private[projection] def adaptedHandlerForGrouped[Offset, Envelope](
- sourceProvider: SourceProvider[Offset, Envelope],
- handlerFactory: () => R2dbcHandler[immutable.Seq[Envelope]],
- offsetStore: R2dbcOffsetStore,
- r2dbcExecutor: R2dbcExecutor)(
- implicit
- ec: ExecutionContext,
- system: ActorSystem[_]): () => Handler[immutable.Seq[Envelope]] = { () =>
- new AdaptedR2dbcHandler(handlerFactory()) {
- override def process(envelopes: immutable.Seq[Envelope]): Future[Done] = {
- offsetStore.filterAccepted(envelopes).flatMap { acceptedEnvelopes =>
- if (acceptedEnvelopes.isEmpty) {
- FutureDone
- } else {
- Future.sequence(acceptedEnvelopes.map(env => loadEnvelope(env, sourceProvider))).flatMap {
- loadedEnvelopes =>
- val offsets = loadedEnvelopes.iterator.map(extractOffsetPidSeqNr(sourceProvider, _)).toVector
- val filteredEnvelopes = loadedEnvelopes.filterNot(isFilteredEvent)
- if (filteredEnvelopes.isEmpty) {
- offsetStore.saveOffsets(offsets)
- } else {
- r2dbcExecutor.withConnection("grouped handler") { conn =>
- // run users handler
- val session = new R2dbcSession(conn)
- delegate.process(session, filteredEnvelopes).flatMap { _ =>
- offsetStore.saveOffsetsInTx(conn, offsets)
- }
- }
- }
- }
- }
- }
- }
- }
- }
-
- private[projection] def adaptedHandlerForAtLeastOnce[Offset, Envelope](
- sourceProvider: SourceProvider[Offset, Envelope],
- handlerFactory: () => R2dbcHandler[Envelope],
- offsetStore: R2dbcOffsetStore,
- r2dbcExecutor: R2dbcExecutor)(implicit ec: ExecutionContext, system: ActorSystem[_]): () => Handler[Envelope] = {
- () =>
- new AdaptedR2dbcHandler(handlerFactory()) {
- override def process(envelope: Envelope): Future[Done] = {
- offsetStore.isAccepted(envelope).flatMap {
- case true =>
- if (isFilteredEvent(envelope)) {
- offsetStore.addInflight(envelope)
- FutureDone
- } else {
- loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
- r2dbcExecutor
- .withConnection("at-least-once handler") { conn =>
- // run users handler
- val session = new R2dbcSession(conn)
- delegate.process(session, loadedEnvelope)
- }
- .map { _ =>
- offsetStore.addInflight(loadedEnvelope)
- Done
- }
- }
- }
- case false =>
- FutureDone
- }
- }
- }
- }
-
- private[projection] def adaptedHandlerForAtLeastOnceAsync[Offset, Envelope](
- sourceProvider: SourceProvider[Offset, Envelope],
- handlerFactory: () => Handler[Envelope],
- offsetStore: R2dbcOffsetStore)(implicit ec: ExecutionContext, system: ActorSystem[_]): () => Handler[Envelope] = {
- () =>
- new AdaptedHandler(handlerFactory()) {
- override def process(envelope: Envelope): Future[Done] = {
- offsetStore.isAccepted(envelope).flatMap {
- case true =>
- if (isFilteredEvent(envelope)) {
- offsetStore.addInflight(envelope)
- FutureDone
- } else {
- loadEnvelope(envelope, sourceProvider).flatMap { loadedEnvelope =>
- delegate
- .process(loadedEnvelope)
- .map { _ =>
- offsetStore.addInflight(loadedEnvelope)
- Done
- }
- }
- }
- case false =>
- FutureDone
- }
- }
- }
- }
-
- private[projection] def adaptedHandlerForGroupedAsync[Offset, Envelope](
- sourceProvider: SourceProvider[Offset, Envelope],
- handlerFactory: () => Handler[immutable.Seq[Envelope]],
- offsetStore: R2dbcOffsetStore)(
- implicit
- ec: ExecutionContext,
- system: ActorSystem[_]): () => Handler[immutable.Seq[Envelope]] = { () =>
- new AdaptedHandler(handlerFactory()) {
- override def process(envelopes: immutable.Seq[Envelope]): Future[Done] = {
- offsetStore.filterAccepted(envelopes).flatMap { acceptedEnvelopes =>
- if (acceptedEnvelopes.isEmpty) {
- FutureDone
- } else {
- Future.sequence(acceptedEnvelopes.map(env => loadEnvelope(env, sourceProvider))).flatMap {
- loadedEnvelopes =>
- val filteredEnvelopes = loadedEnvelopes.filterNot(isFilteredEvent)
- if (filteredEnvelopes.isEmpty) {
- offsetStore.addInflights(loadedEnvelopes)
- FutureDone
- } else {
- delegate
- .process(filteredEnvelopes)
- .map { _ =>
- offsetStore.addInflights(loadedEnvelopes)
- Done
- }
- }
- }
- }
- }
- }
- }
- }
-
- private[projection] def adaptedHandlerForFlow[Offset, Envelope](
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _],
- offsetStore: R2dbcOffsetStore)(
- implicit
- ec: ExecutionContext,
- system: ActorSystem[_]): FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _] = {
-
- FlowWithContext[Envelope, ProjectionContext]
- .mapAsync(1) { env =>
- offsetStore
- .isAccepted(env)
- .flatMap { ok =>
- if (ok) {
- if (isFilteredEvent(env)) {
- log.info("atLeastOnceFlow doesn't support of skipping envelopes. Envelope [{}] still emitted.", env)
- }
- loadEnvelope(env, sourceProvider).map { loadedEnvelope =>
- offsetStore.addInflight(loadedEnvelope)
- Some(loadedEnvelope)
- }
- } else {
- Future.successful(None)
- }
- }
- }
- .collect { case Some(env) =>
- env
- }
- .via(handler)
- }
-
- abstract class AdaptedR2dbcHandler[E](val delegate: R2dbcHandler[E])(
- implicit
- ec: ExecutionContext,
- system: ActorSystem[_])
- extends Handler[E] {
-
- override def start(): Future[Done] =
- delegate.start()
-
- override def stop(): Future[Done] =
- delegate.stop()
- }
-
- abstract class AdaptedHandler[E](val delegate: Handler[E])(implicit ec: ExecutionContext, system: ActorSystem[_])
- extends Handler[E] {
-
- override def start(): Future[Done] =
- delegate.start()
-
- override def stop(): Future[Done] =
- delegate.stop()
- }
-
-}
-
-/**
- * INTERNAL API
- */
-@InternalApi
-private[projection] class R2dbcProjectionImpl[Offset, Envelope](
- val projectionId: ProjectionId,
- r2dbcSettings: R2dbcProjectionSettings,
- settingsOpt: Option[ProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- restartBackoffOpt: Option[RestartSettings],
- val offsetStrategy: OffsetStrategy,
- handlerStrategy: HandlerStrategy,
- override val statusObserver: StatusObserver[Envelope],
- offsetStore: R2dbcOffsetStore)
- extends scaladsl.ExactlyOnceProjection[Offset, Envelope]
- with javadsl.ExactlyOnceProjection[Offset, Envelope]
- with scaladsl.GroupedProjection[Offset, Envelope]
- with javadsl.GroupedProjection[Offset, Envelope]
- with scaladsl.AtLeastOnceProjection[Offset, Envelope]
- with javadsl.AtLeastOnceProjection[Offset, Envelope]
- with scaladsl.AtLeastOnceFlowProjection[Offset, Envelope]
- with javadsl.AtLeastOnceFlowProjection[Offset, Envelope]
- with SettingsImpl[R2dbcProjectionImpl[Offset, Envelope]]
- with InternalProjection {
-
- private def copy(
- settingsOpt: Option[ProjectionSettings] = this.settingsOpt,
- restartBackoffOpt: Option[RestartSettings] = this.restartBackoffOpt,
- offsetStrategy: OffsetStrategy = this.offsetStrategy,
- handlerStrategy: HandlerStrategy = this.handlerStrategy,
- statusObserver: StatusObserver[Envelope] = this.statusObserver): R2dbcProjectionImpl[Offset, Envelope] =
- new R2dbcProjectionImpl(
- projectionId,
- r2dbcSettings,
- settingsOpt,
- sourceProvider,
- restartBackoffOpt,
- offsetStrategy,
- handlerStrategy,
- statusObserver,
- offsetStore)
-
- type ReadOffset = () => Future[Option[Offset]]
-
- /*
- * Build the final ProjectionSettings to use, if currently set to None fallback to values in config file
- */
- private def settingsOrDefaults(implicit system: ActorSystem[_]): ProjectionSettings = {
- val settings = settingsOpt.getOrElse(ProjectionSettings(system))
- restartBackoffOpt match {
- case None => settings
- case Some(r) => settings.copy(restartBackoff = r)
- }
- }
-
- override def withRestartBackoffSettings(restartBackoff: RestartSettings): R2dbcProjectionImpl[Offset, Envelope] =
- copy(restartBackoffOpt = Some(restartBackoff))
-
- override def withSaveOffset(
- afterEnvelopes: Int,
- afterDuration: FiniteDuration): R2dbcProjectionImpl[Offset, Envelope] =
- copy(offsetStrategy = offsetStrategy
- .asInstanceOf[AtLeastOnce]
- .copy(afterEnvelopes = Some(afterEnvelopes), orAfterDuration = Some(afterDuration)))
-
- override def withGroup(
- groupAfterEnvelopes: Int,
- groupAfterDuration: FiniteDuration): R2dbcProjectionImpl[Offset, Envelope] =
- copy(handlerStrategy = handlerStrategy
- .asInstanceOf[GroupedHandlerStrategy[Envelope]]
- .copy(afterEnvelopes = Some(groupAfterEnvelopes), orAfterDuration = Some(groupAfterDuration)))
-
- override def withRecoveryStrategy(
- recoveryStrategy: HandlerRecoveryStrategy): R2dbcProjectionImpl[Offset, Envelope] = {
- val newStrategy = offsetStrategy match {
- case s: ExactlyOnce => s.copy(recoveryStrategy = Some(recoveryStrategy))
- case s: AtLeastOnce => s.copy(recoveryStrategy = Some(recoveryStrategy))
- // NOTE: AtMostOnce has its own withRecoveryStrategy variant
- // this method is not available for AtMostOnceProjection
- case s: AtMostOnce => s
- }
- copy(offsetStrategy = newStrategy)
- }
-
- override def withStatusObserver(observer: StatusObserver[Envelope]): R2dbcProjectionImpl[Offset, Envelope] =
- copy(statusObserver = observer)
-
- private[projection] def actorHandlerInit[T]: Option[ActorHandlerInit[T]] =
- handlerStrategy.actorHandlerInit
-
- /**
- * INTERNAL API Return a RunningProjection
- */
- override private[projection] def run()(implicit system: ActorSystem[_]): RunningProjection =
- new R2dbcInternalProjectionState(settingsOrDefaults).newRunningInstance()
-
- /**
- * INTERNAL API
- *
- * This method returns the projection Source mapped with user 'handler' function, but before any sink attached. This
- * is mainly intended to be used by the TestKit allowing it to attach a TestSink to it.
- */
- override private[projection] def mappedSource()(implicit system: ActorSystem[_]): Source[Done, Future[Done]] =
- new R2dbcInternalProjectionState(settingsOrDefaults).mappedSource()
-
- private class R2dbcInternalProjectionState(settings: ProjectionSettings)(implicit val system: ActorSystem[_])
- extends InternalProjectionState[Offset, Envelope](
- projectionId,
- sourceProvider,
- offsetStrategy,
- handlerStrategy,
- statusObserver,
- settings) {
-
- implicit val executionContext: ExecutionContext = system.executionContext
- override val logger: LoggingAdapter = Logging(system.classicSystem, classOf[R2dbcInternalProjectionState])
-
- private val isExactlyOnceWithSkip: Boolean =
- offsetStrategy match {
- case ExactlyOnce(Some(Skip)) | ExactlyOnce(Some(_: RetryAndSkip)) => true
- case _ => false
- }
-
- override def readPaused(): Future[Boolean] =
- offsetStore.readManagementState().map(_.exists(_.paused))
-
- override def readOffsets(): Future[Option[Offset]] =
- offsetStore.readOffset()
-
- // Called from InternalProjectionState.saveOffsetAndReport
- override def saveOffset(projectionId: ProjectionId, offset: Offset): Future[Done] = {
- // need the envelope to be able to call offsetStore.saveOffset
- // FIXME maybe we can cleanup this mess when moving R2dbcProjection to the Pekko Projections repository?
- throw new IllegalStateException(
- "Unexpected call to saveOffset. It should have called saveOffsetAndReport. Please report bug at https://github.com/apache/pekko-persistence-r2dbc/issues")
- }
-
- override protected def saveOffsetAndReport(
- projectionId: ProjectionId,
- projectionContext: ProjectionContextImpl[Offset, Envelope],
- batchSize: Int): Future[Done] = {
- import R2dbcProjectionImpl.FutureDone
- val envelope = projectionContext.envelope
-
- if (offsetStore.isInflight(envelope) || isExactlyOnceWithSkip) {
- val offset = R2dbcProjectionImpl.extractOffsetPidSeqNr(projectionContext.offset, envelope)
- offsetStore
- .saveOffset(offset)
- .map { done =>
- try {
- statusObserver.offsetProgress(projectionId, envelope)
- } catch {
- case NonFatal(_) => // ignore
- }
- getTelemetry().onOffsetStored(batchSize)
- done
- }
- } else {
- FutureDone
- }
- }
-
- override protected def saveOffsetsAndReport(
- projectionId: ProjectionId,
- batch: immutable.Seq[ProjectionContextImpl[Offset, Envelope]]): Future[Done] = {
- import R2dbcProjectionImpl.FutureDone
-
- val acceptedContexts =
- if (isExactlyOnceWithSkip)
- batch.toVector
- else {
- batch.iterator.filter { ctx =>
- val env = ctx.envelope
- offsetStore.isInflight(env)
- }.toVector
- }
-
- if (acceptedContexts.isEmpty) {
- FutureDone
- } else {
- val offsets = acceptedContexts.map(ctx => R2dbcProjectionImpl.extractOffsetPidSeqNr(ctx.offset, ctx.envelope))
- offsetStore
- .saveOffsets(offsets)
- .map { done =>
- val batchSize = acceptedContexts.map { _.groupSize }.sum
- val last = acceptedContexts.last
- try {
- statusObserver.offsetProgress(projectionId, last.envelope)
- } catch {
- case NonFatal(_) => // ignore
- }
- getTelemetry().onOffsetStored(batchSize)
- done
- }
- }
- }
-
- private[projection] def newRunningInstance(): RunningProjection =
- new R2dbcRunningProjection(RunningProjection.withBackoff(() => this.mappedSource(), settings), this)
- }
-
- private class R2dbcRunningProjection(source: Source[Done, _], projectionState: R2dbcInternalProjectionState)(implicit
- system: ActorSystem[_])
- extends RunningProjection
- with RunningProjectionManagement[Offset] {
-
- private val streamDone = source.run()
-
- override def stop(): Future[Done] = {
- projectionState.killSwitch.shutdown()
- // if the handler is retrying it will be aborted by this,
- // otherwise the stream would not be completed by the killSwitch until after all retries
- projectionState.abort.failure(AbortProjectionException)
- streamDone
- }
-
- // RunningProjectionManagement
- override def getOffset(): Future[Option[Offset]] = {
- offsetStore.getOffset()
- }
-
- // RunningProjectionManagement
- override def setOffset(offset: Option[Offset]): Future[Done] = {
- offset match {
- case Some(o) => offsetStore.managementSetOffset(o)
- case None => offsetStore.managementClearOffset()
- }
- }
-
- // RunningProjectionManagement
- override def getManagementState(): Future[Option[ManagementState]] =
- offsetStore.readManagementState()
-
- // RunningProjectionManagement
- override def setPaused(paused: Boolean): Future[Done] =
- offsetStore.savePaused(paused)
- }
-
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/mysql/MySQLR2dbcOffsetStore.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/mysql/MySQLR2dbcOffsetStore.scala
deleted file mode 100644
index fe988c0..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/internal/mysql/MySQLR2dbcOffsetStore.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pekko.projection.r2dbc.internal.mysql
-
-import java.time.Clock
-
-import org.apache.pekko
-import pekko.actor.typed.ActorSystem
-import pekko.annotation.InternalApi
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
-import pekko.projection.BySlicesSourceProvider
-import pekko.projection.ProjectionId
-import pekko.projection.r2dbc.R2dbcProjectionSettings
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore
-
-/**
- * INTERNAL API
- */
-@InternalApi
-private[projection] class MySQLR2dbcOffsetStore(
- projectionId: ProjectionId,
- sourceProvider: Option[BySlicesSourceProvider],
- system: ActorSystem[_],
- settings: R2dbcProjectionSettings,
- r2dbcExecutor: R2dbcExecutor,
- clock: Clock = Clock.systemUTC())
- extends R2dbcOffsetStore(projectionId, sourceProvider, system, settings, r2dbcExecutor, clock) {
-
- override lazy val timestampSql: String = "NOW(6)"
-
- override val upsertOffsetSql: String = sql"""
- INSERT INTO $offsetTable
- (projection_name, projection_key, current_offset, manifest, mergeable, last_updated)
- VALUES (?,?,?,?,?,?) AS excluded
- ON DUPLICATE KEY UPDATE
- current_offset = excluded.current_offset,
- manifest = excluded.manifest,
- mergeable = excluded.mergeable,
- last_updated = excluded.last_updated"""
-
- override val updateManagementStateSql: String = sql"""
- INSERT INTO $managementTable
- (projection_name, projection_key, paused, last_updated)
- VALUES (?,?,?,?) AS excluded
- ON DUPLICATE KEY UPDATE
- paused = excluded.paused,
- last_updated = excluded.last_updated"""
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcHandler.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcHandler.scala
deleted file mode 100644
index 6657061..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcHandler.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.javadsl
-
-import java.util.concurrent.CompletableFuture
-import java.util.concurrent.CompletionStage
-import java.util.function.BiFunction
-
-import org.apache.pekko
-import pekko.Done
-import pekko.annotation.ApiMayChange
-import pekko.annotation.InternalApi
-import pekko.projection.javadsl.HandlerLifecycle
-
-object R2dbcHandler {
-
- /**
- * INTERNAL API
- */
- @InternalApi
- private class HandlerFunction[Envelope](handler: BiFunction[R2dbcSession, Envelope, CompletionStage[Done]])
- extends R2dbcHandler[Envelope] {
- override def process(session: R2dbcSession, envelope: Envelope): CompletionStage[Done] =
- handler.apply(session, envelope)
- }
-
- /**
- * Handler that can be defined from a simple function
- */
- def fromFunction[Envelope](
- handler: BiFunction[R2dbcSession, Envelope, CompletionStage[Done]]): R2dbcHandler[Envelope] =
- new HandlerFunction(handler)
-
-}
-
-/**
- * Implement this interface for the Envelope handler for R2DBC Projections.
- *
- * It can be stateful, with variables and mutable data structures. It is invoked by the `Projection` machinery one
- * envelope at a time and visibility guarantees between the invocations are handled automatically, i.e. no volatile or
- * other concurrency primitives are needed for managing the state.
- *
- * Supported error handling strategies for when processing an `Envelope` fails can be defined in configuration or using
- * the `withRecoveryStrategy` method of a `Projection` implementation.
- */
-@ApiMayChange
-abstract class R2dbcHandler[Envelope] extends HandlerLifecycle {
-
- /**
- * The `process` method is invoked for each `Envelope`. Each time a new `Connection` is passed with a new open
- * transaction. You can use `createStatement`, `update` and other methods provided by the [[R2dbcSession]]. The
- * results of several statements can be combined with `CompletionStage` composition (e.g. `thenCompose`). The
- * transaction will be automatically committed or rolled back when the returned `CompletionStage` is completed.
- *
- * One envelope is processed at a time. It will not be invoked with the next envelope until after this method returns
- * and the returned `CompletionStage` is completed.
- */
- @throws(classOf[Exception])
- def process(session: R2dbcSession, envelope: Envelope): CompletionStage[Done]
-
- /**
- * Invoked when the projection is starting, before first envelope is processed. Can be overridden to implement
- * initialization. It is also called when the `Projection` is restarted after a failure.
- */
- def start(): CompletionStage[Done] =
- CompletableFuture.completedFuture(Done)
-
- /**
- * Invoked when the projection has been stopped. Can be overridden to implement resource cleanup. It is also called
- * when the `Projection` is restarted after a failure.
- */
- def stop(): CompletionStage[Done] =
- CompletableFuture.completedFuture(Done)
-
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcProjection.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcProjection.scala
deleted file mode 100644
index b70441c..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcProjection.scala
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.javadsl
-
-import java.util.Optional
-import java.util.function.Supplier
-
-import scala.jdk.OptionConverters._
-
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.typed.ActorSystem
-import pekko.annotation.ApiMayChange
-import pekko.projection.BySlicesSourceProvider
-import pekko.projection.ProjectionContext
-import pekko.projection.ProjectionId
-import pekko.projection.internal.GroupedHandlerAdapter
-import pekko.projection.internal.HandlerAdapter
-import pekko.projection.internal.SourceProviderAdapter
-import pekko.projection.javadsl.AtLeastOnceFlowProjection
-import pekko.projection.javadsl.AtLeastOnceProjection
-import pekko.projection.javadsl.ExactlyOnceProjection
-import pekko.projection.javadsl.GroupedProjection
-import pekko.projection.javadsl.Handler
-import pekko.projection.javadsl.SourceProvider
-import pekko.projection.r2dbc.R2dbcProjectionSettings
-import pekko.projection.r2dbc.internal.BySliceSourceProviderAdapter
-import pekko.projection.r2dbc.internal.R2dbcGroupedHandlerAdapter
-import pekko.projection.r2dbc.internal.R2dbcHandlerAdapter
-import pekko.projection.r2dbc.scaladsl
-import pekko.stream.javadsl.FlowWithContext
-
-@ApiMayChange
-object R2dbcProjection {
-
- /**
- * Create a [[pekko.projection.Projection]] with exactly-once processing semantics.
- *
- * It stores the offset in a relational database table using R2DBC in the same transaction as the user defined
- * `handler`.
- */
- def exactlyOnce[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Optional[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: Supplier[R2dbcHandler[Envelope]],
- system: ActorSystem[_]): ExactlyOnceProjection[Offset, Envelope] = {
- scaladsl.R2dbcProjection
- .exactlyOnce[Offset, Envelope](
- projectionId,
- settings.toScala,
- adaptSourceProvider(sourceProvider),
- () => new R2dbcHandlerAdapter(handler.get()))(system)
- .asInstanceOf[ExactlyOnceProjection[Offset, Envelope]]
- }
-
- /**
- * Create a [[pekko.projection.Projection]] with at-least-once processing semantics.
- *
- * It stores the offset in a relational database table using R2DBC after the `handler` has processed the envelope.
- * This means that if the projection is restarted from previously stored offset then some elements may be processed
- * more than once.
- *
- * The [[R2dbcHandler.process]] in `handler` will be wrapped in a transaction. The transaction will be committed after
- * invoking [[R2dbcHandler.process]].
- *
- * The offset is stored after a time window, or limited by a number of envelopes, whatever happens first. This window
- * can be defined with [[AtLeastOnceProjection.withSaveOffset]] of the returned `AtLeastOnceProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.at-least-once`.
- */
- def atLeastOnce[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Optional[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: Supplier[R2dbcHandler[Envelope]],
- system: ActorSystem[_]): AtLeastOnceProjection[Offset, Envelope] = {
- scaladsl.R2dbcProjection
- .atLeastOnce[Offset, Envelope](
- projectionId,
- settings.toScala,
- adaptSourceProvider(sourceProvider),
- () => new R2dbcHandlerAdapter(handler.get()))(system)
- .asInstanceOf[AtLeastOnceProjection[Offset, Envelope]]
- }
-
- /**
- * Create a [[pekko.projection.Projection]] with at-least-once processing semantics.
- *
- * Compared to [[R2dbcProjection.atLeastOnce]] the [[Handler]] is not storing the projected result in the database,
- * but is integrating with something else.
- *
- * It stores the offset in a relational database table using R2DBC after the `handler` has processed the envelope.
- * This means that if the projection is restarted from previously stored offset then some elements may be processed
- * more than once.
- *
- * The offset is stored after a time window, or limited by a number of envelopes, whatever happens first. This window
- * can be defined with [[AtLeastOnceProjection.withSaveOffset]] of the returned `AtLeastOnceProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.at-least-once`.
- */
- def atLeastOnceAsync[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Optional[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: Supplier[Handler[Envelope]],
- system: ActorSystem[_]): AtLeastOnceProjection[Offset, Envelope] = {
-
- scaladsl.R2dbcProjection
- .atLeastOnceAsync[Offset, Envelope](
- projectionId,
- settings.toScala,
- adaptSourceProvider(sourceProvider),
- () => HandlerAdapter(handler.get()))(system)
- .asInstanceOf[AtLeastOnceProjection[Offset, Envelope]]
- }
-
- /**
- * Create a [[pekko.projection.Projection]] that groups envelopes and calls the `handler` with a group of `Envelopes`.
- * The envelopes are grouped within a time window, or limited by a number of envelopes, whatever happens first. This
- * window can be defined with [[GroupedProjection.withGroup]] of the returned `GroupedProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.grouped`.
- *
- * It stores the offset in a relational database table using R2DBC in the same transaction as the user defined
- * `handler`.
- */
- def groupedWithin[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Optional[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: Supplier[R2dbcHandler[java.util.List[Envelope]]],
- system: ActorSystem[_]): GroupedProjection[Offset, Envelope] = {
- scaladsl.R2dbcProjection
- .groupedWithin[Offset, Envelope](
- projectionId,
- settings.toScala,
- adaptSourceProvider(sourceProvider),
- () => new R2dbcGroupedHandlerAdapter(handler.get()))(system)
- .asInstanceOf[GroupedProjection[Offset, Envelope]]
- }
-
- /**
- * Create a [[pekko.projection.Projection]] that groups envelopes and calls the `handler` with a group of `Envelopes`.
- * The envelopes are grouped within a time window, or limited by a number of envelopes, whatever happens first. This
- * window can be defined with [[GroupedProjection.withGroup]] of the returned `GroupedProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.grouped`.
- *
- * Compared to [[R2dbcProjection.groupedWithin]] the [[Handler]] is not storing the projected result in the database,
- * but is integrating with something else.
- *
- * It stores the offset in a relational database table using R2DBC immediately after the `handler` has processed the
- * envelopes, but that is still with at-least-once processing semantics. This means that if the projection is
- * restarted from previously stored offset the previous group of envelopes may be processed more than once.
- */
- def groupedWithinAsync[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Optional[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: Supplier[Handler[java.util.List[Envelope]]],
- system: ActorSystem[_]): GroupedProjection[Offset, Envelope] = {
- scaladsl.R2dbcProjection
- .groupedWithinAsync[Offset, Envelope](
- projectionId,
- settings.toScala,
- adaptSourceProvider(sourceProvider),
- () => new GroupedHandlerAdapter(handler.get()))(system)
- .asInstanceOf[GroupedProjection[Offset, Envelope]]
- }
-
- /**
- * Create a [[pekko.projection.Projection]] with a [[FlowWithContext]] as the envelope handler. It has at-least-once
- * processing semantics.
- *
- * The flow should emit a `Done` element for each completed envelope. The offset of the envelope is carried in the
- * context of the `FlowWithContext` and is stored in Cassandra when corresponding `Done` is emitted. Since the offset
- * is stored after processing the envelope it means that if the projection is restarted from previously stored offset
- * then some envelopes may be processed more than once.
- *
- * If the flow filters out envelopes the corresponding offset will not be stored, and such envelope will be processed
- * again if the projection is restarted and no later offset was stored.
- *
- * The flow should not duplicate emitted envelopes (`mapConcat`) with same offset, because then it can result in that
- * the first offset is stored and when the projection is restarted that offset is considered completed even though
- * more of the duplicated enveloped were never processed.
- *
- * The flow must not reorder elements, because the offsets may be stored in the wrong order and and when the
- * projection is restarted all envelopes up to the latest stored offset are considered completed even though some of
- * them may not have been processed. This is the reason the flow is restricted to `FlowWithContext` rather than
- * ordinary `Flow`.
- */
- def atLeastOnceFlow[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Optional[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _],
- system: ActorSystem[_]): AtLeastOnceFlowProjection[Offset, Envelope] = {
- scaladsl.R2dbcProjection
- .atLeastOnceFlow[Offset, Envelope](
- projectionId,
- settings.toScala,
- adaptSourceProvider(sourceProvider),
- handler.asScala)(system)
- .asInstanceOf[AtLeastOnceFlowProjection[Offset, Envelope]]
- }
-
- private def adaptSourceProvider[Offset, Envelope](sourceProvider: SourceProvider[Offset, Envelope]) =
- sourceProvider match {
- case _: BySlicesSourceProvider => new BySliceSourceProviderAdapter(sourceProvider)
- case _ => new SourceProviderAdapter(sourceProvider)
- }
-
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcSession.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcSession.scala
deleted file mode 100644
index ee20292..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/javadsl/R2dbcSession.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.javadsl
-
-import java.util.Optional
-import java.util.concurrent.CompletionStage
-
-import scala.concurrent.ExecutionContext
-import scala.jdk.CollectionConverters._
-import scala.jdk.FutureConverters._
-import scala.jdk.OptionConverters._
-
-import org.apache.pekko
-import pekko.actor.typed.ActorSystem
-import pekko.annotation.ApiMayChange
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import io.r2dbc.spi.Connection
-import io.r2dbc.spi.Row
-import io.r2dbc.spi.Statement
-
-@ApiMayChange
-final class R2dbcSession(connection: Connection)(implicit ec: ExecutionContext, system: ActorSystem[_]) {
-
- def createStatement(sql: String): Statement =
- connection.createStatement(sql)
-
- def updateOne(statement: Statement): CompletionStage[java.lang.Long] =
- R2dbcExecutor.updateOneInTx(statement).map(java.lang.Long.valueOf)(ExecutionContext.parasitic).asJava
-
- def update(statements: java.util.List[Statement]): CompletionStage[java.util.List[java.lang.Long]] =
- R2dbcExecutor.updateInTx(statements.asScala.toVector).map(results =>
- results.map(java.lang.Long.valueOf).asJava).asJava
-
- def selectOne[A](statement: Statement)(mapRow: Row => A): CompletionStage[Optional[A]] =
- R2dbcExecutor.selectOneInTx(statement, mapRow).map(_.toJava)(ExecutionContext.parasitic).asJava
-
- def select[A](statement: Statement)(mapRow: Row => A): CompletionStage[java.util.List[A]] =
- R2dbcExecutor.selectInTx(statement, mapRow).map(_.asJava).asJava
-
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcHandler.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcHandler.scala
deleted file mode 100644
index 3657cbd..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcHandler.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.scaladsl
-
-import scala.concurrent.Future
-
-import org.apache.pekko
-import pekko.Done
-import pekko.annotation.ApiMayChange
-import pekko.projection.scaladsl.HandlerLifecycle
-
-/**
- * Implement this interface for the Envelope handler for R2DBC Projections.
- *
- * It can be stateful, with variables and mutable data structures. It is invoked by the `Projection` machinery one
- * envelope at a time and visibility guarantees between the invocations are handled automatically, i.e. no volatile or
- * other concurrency primitives are needed for managing the state.
- *
- * Supported error handling strategies for when processing an `Envelope` fails can be defined in configuration or using
- * the `withRecoveryStrategy` method of a `Projection` implementation.
- */
-@ApiMayChange
-trait R2dbcHandler[Envelope] extends HandlerLifecycle {
-
- /**
- * The `process` method is invoked for each `Envelope`. Each time a new `Connection` is passed with a new open
- * transaction. You can use `createStatement`, `update` and other methods provided by the [[R2dbcSession]]. The
- * results of several statements can be combined with `Future` composition (e.g. `flatMap`). The transaction will be
- * automatically committed or rolled back when the returned `Future[Done]` is completed.
- *
- * One envelope is processed at a time. It will not be invoked with the next envelope until after this method returns
- * and the returned `Future[Done]` is completed.
- */
- def process(session: R2dbcSession, envelope: Envelope): Future[Done]
-
-}
-
-@ApiMayChange
-object R2dbcHandler {
-
- /** R2dbcHandler that can be define from a simple function */
- private class R2dbcHandlerFunction[Envelope](handler: (R2dbcSession, Envelope) => Future[Done])
- extends R2dbcHandler[Envelope] {
-
- override def process(session: R2dbcSession, envelope: Envelope): Future[Done] =
- handler(session, envelope)
- }
-
- def apply[Envelope](handler: (R2dbcSession, Envelope) => Future[Done]): R2dbcHandler[Envelope] =
- new R2dbcHandlerFunction(handler)
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcProjection.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcProjection.scala
deleted file mode 100644
index 0eb71cf..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcProjection.scala
+++ /dev/null
@@ -1,401 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.scaladsl
-
-import scala.collection.immutable
-import scala.concurrent.duration.Duration
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.typed.ActorSystem
-import pekko.annotation.ApiMayChange
-import pekko.persistence.r2dbc.ConnectionFactoryProvider
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import pekko.projection.BySlicesSourceProvider
-import pekko.projection.ProjectionContext
-import pekko.projection.ProjectionId
-import pekko.projection.internal.AtLeastOnce
-import pekko.projection.internal.ExactlyOnce
-import pekko.projection.internal.FlowHandlerStrategy
-import pekko.projection.internal.GroupedHandlerStrategy
-import pekko.projection.internal.NoopStatusObserver
-import pekko.projection.internal.SingleHandlerStrategy
-import pekko.projection.r2dbc.R2dbcProjectionSettings
-import pekko.projection.r2dbc.internal.R2dbcProjectionImpl
-import pekko.projection.scaladsl.AtLeastOnceFlowProjection
-import pekko.projection.scaladsl.AtLeastOnceProjection
-import pekko.projection.scaladsl.ExactlyOnceProjection
-import pekko.projection.scaladsl.GroupedProjection
-import pekko.projection.scaladsl.Handler
-import pekko.projection.scaladsl.SourceProvider
-import pekko.stream.scaladsl.FlowWithContext
-import com.typesafe.config.Config
-import com.typesafe.config.ConfigFactory
-import io.r2dbc.spi.ConnectionFactory
-
-@ApiMayChange
-object R2dbcProjection {
-
- /**
- * Create a [[pekko.projection.Projection]] with exactly-once processing semantics.
- *
- * It stores the offset in a relational database table using R2DBC in the same transaction as the user defined
- * `handler`.
- */
- def exactlyOnce[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => R2dbcHandler[Envelope])(implicit
- system: ActorSystem[_]): ExactlyOnceProjection[Offset, Envelope] =
- exactlyOnce(projectionId, ConfigFactory.empty(), settings, sourceProvider, handler)
-
- /** @since 2.0.0 */
- def exactlyOnce[Offset, Envelope](
- projectionId: ProjectionId,
- config: Config,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => R2dbcHandler[Envelope])(implicit
- system: ActorSystem[_]): ExactlyOnceProjection[Offset, Envelope] = {
- val r2dbcSettings = settings.getOrElse(R2dbcProjectionSettings(config, system))
- val connFactory = connectionFactory(system, config, r2dbcSettings)
- val offsetStore =
- R2dbcProjectionImpl.createOffsetStore(
- projectionId,
- timestampOffsetBySlicesSourceProvider(sourceProvider),
- r2dbcSettings,
- connFactory)
- val r2dbcExecutor = new R2dbcExecutor(connFactory, R2dbcProjectionImpl.log, r2dbcSettings.logDbCallsExceeding)(
- system.executionContext,
- system)
-
- val adaptedHandler =
- R2dbcProjectionImpl.adaptedHandlerForExactlyOnce(sourceProvider, handler, offsetStore, r2dbcExecutor)(
- system.executionContext,
- system)
-
- new R2dbcProjectionImpl(
- projectionId,
- r2dbcSettings,
- settingsOpt = None,
- sourceProvider,
- restartBackoffOpt = None,
- offsetStrategy = ExactlyOnce(),
- handlerStrategy = SingleHandlerStrategy(adaptedHandler),
- NoopStatusObserver,
- offsetStore)
- }
-
- /**
- * Create a [[pekko.projection.Projection]] with at-least-once processing semantics.
- *
- * It stores the offset in a relational database table using R2DBC after the `handler` has processed the envelope.
- * This means that if the projection is restarted from previously stored offset then some elements may be processed
- * more than once.
- *
- * The [[R2dbcHandler.process()]] in `handler` will be wrapped in a transaction. The transaction will be committed
- * after invoking [[R2dbcHandler.process()]].
- *
- * The offset is stored after a time window, or limited by a number of envelopes, whatever happens first. This window
- * can be defined with [[AtLeastOnceProjection.withSaveOffset]] of the returned `AtLeastOnceProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.at-least-once`.
- */
- def atLeastOnce[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => R2dbcHandler[Envelope])(implicit
- system: ActorSystem[_]): AtLeastOnceProjection[Offset, Envelope] =
- atLeastOnce(projectionId, ConfigFactory.empty(), settings, sourceProvider, handler)
-
- /** @since 2.0.0 */
- def atLeastOnce[Offset, Envelope](
- projectionId: ProjectionId,
- config: Config,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => R2dbcHandler[Envelope])(implicit
- system: ActorSystem[_]): AtLeastOnceProjection[Offset, Envelope] = {
-
- val r2dbcSettings = settings.getOrElse(R2dbcProjectionSettings(config, system))
- val connFactory = connectionFactory(system, config, r2dbcSettings)
- val offsetStore =
- R2dbcProjectionImpl.createOffsetStore(
- projectionId,
- timestampOffsetBySlicesSourceProvider(sourceProvider),
- r2dbcSettings,
- connFactory)
- val r2dbcExecutor = new R2dbcExecutor(connFactory, R2dbcProjectionImpl.log, r2dbcSettings.logDbCallsExceeding)(
- system.executionContext,
- system)
-
- val adaptedHandler =
- R2dbcProjectionImpl.adaptedHandlerForAtLeastOnce(sourceProvider, handler, offsetStore, r2dbcExecutor)(
- system.executionContext,
- system)
-
- new R2dbcProjectionImpl(
- projectionId,
- r2dbcSettings,
- settingsOpt = None,
- sourceProvider,
- restartBackoffOpt = None,
- offsetStrategy = AtLeastOnce(),
- handlerStrategy = SingleHandlerStrategy(adaptedHandler),
- NoopStatusObserver,
- offsetStore)
- }
-
- /**
- * Create a [[pekko.projection.Projection]] with at-least-once processing semantics.
- *
- * Compared to [[R2dbcProjection.atLeastOnce]] the [[Handler]] is not storing the projected result in the database,
- * but is integrating with something else.
- *
- * It stores the offset in a relational database table using R2DBC after the `handler` has processed the envelope.
- * This means that if the projection is restarted from previously stored offset then some elements may be processed
- * more than once.
- *
- * The offset is stored after a time window, or limited by a number of envelopes, whatever happens first. This window
- * can be defined with [[AtLeastOnceProjection.withSaveOffset]] of the returned `AtLeastOnceProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.at-least-once`.
- */
- def atLeastOnceAsync[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => Handler[Envelope])(implicit system: ActorSystem[_]): AtLeastOnceProjection[Offset, Envelope] =
- atLeastOnceAsync(projectionId, ConfigFactory.empty(), settings, sourceProvider, handler)
-
- /** @since 2.0.0 */
- def atLeastOnceAsync[Offset, Envelope](
- projectionId: ProjectionId,
- config: Config,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => Handler[Envelope])(implicit system: ActorSystem[_]): AtLeastOnceProjection[Offset, Envelope] = {
-
- val r2dbcSettings = settings.getOrElse(R2dbcProjectionSettings(config, system))
- val connFactory = connectionFactory(system, config, r2dbcSettings)
- val offsetStore =
- R2dbcProjectionImpl.createOffsetStore(
- projectionId,
- timestampOffsetBySlicesSourceProvider(sourceProvider),
- r2dbcSettings,
- connFactory)
-
- val adaptedHandler =
- R2dbcProjectionImpl.adaptedHandlerForAtLeastOnceAsync(sourceProvider, handler, offsetStore)(
- system.executionContext,
- system)
-
- new R2dbcProjectionImpl(
- projectionId,
- r2dbcSettings,
- settingsOpt = None,
- sourceProvider,
- restartBackoffOpt = None,
- offsetStrategy = AtLeastOnce(),
- handlerStrategy = SingleHandlerStrategy(adaptedHandler),
- NoopStatusObserver,
- offsetStore)
- }
-
- /**
- * Create a [[pekko.projection.Projection]] that groups envelopes and calls the `handler` with a group of `Envelopes`.
- * The envelopes are grouped within a time window, or limited by a number of envelopes, whatever happens first. This
- * window can be defined with [[GroupedProjection.withGroup]] of the returned `GroupedProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.grouped`.
- *
- * It stores the offset in a relational database table using R2DBC in the same transaction as the user defined
- * `handler`.
- */
- def groupedWithin[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => R2dbcHandler[immutable.Seq[Envelope]])(implicit
- system: ActorSystem[_]): GroupedProjection[Offset, Envelope] =
- groupedWithin(projectionId, ConfigFactory.empty(), settings, sourceProvider, handler)
-
- /** @since 2.0.0 */
- def groupedWithin[Offset, Envelope](
- projectionId: ProjectionId,
- config: Config,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => R2dbcHandler[immutable.Seq[Envelope]])(implicit
- system: ActorSystem[_]): GroupedProjection[Offset, Envelope] = {
-
- val r2dbcSettings = settings.getOrElse(R2dbcProjectionSettings(config, system))
- val connFactory = connectionFactory(system, config, r2dbcSettings)
- val offsetStore =
- R2dbcProjectionImpl.createOffsetStore(
- projectionId,
- timestampOffsetBySlicesSourceProvider(sourceProvider),
- r2dbcSettings,
- connFactory)
- val r2dbcExecutor = new R2dbcExecutor(connFactory, R2dbcProjectionImpl.log, r2dbcSettings.logDbCallsExceeding)(
- system.executionContext,
- system)
-
- val adaptedHandler =
- R2dbcProjectionImpl.adaptedHandlerForGrouped(sourceProvider, handler, offsetStore, r2dbcExecutor)(
- system.executionContext,
- system)
-
- new R2dbcProjectionImpl(
- projectionId,
- r2dbcSettings,
- settingsOpt = None,
- sourceProvider,
- restartBackoffOpt = None,
- offsetStrategy = ExactlyOnce(),
- handlerStrategy = GroupedHandlerStrategy(adaptedHandler),
- NoopStatusObserver,
- offsetStore)
- }
-
- /**
- * Create a [[pekko.projection.Projection]] that groups envelopes and calls the `handler` with a group of `Envelopes`.
- * The envelopes are grouped within a time window, or limited by a number of envelopes, whatever happens first. This
- * window can be defined with [[GroupedProjection.withGroup]] of the returned `GroupedProjection`. The default
- * settings for the window is defined in configuration section `pekko.projection.grouped`.
- *
- * Compared to [[R2dbcProjection.groupedWithin]] the [[Handler]] is not storing the projected result in the database,
- * but is integrating with something else.
- *
- * It stores the offset in a relational database table using R2DBC immediately after the `handler` has processed the
- * envelopes, but that is still with at-least-once processing semantics. This means that if the projection is
- * restarted from previously stored offset the previous group of envelopes may be processed more than once.
- */
- def groupedWithinAsync[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => Handler[immutable.Seq[Envelope]])(implicit
- system: ActorSystem[_]): GroupedProjection[Offset, Envelope] =
- groupedWithinAsync(projectionId, ConfigFactory.empty(), settings, sourceProvider, handler)
-
- /** @since 2.0.0 */
- def groupedWithinAsync[Offset, Envelope](
- projectionId: ProjectionId,
- config: Config,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: () => Handler[immutable.Seq[Envelope]])(implicit
- system: ActorSystem[_]): GroupedProjection[Offset, Envelope] = {
-
- val r2dbcSettings = settings.getOrElse(R2dbcProjectionSettings(config, system))
- val connFactory = connectionFactory(system, config, r2dbcSettings)
- val offsetStore =
- R2dbcProjectionImpl.createOffsetStore(
- projectionId,
- timestampOffsetBySlicesSourceProvider(sourceProvider),
- r2dbcSettings,
- connFactory)
-
- val adaptedHandler =
- R2dbcProjectionImpl.adaptedHandlerForGroupedAsync(sourceProvider, handler, offsetStore)(
- system.executionContext,
- system)
-
- new R2dbcProjectionImpl(
- projectionId,
- r2dbcSettings,
- settingsOpt = None,
- sourceProvider,
- restartBackoffOpt = None,
- offsetStrategy = AtLeastOnce(afterEnvelopes = Some(1), orAfterDuration = Some(Duration.Zero)),
- handlerStrategy = GroupedHandlerStrategy(adaptedHandler),
- NoopStatusObserver,
- offsetStore)
- }
-
- /**
- * Create a [[pekko.projection.Projection]] with a [[FlowWithContext]] as the envelope handler. It has at-least-once
- * processing semantics.
- *
- * The flow should emit a `Done` element for each completed envelope. The offset of the envelope is carried in the
- * context of the `FlowWithContext` and is stored in Cassandra when corresponding `Done` is emitted. Since the offset
- * is stored after processing the envelope it means that if the projection is restarted from previously stored offset
- * then some envelopes may be processed more than once.
- *
- * If the flow filters out envelopes the corresponding offset will not be stored, and such envelope will be processed
- * again if the projection is restarted and no later offset was stored.
- *
- * The flow should not duplicate emitted envelopes (`mapConcat`) with same offset, because then it can result in that
- * the first offset is stored and when the projection is restarted that offset is considered completed even though
- * more of the duplicated enveloped were never processed.
- *
- * The flow must not reorder elements, because the offsets may be stored in the wrong order and and when the
- * projection is restarted all envelopes up to the latest stored offset are considered completed even though some of
- * them may not have been processed. This is the reason the flow is restricted to `FlowWithContext` rather than
- * ordinary `Flow`.
- */
- def atLeastOnceFlow[Offset, Envelope](
- projectionId: ProjectionId,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _])(implicit
- system: ActorSystem[_]): AtLeastOnceFlowProjection[Offset, Envelope] =
- atLeastOnceFlow(projectionId, ConfigFactory.empty(), settings, sourceProvider, handler)
-
- /** @since 2.0.0 */
- def atLeastOnceFlow[Offset, Envelope](
- projectionId: ProjectionId,
- config: Config,
- settings: Option[R2dbcProjectionSettings],
- sourceProvider: SourceProvider[Offset, Envelope],
- handler: FlowWithContext[Envelope, ProjectionContext, Done, ProjectionContext, _])(implicit
- system: ActorSystem[_]): AtLeastOnceFlowProjection[Offset, Envelope] = {
-
- val r2dbcSettings = settings.getOrElse(R2dbcProjectionSettings(config, system))
- val connFactory = connectionFactory(system, config, r2dbcSettings)
- val offsetStore =
- R2dbcProjectionImpl.createOffsetStore(
- projectionId,
- timestampOffsetBySlicesSourceProvider(sourceProvider),
- r2dbcSettings,
- connFactory)
-
- val adaptedHandler =
- R2dbcProjectionImpl.adaptedHandlerForFlow(sourceProvider, handler, offsetStore)(system.executionContext, system)
-
- new R2dbcProjectionImpl(
- projectionId,
- r2dbcSettings,
- settingsOpt = None,
- sourceProvider,
- restartBackoffOpt = None,
- offsetStrategy = AtLeastOnce(),
- handlerStrategy = FlowHandlerStrategy(adaptedHandler),
- NoopStatusObserver,
- offsetStore)
- }
-
- private def connectionFactory(system: ActorSystem[_], config: Config, r2dbcSettings: R2dbcProjectionSettings)
- : ConnectionFactory = {
- ConnectionFactoryProvider(system).connectionFactoryFor(r2dbcSettings.useConnectionFactory, config)
- }
-
- private def timestampOffsetBySlicesSourceProvider(
- sourceProvider: SourceProvider[_, _]): Option[BySlicesSourceProvider] = {
- sourceProvider match {
- case provider: BySlicesSourceProvider => Some(provider)
- case _ => None // source provider is not using slices
- }
- }
-
-}
diff --git a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcSession.scala b/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcSession.scala
deleted file mode 100644
index d7b12ab..0000000
--- a/projection/src/main/scala/org/apache/pekko/projection/r2dbc/scaladsl/R2dbcSession.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc.scaladsl
-
-import scala.collection.immutable
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-
-import org.apache.pekko
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import io.r2dbc.spi.Connection
-import io.r2dbc.spi.Row
-import io.r2dbc.spi.Statement
-
-final class R2dbcSession(val connection: Connection)(implicit val ec: ExecutionContext, val system: ActorSystem[_]) {
-
- def createStatement(sql: String): Statement =
- connection.createStatement(sql)
-
- def updateOne(statement: Statement): Future[Long] =
- R2dbcExecutor.updateOneInTx(statement)
-
- def update(statements: immutable.IndexedSeq[Statement]): Future[immutable.IndexedSeq[Long]] =
- R2dbcExecutor.updateInTx(statements)
-
- def selectOne[A](statement: Statement)(mapRow: Row => A): Future[Option[A]] =
- R2dbcExecutor.selectOneInTx(statement, mapRow)
-
- def select[A](statement: Statement)(mapRow: Row => A): Future[immutable.IndexedSeq[A]] =
- R2dbcExecutor.selectInTx(statement, mapRow)
-
-}
diff --git a/projection/src/test/resources/logback-test.xml b/projection/src/test/resources/logback-test.xml
deleted file mode 100644
index e2ae561..0000000
--- a/projection/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>[%date{ISO8601}] [%level] [%logger] [%X{pekkoAddress}] [%marker] [%thread] - %msg%n</pattern>
- </encoder>
- </appender>
-
- <appender name="CapturingAppender" class="org.apache.pekko.actor.testkit.typed.internal.CapturingAppender"/>
- <logger name="org.apache.pekko.actor.testkit.typed.internal.CapturingAppenderDelegate">
- <appender-ref ref="STDOUT"/>
- </logger>
-
- <logger name="org.apache.pekko.projection.r2dbc" level="DEBUG" />
- <logger name="org.apache.pekko.persistence.r2dbc" level="DEBUG" />
-<!-- <logger name="io.r2dbc.postgresql.QUERY" level="DEBUG" />-->
-<!-- <logger name="io.r2dbc.pool" level="DEBUG" />-->
-
-
- <root level="INFO">
- <appender-ref ref="CapturingAppender"/>
-<!-- <appender-ref ref="STDOUT"/>-->
- </root>
-
-</configuration>
diff --git a/projection/src/test/scala/org/apache/pekko/projection/TestStatusObserver.scala b/projection/src/test/scala/org/apache/pekko/projection/TestStatusObserver.scala
deleted file mode 100644
index 8dfe460..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/TestStatusObserver.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection
-
-import org.apache.pekko.actor.typed.ActorRef
-
-// FIXME this is an exact copy of projection-core-test/src/test/scala/org/apache/pekko/projection/TestStatusObserver.scala
-
-object TestStatusObserver {
- sealed trait Status
-
- case object Started extends Status
- case object Failed extends Status
- case object Stopped extends Status
-
- final case class OffsetProgress[Envelope](envelope: Envelope) extends Status
-
- trait EnvelopeProgress[Envelope] extends Status
- final case class Before[Envelope](envelope: Envelope) extends EnvelopeProgress[Envelope]
- final case class After[Envelope](envelope: Envelope) extends EnvelopeProgress[Envelope]
-
- final case class Err[Envelope](env: Envelope, cause: Throwable) extends Status {
- // don't include cause message in equals
- override def equals(obj: Any): Boolean = obj match {
- case Err(`env`, e) => e.getClass == cause.getClass
- case _ => false
- }
-
- override def hashCode(): Int = env.hashCode()
- }
-}
-
-class TestStatusObserver[Envelope](
- probe: ActorRef[TestStatusObserver.Status],
- lifecycle: Boolean = false,
- offsetProgressProbe: Option[ActorRef[TestStatusObserver.OffsetProgress[Envelope]]] = None,
- beforeEnvelopeProbe: Option[ActorRef[TestStatusObserver.Before[Envelope]]] = None,
- afterEnvelopeProbe: Option[ActorRef[TestStatusObserver.After[Envelope]]] = None)
- extends StatusObserver[Envelope] {
- import TestStatusObserver._
-
- override def started(projectionId: ProjectionId): Unit = {
- if (lifecycle)
- probe ! Started
- }
-
- override def failed(projectionId: ProjectionId, cause: Throwable): Unit = {
- if (lifecycle)
- probe ! Failed
- }
-
- override def stopped(projectionId: ProjectionId): Unit = {
- if (lifecycle)
- probe ! Stopped
- }
-
- override def beforeProcess(projectionId: ProjectionId, envelope: Envelope): Unit = {
- beforeEnvelopeProbe.foreach(_ ! Before(envelope))
- }
-
- override def afterProcess(projectionId: ProjectionId, envelope: Envelope): Unit = {
- afterEnvelopeProbe.foreach(_ ! After(envelope))
- }
-
- override def offsetProgress(projectionId: ProjectionId, envelope: Envelope): Unit = {
- offsetProgressProbe.foreach(_ ! OffsetProgress(envelope))
- }
-
- override def error(
- projectionId: ProjectionId,
- envelope: Envelope,
- cause: Throwable,
- recoveryStrategy: HandlerRecoveryStrategy): Unit = {
- probe ! Err(envelope, cause)
- }
-
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/DurableStateEndToEndSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/DurableStateEndToEndSpec.scala
deleted file mode 100644
index 5cc2731..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/DurableStateEndToEndSpec.scala
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.util.UUID
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.typed.ActorRef
-import pekko.actor.typed.ActorSystem
-import pekko.actor.typed.Behavior
-import pekko.actor.typed.scaladsl.Behaviors
-import pekko.persistence.query.DurableStateChange
-import pekko.persistence.query.UpdatedDurableState
-import pekko.persistence.r2dbc.state.scaladsl.R2dbcDurableStateStore
-import pekko.persistence.typed.PersistenceId
-import pekko.persistence.typed.state.scaladsl.DurableStateBehavior
-import pekko.projection.ProjectionBehavior
-import pekko.projection.ProjectionId
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import pekko.projection.state.scaladsl.DurableStateSourceProvider
-import com.typesafe.config.Config
-import com.typesafe.config.ConfigFactory
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-object DurableStateEndToEndSpec {
-
- val config: Config = ConfigFactory
- .parseString("""
- pekko.persistence.r2dbc {
- query {
- refresh-interval = 1s
- # stress more by using a small buffer (sql limit)
- buffer-size = 10
- }
- }
- """)
- .withFallback(TestConfig.config)
-
- object DurableStatePersister {
- import pekko.persistence.typed.state.scaladsl.Effect
-
- sealed trait Command
- final case class Persist(payload: Any) extends Command
- final case class PersistWithAck(payload: Any, replyTo: ActorRef[Done]) extends Command
- final case class Ping(replyTo: ActorRef[Done]) extends Command
- final case class Stop(replyTo: ActorRef[Done]) extends Command
-
- def apply(pid: String): Behavior[Command] =
- apply(PersistenceId.ofUniqueId(pid))
-
- def apply(pid: PersistenceId): Behavior[Command] = {
- Behaviors.setup { context =>
- DurableStateBehavior[Command, Any](
- persistenceId = pid,
- "",
- { (_, command) =>
- command match {
- case command: Persist =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- DurableStateBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload)
- case command: PersistWithAck =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- DurableStateBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload).thenRun(_ => command.replyTo ! Done)
- case Ping(replyTo) =>
- replyTo ! Done
- Effect.none
- case Stop(replyTo) =>
- replyTo ! Done
- Effect.stop()
- }
- })
- }
- }
- }
-
- class TestHandler(val projectionId: ProjectionId, val sliceRange: Range)
- extends R2dbcHandler[DurableStateChange[String]] {
- private val log = LoggerFactory.getLogger(getClass)
-
- var processed = Vector.empty[DurableStateChange[String]]
-
- override def process(session: R2dbcSession, envelope: DurableStateChange[String]): Future[Done] = {
- envelope match {
- case upd: UpdatedDurableState[String] =>
- log.debug("{} Processed {} revision {}", projectionId.key, upd.value, upd.revision: java.lang.Long)
- case _ =>
- }
- processed :+= envelope
- Future.successful(Done)
- }
- }
-
-}
-
-class DurableStateEndToEndSpec
- extends ScalaTestWithActorTestKit(DurableStateEndToEndSpec.config)
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
- import DurableStateEndToEndSpec._
-
- override def typedSystem: ActorSystem[_] = system
- private implicit val ec: ExecutionContext = system.executionContext
-
- private val settings = R2dbcProjectionSettings(testKit.system)
-
- override protected def beforeAll(): Unit = {
- super.beforeAll()
- }
-
- private def createHandlers(projectionName: String, nrOfProjections: Int): Map[ProjectionId, TestHandler] = {
- val sliceRanges =
- DurableStateSourceProvider.sliceRanges(system, R2dbcDurableStateStore.Identifier, nrOfProjections)
- sliceRanges.map { range =>
- val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
- projectionId -> new TestHandler(projectionId, range)
- }.toMap
- }
-
- private def startProjections(
- entityType: String,
- projectionName: String,
- nrOfProjections: Int,
- handlers: Map[ProjectionId, TestHandler]): Vector[ActorRef[ProjectionBehavior.Command]] = {
- val sliceRanges =
- DurableStateSourceProvider.sliceRanges(system, R2dbcDurableStateStore.Identifier, nrOfProjections)
-
- sliceRanges.map { range =>
- val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
- val sourceProvider =
- DurableStateSourceProvider.changesBySlices[String](
- system,
- R2dbcDurableStateStore.Identifier,
- entityType,
- range.min,
- range.max)
- val projection = R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider,
- handler = () => handlers(projectionId))
- spawn(ProjectionBehavior(projection))
- }.toVector
- }
-
- "A R2DBC projection with changesBySlices source" must {
-
- "handle latest updated state exactlyOnce" in {
- val numberOfEntities = 20
- val numberOfChanges = 10 * numberOfEntities
- val entityType = nextEntityType()
-
- val entities = (0 until numberOfEntities).map { n =>
- val persistenceId = PersistenceId(entityType, s"p$n")
- spawn(DurableStatePersister(persistenceId), s"p$n")
- }
-
- var revisionPerEntity = Map.empty[Int, Int]
-
- // write some before starting the projections
- (1 to 50).foreach { n =>
- val p = n % numberOfEntities
- val revision = revisionPerEntity.getOrElse(p, 0) + 1
- revisionPerEntity = revisionPerEntity.updated(p, revision)
- entities(p) ! DurableStatePersister.Persist(s"s$p-$revision")
- }
-
- val projectionName = UUID.randomUUID().toString
- val handlers = createHandlers(projectionName, nrOfProjections = 4)
- val projections = startProjections(entityType, projectionName, nrOfProjections = 4, handlers)
-
- // give them some time to start before writing more events
- Thread.sleep(500)
-
- var n = 51
- while (n <= numberOfChanges) {
- val p = n % numberOfEntities
- val revision = revisionPerEntity.getOrElse(p, 0) + 1
- revisionPerEntity = revisionPerEntity.updated(p, revision)
- entities(p) ! DurableStatePersister.Persist(s"s$p-$revision")
-
- // stop projections
- if (n == numberOfChanges / 2) {
- val probe = createTestProbe()
- projections.foreach { ref =>
- ref ! ProjectionBehavior.Stop
- probe.expectTerminated(ref)
- }
- }
-
- // resume projections again
- if (n == (numberOfChanges / 2) + 20)
- startProjections(entityType, projectionName, nrOfProjections = 4, handlers)
-
- if (n % 10 == 0)
- Thread.sleep(50)
- else if (n % 25 == 0)
- Thread.sleep(1500)
-
- n += 1
- }
-
- handlers.foreach { case (projectionId, handler) =>
- (0 until numberOfEntities).foreach { p =>
- val persistenceId = PersistenceId(entityType, s"p$p")
- val slice = DurableStateSourceProvider.sliceForPersistenceId(
- system,
- R2dbcDurableStateStore.Identifier,
- persistenceId.id)
- withClue(s"projectionId $projectionId, persistenceId $persistenceId, slice $slice: ") {
- if (handler.sliceRange.contains(slice)) {
- eventually {
- val updates = handler.processed.collect {
- case upd: UpdatedDurableState[String] if upd.persistenceId == persistenceId.id => upd
- }
- val revision = revisionPerEntity(p)
- updates.last.revision shouldBe revision
- updates.last.value shouldBe s"s$p-$revision"
- // processed events in right order
- updates shouldBe updates.sortBy(_.revision)
- }
- }
- }
- }
- }
-
- projections.foreach(_ ! ProjectionBehavior.Stop)
- }
- }
-
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedChaosSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedChaosSpec.scala
deleted file mode 100644
index d201fa5..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedChaosSpec.scala
+++ /dev/null
@@ -1,316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.util.UUID
-import java.util.concurrent.ConcurrentHashMap
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import scala.concurrent.duration._
-import scala.util.Random
-
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.testkit.typed.TestException
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.typed.ActorRef
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.r2dbc.query.scaladsl.R2dbcReadJournal
-import pekko.persistence.typed.PersistenceId
-import pekko.projection.ProjectionBehavior
-import pekko.projection.ProjectionId
-import pekko.projection.eventsourced.scaladsl.EventSourcedProvider
-import pekko.projection.r2dbc.EventSourcedChaosSpec.FailingTestHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import com.typesafe.config.Config
-import com.typesafe.config.ConfigFactory
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-object EventSourcedChaosSpec {
- import pekko.projection.r2dbc.EventSourcedEndToEndSpec.Processed
-
- val config: Config = ConfigFactory
- .parseString("""
- pekko.persistence.r2dbc {
- query {
- refresh-interval = 1s
- # Stress more by using a smaller buffer (sql limit) than default.
- # Note that it shouldn't be set too small since that would prevent progress if more than
- # this number of events are stored for the same timestamp.
- buffer-size = 100
- }
- }
- pekko.projection {
- restart-backoff {
- min-backoff = 500 ms
- max-backoff = 2 s
- }
- }
- """)
- .withFallback(TestConfig.config)
- .withFallback(ConfigFactory.parseString("""
- # TODO in nightly CI we can set this to 10
- pekko.projection.r2dbc.ChaosSpec.nr-iterations = 2
- """))
-
- class FailingTestHandler(
- projectionId: ProjectionId,
- probe: ActorRef[Processed],
- failEvents: ConcurrentHashMap[String, Int])
- extends R2dbcHandler[EventEnvelope[String]] {
- private val log = LoggerFactory.getLogger(getClass)
-
- override def process(session: R2dbcSession, envelope: EventEnvelope[String]): Future[Done] = {
- val failCount = failEvents.getOrDefault(envelope.eventOption, 0)
- if (failCount > 0) {
- failEvents.put(envelope.event, failCount - 1)
- log.debug(
- "{} Fail event [{}], pid [{}], seqNr [{}]",
- projectionId.key,
- envelope.event,
- envelope.persistenceId,
- envelope.sequenceNr: java.lang.Long)
- throw TestException(s"Fail event [${envelope.event}]")
- } else {
- log.debug(
- "{} Processed event [{}], pid [{}], seqNr [{}]",
- projectionId.key,
- envelope.event,
- envelope.persistenceId,
- envelope.sequenceNr: java.lang.Long)
- probe ! Processed(projectionId, envelope)
- Future.successful(Done)
- }
- }
- }
-}
-
-class EventSourcedChaosSpec
- extends ScalaTestWithActorTestKit(EventSourcedChaosSpec.config)
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
- import EventSourcedEndToEndSpec._
-
- override def typedSystem: ActorSystem[_] = system
- private implicit val ec: ExecutionContext = system.executionContext
-
- private val settings = R2dbcProjectionSettings(testKit.system)
- private val log = LoggerFactory.getLogger(getClass)
-
- private val seed = System.currentTimeMillis()
-
- override protected def beforeAll(): Unit = {
- super.beforeAll()
- }
-
- private def mkEvent(n: Int): String = {
- val template = "0000000"
- val s = n.toString
- "e" + (template + s).takeRight(5)
- }
-
- "A R2DBC projection under random conditions" must {
-
- "handle all events exactlyOnce" in {
- val rnd = new Random(seed)
- val numberOfIterations = system.settings.config.getInt("pekko.projection.r2dbc.ChaosSpec.nr-iterations")
- val numberOfEntities = 20 + rnd.nextInt(80)
- val numberOfProjections = 1 << rnd.nextInt(4) // 1 to 8 projections
- val entityType = nextEntityType()
- val failProbability = 0.01
- val stopProbability = 0.01
-
- var startedEntities = Map.empty[Int, ActorRef[Persister.Command]]
- def entity(i: Int): ActorRef[Persister.Command] = {
- startedEntities.get(i) match {
- case Some(ref) => ref
- case None =>
- val persistenceId = PersistenceId(entityType, s"p$i")
- val ref = spawn(Persister(persistenceId), s"p$i")
- startedEntities = startedEntities.updated(i, ref)
- ref
- }
- }
-
- val projectionName = UUID.randomUUID().toString
- log.debug("Using [{}] projection instances, with name [{}]", numberOfProjections, projectionName)
- val processedProbe = createTestProbe[Processed]()
- val failEvents = new ConcurrentHashMap[String, Int]
- val probe = createTestProbe()
-
- var runningProjections = Map.empty[Int, ActorRef[ProjectionBehavior.Command]]
- def startProjection(projectionIndex: Int): ActorRef[ProjectionBehavior.Command] = {
- runningProjections.get(projectionIndex) match {
- case Some(ref) => ref
- case None =>
- val range = EventSourcedProvider.sliceRanges(system, R2dbcReadJournal.Identifier, numberOfProjections)(
- projectionIndex)
-
- val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
- log.debug("Starting projection index [{}] with projectionId [{}]", projectionIndex, projectionId)
- val sourceProvider =
- EventSourcedProvider
- .eventsBySlices[String](system, R2dbcReadJournal.Identifier, entityType, range.min, range.max)
- val projection = R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider,
- handler = () => new FailingTestHandler(projectionId, processedProbe.ref, failEvents))
- val ref = spawn(ProjectionBehavior(projection))
- runningProjections = runningProjections.updated(projectionIndex, ref)
- ref
-
- }
- }
-
- var verifiedEventCounter = 0
- var eventCounter = 0
-
- def verify(iteration: Int): Unit = {
- (0 until numberOfProjections).foreach(startProjection)
-
- val pingProbe = createTestProbe[Done]()
- startedEntities.valuesIterator.foreach { ref =>
- ref ! Persister.Ping(pingProbe.ref)
- }
- pingProbe.receiveMessages(startedEntities.size, 20.seconds)
-
- var processed = Vector.empty[Processed]
- val expectedEventCounts = eventCounter - verifiedEventCounter
- val expectedEvents = (verifiedEventCounter + 1 to eventCounter).map(mkEvent).toVector
- (1 to expectedEventCounts).foreach { _ =>
- // not using receiveMessages(expectedEvents) for better logging in case of failure
- try {
- processed :+= processedProbe.receiveMessage(30.seconds)
- } catch {
- case e: AssertionError =>
- val missing = expectedEvents.diff(processed.map(_.envelope.event))
- log.error(
- s"Iteration #$iteration. Processed [${processed.size}] events, but expected [$expectedEventCounts]. " +
- s"Missing [${missing.mkString(",")}]. " +
- s"Received [${processed.map(p =>
- s"(${p.envelope.event}, ${p.envelope.persistenceId}, ${p.envelope.sequenceNr})").mkString(", ")}]. " +
- s"Seed [$seed].")
- throw e
- }
- }
-
- val processedEvents = processed.map(_.envelope.event)
- processedEvents.toSet shouldBe expectedEvents.toSet
- processedEvents.size shouldBe expectedEvents.size
-
- val byPid = processed.groupBy(_.envelope.persistenceId)
- val logMsg =
- s"Iteration #$iteration. Processed [$expectedEventCounts] events in [$numberOfProjections] projections from [${byPid.size}] pids."
- log.debug(logMsg)
- byPid.foreach { case (_, processedByPid) =>
- // all events of a pid must be processed by the same projection instance
- processedByPid.map(_.projectionId).toSet.size shouldBe 1
- // processed events in right order
- val seqNrs = processedByPid.map(_.envelope.sequenceNr).toVector
- (seqNrs.last - seqNrs.head + 1) shouldBe processedByPid.size
- seqNrs shouldBe (seqNrs.head to seqNrs.last).toVector
- }
- info(logMsg)
-
- verifiedEventCounter = eventCounter
- }
-
- for (iteration <- 1 to numberOfIterations) {
- log.info("Starting iteration [{}] with seed [{}]", iteration, seed)
- for (_ <- 1 to 200) {
- // Start projection
- if (rnd.nextDouble() < 0.1) {
- val projectionIndex = rnd.nextInt(numberOfProjections)
- startProjection(projectionIndex)
- }
-
- // Stop projection
- if (rnd.nextDouble() < stopProbability) {
- val projectionIndex = rnd.nextInt(numberOfProjections)
- runningProjections.get(projectionIndex).foreach { ref =>
- log.debug("Stopping projection [{}]", projectionIndex)
- ref ! ProjectionBehavior.Stop
- probe.expectTerminated(ref, 5.seconds)
- runningProjections -= projectionIndex
- }
- }
-
- // persist 1 event
- if (rnd.nextDouble() < 0.9) {
- val entityIndex = rnd.nextInt(numberOfEntities)
- eventCounter += 1
- val event = mkEvent(eventCounter)
- if (rnd.nextDouble() <= failProbability) {
- val failCount = 1 + rnd.nextInt(3)
- failEvents.put(event, failCount)
- log.debug("Persisting event [{}], it will fail in projection [{}] times", event, failCount)
- } else {
- log.debug("Persisting event [{}]", event)
- }
- entity(entityIndex) ! Persister.Persist(event)
- }
-
- // persist all (2 to 5) events
- if (rnd.nextDouble() < 0.1) {
- val entityIndex = rnd.nextInt(numberOfEntities)
- val numberOfEvents = 2 + rnd.nextInt(3)
- val events = (1 to numberOfEvents).map(e => mkEvent(e + eventCounter)).toList
- eventCounter += numberOfEvents
- if (rnd.nextDouble() < failProbability) {
- val failCount = 1 + rnd.nextInt(3)
- val i = rnd.nextInt(events.size)
- failEvents.put(events(i), failCount)
- log.debug(
- "Persisting events [{}], it will fail [{}] in projection [{}] times",
- events.mkString(", "),
- events(i),
- failCount: java.lang.Integer)
- } else {
- log.debug("Persisting events [{}]", events.mkString(", "))
- }
-
- entity(entityIndex) ! Persister.PersistAll(events)
- }
-
- // short sleep
- if (rnd.nextDouble() <= 0.05) {
- Thread.sleep(100)
- }
-
- // long sleep
- if (rnd.nextDouble() <= 0.01) {
- log.debug("Sleeping")
- Thread.sleep(2000)
- }
-
- }
-
- verify(iteration)
- }
-
- runningProjections.valuesIterator.foreach(_ ! ProjectionBehavior.Stop)
- }
- }
-
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedEndToEndSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedEndToEndSpec.scala
deleted file mode 100644
index ecf6730..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedEndToEndSpec.scala
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.time.Instant
-import java.util.UUID
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import scala.concurrent.duration._
-import org.apache.pekko.persistence.r2dbc.QuerySettings
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.typed.ActorRef
-import pekko.actor.typed.ActorSystem
-import pekko.actor.typed.Behavior
-import pekko.actor.typed.scaladsl.Behaviors
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.r2dbc.Dialect
-import pekko.persistence.r2dbc.internal.PayloadCodec
-import pekko.persistence.r2dbc.internal.PayloadCodec.RichStatement
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
-import pekko.persistence.r2dbc.query.scaladsl.R2dbcReadJournal
-import pekko.persistence.typed.PersistenceId
-import pekko.persistence.typed.scaladsl.Effect
-import pekko.persistence.typed.scaladsl.EventSourcedBehavior
-import pekko.projection.ProjectionBehavior
-import pekko.projection.ProjectionId
-import pekko.projection.eventsourced.scaladsl.EventSourcedProvider
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import pekko.serialization.SerializationExtension
-import com.typesafe.config.Config
-import com.typesafe.config.ConfigFactory
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-object EventSourcedEndToEndSpec {
-
- val config: Config = ConfigFactory
- .parseString("""
- pekko.persistence.r2dbc {
- query {
- refresh-interval = 500 millis
- # stress more by using a small buffer (sql limit)
- buffer-size = 10
-
- backtracking.behind-current-time = 5 seconds
- }
- }
- """)
- .withFallback(TestConfig.config)
-
- object Persister {
- sealed trait Command
- final case class Persist(payload: Any) extends Command
- final case class PersistWithAck(payload: Any, replyTo: ActorRef[Done]) extends Command
- final case class PersistAll(payloads: List[Any]) extends Command
- final case class Ping(replyTo: ActorRef[Done]) extends Command
- final case class Stop(replyTo: ActorRef[Done]) extends Command
-
- def apply(pid: PersistenceId): Behavior[Command] = {
- Behaviors.setup { context =>
- EventSourcedBehavior[Command, Any, String](
- persistenceId = pid,
- "",
- { (_, command) =>
- command match {
- case command: Persist =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- EventSourcedBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload)
- case command: PersistWithAck =>
- context.log.debug(
- "Persist [{}], pid [{}], seqNr [{}]",
- command.payload.toString,
- pid.id,
- EventSourcedBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payload).thenRun(_ => command.replyTo ! Done)
- case command: PersistAll =>
- if (context.log.isDebugEnabled)
- context.log.debug(
- "PersistAll [{}], pid [{}], seqNr [{}]",
- command.payloads.mkString(","),
- pid.id,
- EventSourcedBehavior.lastSequenceNumber(context) + 1: java.lang.Long)
- Effect.persist(command.payloads)
- case Ping(replyTo) =>
- replyTo ! Done
- Effect.none
- case Stop(replyTo) =>
- replyTo ! Done
- Effect.stop()
- }
- },
- (_, _) => "")
- }
- }
- }
-
- final case class Processed(projectionId: ProjectionId, envelope: EventEnvelope[String])
-
- class TestHandler(projectionId: ProjectionId, probe: ActorRef[Processed])
- extends R2dbcHandler[EventEnvelope[String]] {
- private val log = LoggerFactory.getLogger(getClass)
-
- override def process(session: R2dbcSession, envelope: EventEnvelope[String]): Future[Done] = {
- log.debug("{} Processed {}", projectionId.key: Any, envelope.event: Any)
- probe ! Processed(projectionId, envelope)
- Future.successful(Done)
- }
- }
-
-}
-
-class EventSourcedEndToEndSpec
- extends ScalaTestWithActorTestKit(EventSourcedEndToEndSpec.config)
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
- import EventSourcedEndToEndSpec._
-
- override def typedSystem: ActorSystem[_] = system
- private implicit val ec: ExecutionContext = system.executionContext
-
- private val log = LoggerFactory.getLogger(getClass)
-
- private val querySettings = QuerySettings(system.settings.config.getConfig("pekko.persistence.r2dbc.query"))
- private val projectionSettings = R2dbcProjectionSettings(system)
- private val stringSerializer = SerializationExtension(system).serializerFor(classOf[String])
- private implicit val journalPayloadCodec: PayloadCodec = querySettings.journalPayloadCodec
-
- override protected def beforeAll(): Unit = {
- super.beforeAll()
- }
-
- // to be able to store events with specific timestamps
- private def writeEvent(persistenceId: String, seqNr: Long, timestamp: Instant, event: String): Unit = {
- log.debug("Write test event [{}] [{}] [{}] at time [{}]", persistenceId, seqNr: java.lang.Long, event, timestamp)
- implicit val dialect: Dialect = projectionSettings.dialect
- val insertEventSql = sql"""
- INSERT INTO ${querySettings.journalTableWithSchema}
- (slice, entity_type, persistence_id, seq_nr, db_timestamp, writer, adapter_manifest, event_ser_id, event_ser_manifest, event_payload)
- VALUES (?, ?, ?, ?, ?, '', '', ?, '', ?)"""
-
- val entityType = PersistenceId.extractEntityType(persistenceId)
- val slice = persistenceExt.sliceForPersistenceId(persistenceId)
-
- val result = r2dbcExecutor.updateOne("test writeEvent") { connection =>
- connection
- .createStatement(insertEventSql)
- .bind(0, slice)
- .bind(1, entityType)
- .bind(2, persistenceId)
- .bind(3, seqNr)
- .bind(4, timestamp)
- .bind(5, stringSerializer.identifier)
- .bindPayload(6, stringSerializer.toBinary(event))
- }
- result.futureValue shouldBe 1
- }
-
- private def startProjections(
- entityType: String,
- projectionName: String,
- nrOfProjections: Int,
- processedProbe: ActorRef[Processed]): Vector[ActorRef[ProjectionBehavior.Command]] = {
- val sliceRanges = EventSourcedProvider.sliceRanges(system, R2dbcReadJournal.Identifier, nrOfProjections)
-
- sliceRanges.map { range =>
- val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
- val sourceProvider =
- EventSourcedProvider.eventsBySlices[String](
- system,
- R2dbcReadJournal.Identifier,
- entityType,
- range.min,
- range.max)
- val projection = R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(projectionSettings),
- sourceProvider = sourceProvider,
- handler = () => new TestHandler(projectionId, processedProbe.ref))
- spawn(ProjectionBehavior(projection))
- }.toVector
- }
-
- private def mkEvent(n: Int): String = {
- val template = "0000000"
- val s = n.toString
- "e" + (template + s).takeRight(5)
- }
-
- "A R2DBC projection with eventsBySlices source" must {
-
- "handle all events exactlyOnce" in {
- val numberOfEntities = 20
- val numberOfEvents = numberOfEntities * 10
- val entityType = nextEntityType()
-
- val entities = (0 until numberOfEntities).map { n =>
- val persistenceId = PersistenceId(entityType, s"p$n")
- spawn(Persister(persistenceId), s"p$n")
- }
-
- // write some before starting the projections
- var n = 1
- while (n <= 50) {
- val p = n % numberOfEntities
- // mix some persist 1 and persist 3 events
- if (n % 7 == 0) {
- entities(p) ! Persister.PersistAll((0 until 3).map(i => mkEvent(n + i)).toList)
- n += 3
- } else {
- entities(p) ! Persister.Persist(mkEvent(n))
- n += 1
- }
- }
-
- val projectionName = UUID.randomUUID().toString
- val processedProbe = createTestProbe[Processed]()
- val projections = startProjections(entityType, projectionName, nrOfProjections = 4, processedProbe.ref)
-
- // give them some time to start before writing more events
- Thread.sleep(500)
-
- while (n <= numberOfEvents) {
- val p = n % numberOfEntities
- entities(p) ! Persister.Persist(mkEvent(n))
-
- // stop projections
- if (n == numberOfEvents / 2) {
- val probe = createTestProbe()
- projections.foreach { ref =>
- ref ! ProjectionBehavior.Stop
- probe.expectTerminated(ref)
- }
- }
-
- // resume projections again
- if (n == (numberOfEvents / 2) + 20)
- startProjections(entityType, projectionName, nrOfProjections = 4, processedProbe.ref)
-
- if (n % 10 == 0)
- Thread.sleep(50)
- else if (n % 25 == 0)
- Thread.sleep(1500)
-
- n += 1
- }
-
- var processed = Vector.empty[Processed]
- val expectedEvents = (1 to numberOfEvents).map(mkEvent).toVector
- (1 to numberOfEvents).foreach { _ =>
- // not using receiveMessages(expectedEvents) for better logging in case of failure
- try {
- processed :+= processedProbe.receiveMessage(30.seconds)
- } catch {
- case e: AssertionError =>
- val missing = expectedEvents.diff(processed.map(_.envelope.event))
- log.error(s"Processed [${processed.size}] events, but expected [$numberOfEvents]. " +
- s"Missing [${missing.mkString(",")}]. " +
- s"Received [${processed.map(p =>
- s"(${p.envelope.event}, ${p.envelope.persistenceId}, ${p.envelope.sequenceNr})").mkString(", ")}]. ")
- throw e
- }
- }
-
- val byPid = processed.groupBy(_.envelope.persistenceId)
- byPid.foreach { case (_, processedByPid) =>
- // all events of a pid must be processed by the same projection instance
- processedByPid.map(_.projectionId).toSet.size shouldBe 1
- // processed events in right order
- processedByPid.map(_.envelope.sequenceNr).toVector shouldBe (1 to processedByPid.size).toVector
- }
-
- projections.foreach(_ ! ProjectionBehavior.Stop)
- }
-
- "accept unknown sequence number if previous is old" in {
- val entityType = nextEntityType()
- val pid1 = nextPid(entityType)
- val pid2 = nextPid(entityType)
- val pid3 = nextPid(entityType)
-
- val startTime = TestClock.nowMicros().instant()
- val oldTime = startTime.minus(projectionSettings.timeWindow).minusSeconds(60)
- writeEvent(pid1, 1L, startTime, "e1-1")
-
- val projectionName = UUID.randomUUID().toString
- val processedProbe = createTestProbe[Processed]()
- val projection = startProjections(entityType, projectionName, nrOfProjections = 1, processedProbe.ref).head
-
- processedProbe.receiveMessage().envelope.event shouldBe "e1-1"
-
- // old event for pid2, seqN3. will not be picked up by backtracking because outside time window
- writeEvent(pid2, 3L, oldTime, "e2-3")
- // pid2, seqNr 3 is unknown when receiving 4 so will lookup timestamp of 3
- // and accept 4 because 3 was older than time window
- writeEvent(pid2, 4L, startTime.plusMillis(1), "e2-4")
- processedProbe.receiveMessage().envelope.event shouldBe "e2-4"
-
- // pid3, seqNr 6 is unknown when receiving 7 so will lookup 6, but not found
- // and that will be accepted (could have been deleted)
- writeEvent(pid3, 7L, startTime.plusMillis(2), "e3-7")
- processedProbe.receiveMessage().envelope.event shouldBe "e3-7"
-
- // pid3, seqNr 8 is missing (knows 7) when receiving 9
- writeEvent(pid3, 9L, startTime.plusMillis(4), "e3-9")
- processedProbe.expectNoMessage(querySettings.refreshInterval + 2000.millis)
-
- // but backtracking can fill in the gaps, backtracking will pick up pid3 seqNr 8 and 9
- writeEvent(pid3, 8L, startTime.plusMillis(3), "e3-8")
- val possibleDelay =
- querySettings.backtrackingBehindCurrentTime + querySettings.refreshInterval + processedProbe.remainingOrDefault
- processedProbe.receiveMessage(possibleDelay).envelope.event shouldBe "e3-8"
- processedProbe.receiveMessage(possibleDelay).envelope.event shouldBe "e3-9"
-
- projection ! ProjectionBehavior.Stop
- }
- }
-
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedPubSubSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedPubSubSpec.scala
deleted file mode 100644
index 17f6967..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/EventSourcedPubSubSpec.scala
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.util.UUID
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import scala.concurrent.duration._
-
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.testkit.typed.scaladsl.TestProbe
-import pekko.actor.typed.ActorRef
-import pekko.actor.typed.ActorSystem
-import pekko.actor.typed.scaladsl.LoggerOps
-import pekko.persistence.query.TimestampOffset
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.r2dbc.query.scaladsl.R2dbcReadJournal
-import pekko.persistence.typed.PersistenceId
-import pekko.projection.ProjectionBehavior
-import pekko.projection.ProjectionId
-import pekko.projection.eventsourced.scaladsl.EventSourcedProvider
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import com.typesafe.config.Config
-import com.typesafe.config.ConfigFactory
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-object EventSourcedPubSubSpec {
-
- val config: Config = ConfigFactory
- .parseString("""
- pekko.persistence.r2dbc {
- journal.publish-events = on
- query {
- refresh-interval = 3 seconds
- # simulate lost messages by overflowing the buffer
- buffer-size = 10
-
- backtracking {
- behind-current-time = 5 seconds
- window = 20 seconds
- }
- }
- }
- """)
- .withFallback(TestConfig.config)
-
- final case class Processed(projectionId: ProjectionId, envelope: EventEnvelope[String])
-
- class TestHandler(
- projectionId: ProjectionId,
- probe: ActorRef[Processed],
- whenDone: EventEnvelope[String] => Future[Done])(implicit ec: ExecutionContext)
- extends R2dbcHandler[EventEnvelope[String]] {
- private val log = LoggerFactory.getLogger(getClass)
-
- override def process(session: R2dbcSession, envelope: EventEnvelope[String]): Future[Done] = {
- whenDone(envelope).map { _ =>
- val timestampOffset = envelope.offset.asInstanceOf[TimestampOffset]
- val directReplication = timestampOffset.timestamp == timestampOffset.readTimestamp
- log.debugN(
- "{} Processed {}, pid {}, seqNr {}, direct {}",
- projectionId.key,
- envelope.event,
- envelope.persistenceId,
- envelope.sequenceNr,
- directReplication)
- probe ! Processed(projectionId, envelope)
- Done
- }
- }
- }
-
-}
-
-class EventSourcedPubSubSpec
- extends ScalaTestWithActorTestKit(EventSourcedPubSubSpec.config)
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
- import EventSourcedEndToEndSpec.Persister
- import EventSourcedPubSubSpec._
-
- override def typedSystem: ActorSystem[_] = system
- private implicit val ec: ExecutionContext = system.executionContext
-
- private val log = LoggerFactory.getLogger(getClass)
-
- private val projectionSettings = R2dbcProjectionSettings(system)
-
- override protected def beforeAll(): Unit = {
- super.beforeAll()
- }
-
- private def startProjections(
- entityType: String,
- projectionName: String,
- nrOfProjections: Int,
- processedProbe: ActorRef[Processed],
- whenDone: EventEnvelope[String] => Future[Done]): Vector[ActorRef[ProjectionBehavior.Command]] = {
- val sliceRanges = EventSourcedProvider.sliceRanges(system, R2dbcReadJournal.Identifier, nrOfProjections)
-
- sliceRanges.map { range =>
- val projectionId = ProjectionId(projectionName, s"${range.min}-${range.max}")
- val sourceProvider =
- EventSourcedProvider.eventsBySlices[String](
- system,
- R2dbcReadJournal.Identifier,
- entityType,
- range.min,
- range.max)
- val projection = R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(projectionSettings),
- sourceProvider = sourceProvider,
- handler = () => new TestHandler(projectionId, processedProbe.ref, whenDone))
- spawn(ProjectionBehavior(projection))
- }.toVector
- }
-
- private def mkEvent(n: Int): String = {
- val template = "0000000"
- val s = n.toString
- "e" + (template + s).takeRight(5)
- }
-
- def expectProcessed(processedProbe: TestProbe[Processed], expectedFrom: Int, expectedTo: Int): Vector[Processed] = {
- val numberOfEvents = expectedTo - expectedFrom + 1
- var processed = Vector.empty[Processed]
- val expectedEvents = (1 to numberOfEvents).map(mkEvent).toVector
- (1 to numberOfEvents).foreach { _ =>
- // not using receiveMessages(expectedEvents) for better logging in case of failure
- try {
- processed :+= processedProbe.receiveMessage(30.seconds)
- } catch {
- case e: AssertionError =>
- val missing = expectedEvents.diff(processed.map(_.envelope.event))
- log.error(s"Processed [${processed.size}] events, but expected [$numberOfEvents]. " +
- s"Missing [${missing.mkString(",")}]. " +
- s"Received [${processed.map(p =>
- s"(${p.envelope.event}, ${p.envelope.persistenceId}, ${p.envelope.sequenceNr})").mkString(", ")}]. ")
- throw e
- }
- }
- processed
- }
-
- "A R2DBC projection with eventsBySlices source and publish-events" must {
-
- "handle all events exactlyOnce" in {
- val numberOfEntities = 20
- val numberOfEvents = numberOfEntities * 10
- val entityType = nextEntityType()
-
- val entities = (0 until numberOfEntities).map { n =>
- val persistenceId = PersistenceId(entityType, s"p$n")
- spawn(Persister(persistenceId), s"p$n")
- }
-
- // write some before starting the projections
- (1 to 10).foreach { n =>
- val p = n % numberOfEntities
- entities(p) ! Persister.Persist(mkEvent(n))
- }
-
- val projectionName = UUID.randomUUID().toString
- val processedProbe = createTestProbe[Processed]()
- val slowEvents = Set(mkEvent(31), mkEvent(32), mkEvent(33))
- val whenDone: EventEnvelope[String] => Future[Done] = { env =>
- if (slowEvents.contains(env.event))
- pekko.pattern.after(500.millis)(Future.successful(Done))
- else
- Future.successful(Done)
- }
- val projections = startProjections(entityType, projectionName, nrOfProjections = 4, processedProbe.ref, whenDone)
-
- // give them some time to start before writing more events, but it should work anyway
- Thread.sleep(500)
-
- (11 to 20).foreach { n =>
- val p = n % numberOfEntities
- entities(p) ! Persister.Persist(mkEvent(n))
- }
-
- var processed = Vector.empty[Processed]
- processed ++= expectProcessed(processedProbe, 1, 20)
-
- (21 to 30).foreach { n =>
- val p = n % numberOfEntities
- entities(p) ! Persister.Persist(mkEvent(n))
- }
- processed ++= expectProcessed(processedProbe, 21, 30)
-
- // Processing of 31 is slow in the handler, see whenDone above.
- // This will overflow the buffer for the subscribers, simulating lost messages,
- // but they should be picked by the queries.
- (31 to numberOfEvents - 10).foreach { n =>
- val p = n % numberOfEntities
- entities(p) ! Persister.Persist(mkEvent(n))
- }
- processed ++= expectProcessed(processedProbe, 31, numberOfEvents - 10)
-
- (numberOfEvents - 10 + 1 to numberOfEvents).foreach { n =>
- val p = n % numberOfEntities
- entities(p) ! Persister.Persist(mkEvent(n))
- }
-
- processed ++= expectProcessed(processedProbe, numberOfEvents - 10 + 1, numberOfEvents)
-
- val byPid = processed.groupBy(_.envelope.persistenceId)
- byPid.foreach { case (pid, processedByPid) =>
- // all events of a pid must be processed by the same projection instance
- processedByPid.map(_.projectionId).toSet.size shouldBe 1
- // processed events in right order
- processedByPid.map(_.envelope.sequenceNr) shouldBe (1 to processedByPid.size).toVector
-
- val viaPubSub =
- processedByPid.filter(p =>
- p.envelope.offset.asInstanceOf[TimestampOffset].timestamp ==
- p.envelope.offset
- .asInstanceOf[TimestampOffset]
- .readTimestamp)
- log.info("via pub-sub {}: {}", pid: Any, viaPubSub.map(_.envelope.sequenceNr).mkString(", "): Any)
- }
-
- val countViaPubSub = processed.count(p =>
- p.envelope.offset.asInstanceOf[TimestampOffset].timestamp ==
- p.envelope.offset
- .asInstanceOf[TimestampOffset]
- .readTimestamp)
- log.info("Total via pub-sub: {}", countViaPubSub)
- countViaPubSub shouldBe >(0)
-
- projections.foreach(_ ! ProjectionBehavior.Stop)
- }
- }
-
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreSpec.scala
deleted file mode 100644
index 9575e5d..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreSpec.scala
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.time.Instant
-import java.util.UUID
-
-import scala.concurrent.ExecutionContext
-import org.apache.pekko
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.query.Sequence
-import pekko.persistence.query.TimeBasedUUID
-import pekko.persistence.r2dbc.Dialect
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
-import pekko.projection.MergeableOffset
-import pekko.projection.ProjectionId
-import pekko.projection.internal.ManagementState
-import pekko.projection.r2dbc.internal.OffsetPidSeqNr
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore
-import org.scalatest.wordspec.AnyWordSpecLike
-
-class R2dbcOffsetStoreSpec
- extends ScalaTestWithActorTestKit(TestConfig.config)
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
-
- override def typedSystem: ActorSystem[_] = system
-
- // test clock for testing of the `last_updated` Instant
- private val clock = TestClock.nowMillis()
-
- private val settings = R2dbcProjectionSettings(testKit.system)
-
- private def createOffsetStore(projectionId: ProjectionId) =
- R2dbcOffsetStore.fromConfig(projectionId, None, system, settings, r2dbcExecutor, clock)
-
- private val table = settings.offsetTableWithSchema
-
- private implicit val ec: ExecutionContext = system.executionContext
-
- implicit val dialect: Dialect = settings.dialect
- def selectLastSql: String =
- sql"SELECT * FROM $table WHERE projection_name = ? AND projection_key = ?"
-
- private def selectLastUpdated(projectionId: ProjectionId): Instant = {
- r2dbcExecutor
- .selectOne("test")(
- conn =>
- conn
- .createStatement(selectLastSql)
- .bind(0, projectionId.name)
- .bind(1, projectionId.key),
- row => Instant.ofEpochMilli(row.get[java.lang.Long]("last_updated", classOf[java.lang.Long])))
- .futureValue
- .getOrElse(throw new RuntimeException(s"no records found for $projectionId"))
- }
-
- "The R2dbcOffsetStore" must {
-
- "save and read offsets" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- saveOffset(1L)
- val offset1 = offsetStore.readOffset[Long]()
- offset1.futureValue shouldBe Some(1L)
-
- saveOffset(2L)
- val offset2 = offsetStore.readOffset[Long]()
- offset2.futureValue shouldBe Some(2L) // yep, saveOffset overwrites previous
- }
-
- "save and retrieve offsets of type Long" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- saveOffset(1L)
- val offset = offsetStore.readOffset[Long]()
- offset.futureValue shouldBe Some(1L)
- }
-
- "save and retrieve offsets of type java.lang.Long" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- saveOffset(java.lang.Long.valueOf(1L))
- val offset = offsetStore.readOffset[java.lang.Long]()
- offset.futureValue shouldBe Some(1L)
- }
-
- "save and retrieve offsets of type Int" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- saveOffset(1)
- val offset = offsetStore.readOffset[Int]()
- offset.futureValue shouldBe Some(1)
- }
-
- "save and retrieve offsets of type java.lang.Integer" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- saveOffset(java.lang.Integer.valueOf(1))
- val offset = offsetStore.readOffset[java.lang.Integer]()
- offset.futureValue shouldBe Some(1)
- }
-
- "save and retrieve offsets of type String" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- val randOffset = UUID.randomUUID().toString
- saveOffset(randOffset)
- val offset = offsetStore.readOffset[String]()
- offset.futureValue shouldBe Some(randOffset)
- }
-
- "save and retrieve offsets of type pekko.persistence.query.Sequence" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- val seqOffset = Sequence(1L)
- saveOffset(seqOffset)
- val offset = offsetStore.readOffset[Sequence]()
- offset.futureValue shouldBe Some(seqOffset)
- }
-
- "save and retrieve offsets of type pekko.persistence.query.TimeBasedUUID" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- val timeUuidOffset =
- TimeBasedUUID(UUID.fromString("49225740-2019-11ea-a752-ffae2393b6e4")) // 2019-12-16T15:32:36.148Z[UTC]
- saveOffset(timeUuidOffset)
- val offset = offsetStore.readOffset[TimeBasedUUID]()
- offset.futureValue shouldBe Some(timeUuidOffset)
- }
-
- "save and retrieve offsets of unknown custom serializable type" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- val customOffset = "abc"
- saveOffset(customOffset)
- val offset = offsetStore.readOffset[TimeBasedUUID]()
- offset.futureValue shouldBe Some(customOffset)
- }
-
- "save and retrieve MergeableOffset" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- val origOffset = MergeableOffset(Map("abc" -> 1L, "def" -> 1L, "ghi" -> 1L))
- saveOffset(origOffset)
- val offset = offsetStore.readOffset[MergeableOffset[Long]]()
- offset.futureValue shouldBe Some(origOffset)
- }
-
- "add new offsets to MergeableOffset" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- val origOffset = MergeableOffset(Map("abc" -> 1L, "def" -> 1L))
- saveOffset(origOffset)
-
- val offset1 = offsetStore.readOffset[MergeableOffset[Long]]()
- offset1.futureValue shouldBe Some(origOffset)
-
- // mix updates and inserts
- val updatedOffset = MergeableOffset(Map("abc" -> 2L, "def" -> 2L, "ghi" -> 1L))
- saveOffset(updatedOffset)
-
- val offset2 = offsetStore.readOffset[MergeableOffset[Long]]()
- offset2.futureValue shouldBe Some(updatedOffset)
- }
-
- "update timestamp" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- val instant0 = clock.instant()
- saveOffset(15)
-
- val instant1 = selectLastUpdated(projectionId)
- instant1 shouldBe instant0
-
- val instant2 = clock.tick(java.time.Duration.ofMillis(5))
- saveOffset(16)
-
- val instant3 = selectLastUpdated(projectionId)
- instant3 shouldBe instant2
- }
-
- "set offset" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- saveOffset(3L)
- offsetStore.readOffset[Long]().futureValue shouldBe Some(3L)
-
- offsetStore.managementSetOffset(2L).futureValue
- offsetStore.readOffset[Long]().futureValue shouldBe Some(2L)
- }
-
- "clear offset" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
- def saveOffset(offset: Any): Unit =
- offsetStore.saveOffset(OffsetPidSeqNr(offset)).futureValue
-
- saveOffset(3L)
- offsetStore.readOffset[Long]().futureValue shouldBe Some(3L)
-
- offsetStore.managementClearOffset().futureValue
- offsetStore.readOffset[Long]().futureValue shouldBe None
- }
-
- "read and save paused" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- offsetStore.readManagementState().futureValue shouldBe None
-
- offsetStore.savePaused(paused = true).futureValue
- offsetStore.readManagementState().futureValue shouldBe Some(ManagementState(paused = true))
-
- offsetStore.savePaused(paused = false).futureValue
- offsetStore.readManagementState().futureValue shouldBe Some(ManagementState(paused = false))
- }
- }
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreStateSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreStateSpec.scala
deleted file mode 100644
index 318475d..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcOffsetStoreStateSpec.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.time.Instant
-
-import org.apache.pekko
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore.Record
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore.State
-import org.scalatest.TestSuite
-import org.scalatest.matchers.should.Matchers
-import org.scalatest.wordspec.AnyWordSpec
-
-class R2dbcOffsetStoreStateSpec extends AnyWordSpec with TestSuite with Matchers {
-
- "R2dbcOffsetStore.State" should {
- "add records and keep track of pids and latest offset" in {
- val t0 = TestClock.nowMillis().instant()
- val state1 = State.empty
- .add(Vector(Record("p1", 1, t0), Record("p1", 2, t0.plusMillis(1)), Record("p1", 3, t0.plusMillis(2))))
- state1.byPid("p1").seqNr shouldBe 3L
- state1.latestTimestamp shouldBe t0.plusMillis(2)
- state1.latestOffset.get.seen shouldBe Map("p1" -> 3L)
- state1.oldestTimestamp shouldBe t0
-
- val state2 = state1.add(Vector(Record("p2", 2, t0.plusMillis(1))))
- state2.byPid("p1").seqNr shouldBe 3L
- state2.byPid("p2").seqNr shouldBe 2L
- // latest not updated because timestamp of p2 was before latest
- state2.latestTimestamp shouldBe t0.plusMillis(2)
- state2.latestOffset.get.seen shouldBe Map("p1" -> 3L)
- state2.oldestTimestamp shouldBe t0
-
- val state3 = state2.add(Vector(Record("p3", 10, t0.plusMillis(3))))
- state3.byPid("p1").seqNr shouldBe 3L
- state3.byPid("p2").seqNr shouldBe 2L
- state3.byPid("p3").seqNr shouldBe 10L
- state3.latestTimestamp shouldBe t0.plusMillis(3)
- state3.latestOffset.get.seen shouldBe Map("p3" -> 10L)
- state3.oldestTimestamp shouldBe t0
- }
-
- // reproducer of issue #173
- "include highest seqNr in seen of latestOffset" in {
- val t0 = TestClock.nowMillis().instant()
- val records =
- Vector(Record("p4", 9, t0), Record("p2", 2, t0), Record("p3", 5, t0), Record("p2", 1, t0), Record("p1", 3, t0))
- val state = State(records)
- state.byPid("p2").seqNr shouldBe 2L
- // p2 -> 2 should be included even though p2 -> 1 was added afterwards (same timestamp)
- state.latestOffset.get.seen shouldBe Map("p1" -> 3L, "p2" -> 2L, "p3" -> 5L, "p4" -> 9L)
- }
-
- "evict old" in {
- val t0 = TestClock.nowMillis().instant()
- val state1 = State.empty
- .add(
- Vector(
- Record("p1", 1, t0),
- Record("p2", 2, t0.plusMillis(1)),
- Record("p3", 3, t0.plusMillis(2)),
- Record("p4", 4, t0.plusMillis(3)),
- Record("p5", 5, t0.plusMillis(4))))
- state1.latestOffset.get.seen shouldBe Map("p5" -> 5L)
- state1.oldestTimestamp shouldBe t0
- state1.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map(
- "p1" -> 1L,
- "p2" -> 2L,
- "p3" -> 3L,
- "p4" -> 4L,
- "p5" -> 5L)
-
- val state2 = state1.evict(t0.plusMillis(2), keepNumberOfEntries = 1)
- state2.latestOffset.get.seen shouldBe Map("p5" -> 5L)
- state2.oldestTimestamp shouldBe t0.plusMillis(2)
- state2.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map("p3" -> 3L, "p4" -> 4L, "p5" -> 5L)
-
- // keep all
- state1.evict(t0.plusMillis(2), keepNumberOfEntries = 100) shouldBe state1
-
- // keep 4
- val state3 = state1.evict(t0.plusMillis(2), keepNumberOfEntries = 4)
- state3.latestOffset.get.seen shouldBe Map("p5" -> 5L)
- state3.oldestTimestamp shouldBe t0.plusMillis(1)
- state3.byPid.map { case (pid, r) => pid -> r.seqNr } shouldBe Map("p2" -> 2L, "p3" -> 3L, "p4" -> 4L, "p5" -> 5L)
- }
-
- "find duplicate" in {
- val t0 = TestClock.nowMillis().instant()
- val state =
- State(Vector(Record("p1", 1, t0), Record("p2", 2, t0.plusMillis(1)), Record("p3", 3, t0.plusMillis(2))))
- state.isDuplicate(Record("p1", 1, t0)) shouldBe true
- state.isDuplicate(Record("p1", 2, t0.plusMillis(10))) shouldBe false
- state.isDuplicate(Record("p2", 1, t0)) shouldBe true
- state.isDuplicate(Record("p4", 4, t0.plusMillis(10))) shouldBe false
- }
- }
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSpec.scala
deleted file mode 100644
index 04710c5..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcProjectionSpec.scala
+++ /dev/null
@@ -1,1408 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.util.UUID
-import java.util.concurrent.atomic.AtomicInteger
-import java.util.concurrent.atomic.AtomicReference
-
-import scala.annotation.tailrec
-import scala.collection.immutable
-import scala.concurrent.Await
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import scala.concurrent.duration._
-import org.apache.pekko
-import pekko.Done
-import pekko.NotUsed
-import pekko.actor.testkit.typed.TestException
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.typed.ActorRef
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.r2dbc.Dialect
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import pekko.persistence.r2dbc.internal.Sql.DialectInterpolation
-import pekko.projection.HandlerRecoveryStrategy
-import pekko.projection.OffsetVerification
-import pekko.projection.OffsetVerification.VerificationFailure
-import pekko.projection.OffsetVerification.VerificationSuccess
-import pekko.projection.ProjectionBehavior
-import pekko.projection.ProjectionContext
-import pekko.projection.ProjectionId
-import pekko.projection.TestStatusObserver
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import pekko.projection.scaladsl.Handler
-import pekko.projection.scaladsl.ProjectionManagement
-import pekko.projection.scaladsl.SourceProvider
-import pekko.projection.testkit.scaladsl.ProjectionTestKit
-import pekko.projection.testkit.scaladsl.TestSourceProvider
-import pekko.stream.scaladsl.FlowWithContext
-import pekko.stream.scaladsl.Source
-import pekko.stream.testkit.TestPublisher
-import pekko.stream.testkit.TestSubscriber
-import pekko.stream.testkit.scaladsl.TestSource
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-object R2dbcProjectionSpec {
- final case class Envelope(id: String, offset: Long, message: String)
-
- def sourceProvider(
- id: String,
- complete: Boolean = true,
- verifyOffsetFn: Long => OffsetVerification = _ => VerificationSuccess): SourceProvider[Long, Envelope] = {
- val envelopes: Source[Envelope, NotUsed] =
- Source(
- List(
- Envelope(id, 1L, "e1"),
- Envelope(id, 2L, "e2"),
- Envelope(id, 3L, "e3"),
- Envelope(id, 4L, "e4"),
- Envelope(id, 5L, "e5"),
- Envelope(id, 6L, "e6")))
-
- val sp = TestSourceProvider[Long, Envelope](envelopes, _.offset)
- .withOffsetVerification(verifyOffsetFn)
- .withStartSourceFrom((lastProcessedOffset, offset) => offset <= lastProcessedOffset)
- if (complete) sp.withAllowCompletion(true)
- else sp
- }
-
- // test model is as simple as a text that gets other string concatenated to it
- case class ConcatStr(id: String, text: String) {
- def concat(newMsg: String): ConcatStr = {
- if (text == "")
- copy(id, newMsg)
- else
- copy(text = text + "|" + newMsg)
- }
- }
-
- object TestRepository {
- val table = "projection_spec_model"
-
- val createTableSql: String =
- s"""|CREATE table IF NOT EXISTS $table (
- | id VARCHAR(255) NOT NULL,
- | concatenated VARCHAR(255) NOT NULL,
- | PRIMARY KEY(id)
- |);""".stripMargin
- }
-
- final case class TestRepository(session: R2dbcSession, settings: R2dbcProjectionSettings)(
- implicit ec: ExecutionContext, system: ActorSystem[_]) {
- import TestRepository.table
-
- private val logger = LoggerFactory.getLogger(this.getClass)
-
- implicit private val dialect: Dialect = settings.dialect
-
- def concatToText(id: String, payload: String): Future[Done] = {
- val savedStrOpt = findById(id)
-
- savedStrOpt.flatMap { strOpt =>
- val newConcatStr = strOpt
- .map {
- _.concat(payload)
- }
- .getOrElse(ConcatStr(id, payload))
-
- upsert(newConcatStr)
- }
- }
-
- def update(id: String, payload: String): Future[Done] = {
- upsert(ConcatStr(id, payload))
- }
-
- def updateWithNullValue(id: String): Future[Done] = {
- upsert(ConcatStr(id, null))
- }
-
- private def upsert(concatStr: ConcatStr): Future[Done] = {
- logger.debug("TestRepository.upsert: [{}]", concatStr)
-
- val stmtSql = dialect match {
- case Dialect.Postgres | Dialect.Yugabyte =>
- sql"""
- INSERT INTO $table (id, concatenated) VALUES (?, ?)
- ON CONFLICT (id)
- DO UPDATE SET
- id = excluded.id,
- concatenated = excluded.concatenated
- """
- case Dialect.MySQL =>
- sql"""
- INSERT INTO $table (id, concatenated) VALUES (?, ?) AS excluded
- ON DUPLICATE KEY UPDATE
- id = excluded.id,
- concatenated = excluded.concatenated
- """
- }
- val stmt = session
- .createStatement(stmtSql)
- .bind(0, concatStr.id)
- .bind(1, concatStr.text)
-
- R2dbcExecutor
- .updateOneInTx(stmt)
- .map(_ => Done)
- }
-
- def findById(id: String): Future[Option[ConcatStr]] = {
- logger.debug("TestRepository.findById: [{}]", id)
-
- val stmtSql = sql"SELECT * FROM $table WHERE id = ?"
- val stmt = session
- .createStatement(stmtSql)
- .bind(0, id)
-
- R2dbcExecutor.selectOneInTx(
- stmt,
- row => ConcatStr(row.get("id", classOf[String]), row.get("concatenated", classOf[String])))
- }
-
- }
-}
-
-class R2dbcProjectionSpec
- extends ScalaTestWithActorTestKit(TestConfig.config)
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
- import R2dbcProjectionSpec._
-
- override def typedSystem: ActorSystem[_] = system
- private implicit val ec: ExecutionContext = system.executionContext
-
- private val logger = LoggerFactory.getLogger(getClass)
- private val settings = R2dbcProjectionSettings(testKit.system)
- private def createOffsetStore(projectionId: ProjectionId): R2dbcOffsetStore =
- R2dbcOffsetStore.fromConfig(projectionId, None, system, settings, r2dbcExecutor)
- private val projectionTestKit = ProjectionTestKit(system)
-
- override protected def beforeAll(): Unit = {
- super.beforeAll()
-
- Await.result(
- r2dbcExecutor.executeDdl("beforeAll createTable") { conn =>
- conn.createStatement(TestRepository.createTableSql)
- },
- 10.seconds)
- Await.result(
- r2dbcExecutor.updateOne("beforeAll delete")(_.createStatement(s"delete from ${TestRepository.table}")),
- 10.seconds)
- }
-
- private def offsetShouldBe(expected: Long)(implicit offsetStore: R2dbcOffsetStore) = {
- offsetStore.readOffset[Long]().futureValue shouldBe Some(expected)
- }
-
- private def offsetShouldBeEmpty()(implicit offsetStore: R2dbcOffsetStore) = {
- offsetStore.readOffset[Long]().futureValue shouldBe empty
- }
-
- private def projectedValueShouldBe(expected: String)(implicit entityId: String) = {
- val opt = withRepo(_.findById(entityId)).futureValue.map(_.text)
- opt shouldBe Some(expected)
- }
-
- private def projectedValueShouldInclude(expected: String)(implicit entityId: String) = {
- withClue(s"checking projected value contains [$expected]: ") {
- val text = withRepo(_.findById(entityId)).futureValue.get.text
- text should include(expected)
- }
- }
-
- private def projectedValueShould(actual: String => Boolean)(implicit entityId: String) = {
- withClue(s"checking projected value fulfil predicate: ") {
- val text = withRepo(_.findById(entityId)).futureValue.get.text
- actual(text)
- }
- }
-
- private def projectedValueShouldIncludeNTimes(expected: String, nTimes: Int)(implicit entityId: String) = {
- withClue(s"checking projected value contains [$expected] $nTimes times: ") {
- val text = withRepo(_.findById(entityId)).futureValue.get.text
- text.split("\\|").count(_ == expected) shouldBe nTimes
- }
- }
-
- // TODO: extract this to some utility
- @tailrec private def eventuallyExpectError(sinkProbe: TestSubscriber.Probe[_]): Throwable = {
- sinkProbe.expectNextOrError() match {
- case Right(_) => eventuallyExpectError(sinkProbe)
- case Left(exc) => exc
- }
- }
-
- private val concatHandlerFail4Msg = "fail on fourth envelope"
-
- private def withRepo[R](fun: TestRepository => Future[R]): Future[R] = {
- r2dbcExecutor.withConnection("test") { conn =>
- val session = new R2dbcSession(conn)
- fun(TestRepository(session, settings))
- }
- }
-
- class ConcatHandler(failPredicate: Long => Boolean = _ => false) extends R2dbcHandler[Envelope] {
-
- private val _attempts = new AtomicInteger()
- def attempts: Int = _attempts.get
-
- override def process(session: R2dbcSession, envelope: Envelope): Future[Done] = {
- if (failPredicate(envelope.offset)) {
- _attempts.incrementAndGet()
- throw TestException(concatHandlerFail4Msg + s" after $attempts attempts")
- } else {
- logger.debug("handling {}", envelope)
- TestRepository(session, settings).concatToText(envelope.id, envelope.message)
- }
- }
- }
-
- "A R2DBC exactly-once projection" must {
-
- "persist projection and offset in the same write operation (transactional)" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val projection =
- R2dbcProjection.exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => new ConcatHandler)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- offsetShouldBe(6L)
- }
-
- "skip failing events when using RecoveryStrategy.skip" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val bogusEventHandler = new ConcatHandler(_ == 4)
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.skip)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- projectedValueShouldBe("e1|e2|e3|e5|e6")
- }
- offsetShouldBe(6L)
- }
-
- "store offset for failing events when using RecoveryStrategy.skip" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val bogusEventHandler = new ConcatHandler(_ == 6)
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.skip)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- projectedValueShouldBe("e1|e2|e3|e4|e5")
- }
- offsetShouldBe(6L)
- }
-
- "skip failing events after retrying when using RecoveryStrategy.retryAndSkip" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val bogusEventHandler = new ConcatHandler(_ == 4)
-
- val statusProbe = createTestProbe[TestStatusObserver.Status]()
- val progressProbe = createTestProbe[TestStatusObserver.OffsetProgress[Envelope]]()
- val statusObserver =
- new TestStatusObserver[Envelope](statusProbe.ref, offsetProgressProbe = Some(progressProbe.ref))
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.retryAndSkip(3, 10.millis))
- .withStatusObserver(statusObserver)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- projectedValueShouldBe("e1|e2|e3|e5|e6")
- }
-
- // 1 + 3 => 1 original attempt and 3 retries
- bogusEventHandler.attempts shouldBe 1 + 3
-
- val someTestException = TestException("err")
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(entityId, 4, "e4"), someTestException))
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(entityId, 4, "e4"), someTestException))
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(entityId, 4, "e4"), someTestException))
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(entityId, 4, "e4"), someTestException))
- statusProbe.expectNoMessage()
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 1, "e1")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 2, "e2")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 3, "e3")))
- // Offset 4 is stored even though it failed and was skipped
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 4, "e4")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 5, "e5")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 6, "e6")))
-
- offsetShouldBe(6L)
- }
-
- "fail after retrying when using RecoveryStrategy.retryAndFail" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val bogusEventHandler = new ConcatHandler(_ == 4)
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.retryAndFail(3, 10.millis))
-
- offsetShouldBeEmpty()
- projectionTestKit.runWithTestSink(projectionFailing) { sinkProbe =>
- sinkProbe.request(1000)
- eventuallyExpectError(sinkProbe).getMessage should startWith(concatHandlerFail4Msg)
- }
- projectedValueShouldBe("e1|e2|e3")
- // 1 + 3 => 1 original attempt and 3 retries
- bogusEventHandler.attempts shouldBe 1 + 3
- offsetShouldBe(3L)
- }
-
- "restart from previous offset - fail with throwing an exception" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- def exactlyOnceProjection(failWhenOffset: Long => Boolean = _ => false) = {
- R2dbcProjection.exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => new ConcatHandler(failWhenOffset))
- }
-
- offsetShouldBeEmpty()
- projectionTestKit.runWithTestSink(exactlyOnceProjection(_ == 4)) { sinkProbe =>
- sinkProbe.request(1000)
- eventuallyExpectError(sinkProbe).getMessage should startWith(concatHandlerFail4Msg)
- }
- projectedValueShouldBe("e1|e2|e3")
- offsetShouldBe(3L)
-
- // re-run projection without failing function
- projectionTestKit.run(exactlyOnceProjection()) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- offsetShouldBe(6L)
- }
-
- "restart from previous offset - fail with bad insert on user code" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val bogusEventHandler = new R2dbcHandler[Envelope] {
- override def process(session: R2dbcSession, envelope: Envelope): Future[Done] = {
- val repo = TestRepository(session, settings)
- if (envelope.offset == 4L) repo.updateWithNullValue(envelope.id)
- else repo.concatToText(envelope.id, envelope.message)
- }
- }
-
- def exactlyOnceProjection(handler: () => R2dbcHandler[Envelope]) =
- R2dbcProjection.exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = handler)
-
- offsetShouldBeEmpty()
- projectionTestKit.runWithTestSink(exactlyOnceProjection(() => bogusEventHandler)) { sinkProbe =>
- sinkProbe.request(4)
- val exc = eventuallyExpectError(sinkProbe)
- exc.getMessage shouldBe "value must not be null"
- exc.getClass shouldBe classOf[IllegalArgumentException]
- }
- projectedValueShouldBe("e1|e2|e3")
- offsetShouldBe(3L)
-
- // re-run projection without failing function
- projectionTestKit.run(exactlyOnceProjection(() => new ConcatHandler())) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- offsetShouldBe(6L)
- }
-
- "verify offsets before and after processing an envelope" in {
-
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- case class ProbeMessage(str: String, offset: Long)
- val verificationProbe = createTestProbe[ProbeMessage]("verification")
- val processProbe = createTestProbe[ProbeMessage]("processing")
-
- val testVerification = (offset: Long) => {
- verificationProbe.ref ! ProbeMessage("verification", offset)
- VerificationSuccess
- }
-
- val handler = new R2dbcHandler[Envelope] {
- override def process(session: R2dbcSession, envelope: Envelope): Future[Done] = {
-
- verificationProbe.receiveMessage().offset shouldEqual envelope.offset
- processProbe.ref ! ProbeMessage("process", envelope.offset)
-
- TestRepository(session, settings).concatToText(envelope.id, envelope.message)
- }
- }
-
- val testSourceProvider = sourceProvider(entityId, verifyOffsetFn = testVerification)
-
- val projection =
- R2dbcProjection.exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = testSourceProvider,
- handler = () => handler)
-
- projectionTestKit.runWithTestSink(projection) { testSink =>
- for (i <- 1 to 6) {
- testSink.request(1)
- processProbe.receiveMessage().offset shouldBe i
- }
- }
-
- }
-
- "skip record if offset verification fails before processing envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val testVerification = (offset: Long) => {
- if (offset == 3L)
- VerificationFailure("test")
- else
- VerificationSuccess
- }
-
- val testSourceProvider = sourceProvider(entityId, verifyOffsetFn = testVerification)
-
- val projection =
- R2dbcProjection.exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = testSourceProvider,
- handler = () => new ConcatHandler())
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e4|e5|e6") // `e3` was skipped
- }
- }
-
- "skip record if offset verification fails after processing envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val testVerification = (offset: Long) => {
- if (offset == 3L)
- VerificationFailure("test")
- else
- VerificationSuccess
- }
-
- val testSourceProvider = sourceProvider(entityId, verifyOffsetFn = testVerification)
-
- val projection =
- R2dbcProjection.exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = testSourceProvider,
- handler = () => new ConcatHandler())
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e4|e5|e6") // `e3` was skipped
- }
- }
- }
-
- "A R2DBC grouped projection" must {
- "persist projection and offset in the same write operation (transactional)" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val handlerCalled = "called"
- val handlerProbe = createTestProbe[String]("calls-to-handler")
-
- val projection =
- R2dbcProjection
- .groupedWithin(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () =>
- R2dbcHandler[immutable.Seq[Envelope]] { (session, envelopes) =>
- handlerProbe.ref ! handlerCalled
- if (envelopes.isEmpty)
- Future.successful(Done)
- else {
- val repo = TestRepository(session, settings)
- val id = envelopes.head.id
- repo.findById(id).flatMap { existing =>
- val newConcatStr = envelopes.foldLeft(existing.getOrElse(ConcatStr(id, ""))) { (acc, env) =>
- acc.concat(env.message)
- }
- repo.update(id, newConcatStr.text)
- }
- }
- })
- .withGroup(3, 3.seconds)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- offsetShouldBe(6L)
-
- // handler probe is called twice
- handlerProbe.expectMessage(handlerCalled)
- handlerProbe.expectMessage(handlerCalled)
- }
-
- "handle grouped async projection and store offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val result = new StringBuffer()
-
- def handler(): Handler[immutable.Seq[Envelope]] = new Handler[immutable.Seq[Envelope]] {
- override def process(envelopes: immutable.Seq[Envelope]): Future[Done] = {
- Future {
- envelopes.foreach(env => result.append(env.message).append("|"))
- }.map(_ => Done)
- }
- }
-
- val projection =
- R2dbcProjection
- .groupedWithinAsync(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => handler())
- .withGroup(2, 3.seconds)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- result.toString shouldBe "e1|e2|e3|e4|e5|e6|"
- }
- eventually {
- offsetShouldBe(6L)
- }
- }
- }
-
- "A R2DBC at-least-once projection" must {
-
- "persist projection and offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val projection =
- R2dbcProjection.atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => new ConcatHandler)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- eventually {
- offsetShouldBe(6L)
- }
- }
-
- "skip failing events when using RecoveryStrategy.skip, save after 1" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val projection =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => new ConcatHandler(_ == 4))
- .withSaveOffset(1, 1.minute)
- .withRecoveryStrategy(HandlerRecoveryStrategy.skip)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e5|e6")
- }
- eventually {
- offsetShouldBe(6L)
- }
- }
-
- "skip failing events when using RecoveryStrategy.skip, save after 2" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val projection =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => new ConcatHandler(_ == 4))
- .withSaveOffset(2, 1.minute)
- .withRecoveryStrategy(HandlerRecoveryStrategy.skip)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e5|e6")
- }
- eventually {
- offsetShouldBe(6L)
- }
- }
-
- "restart from previous offset - handler throwing an exception, save after 1" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- def atLeastOnceProjection(failWhenOffset: Long => Boolean = _ => false) =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => new ConcatHandler(failWhenOffset))
- .withSaveOffset(1, Duration.Zero)
-
- offsetShouldBeEmpty()
- // run again up to safe point
- projectionTestKit.runWithTestSink(atLeastOnceProjection(_ == 4)) { sinkProbe =>
- sinkProbe.request(3)
- eventually {
- projectedValueShouldBe("e1|e2|e3")
- // we are saving after each envelope!
- offsetShouldBe(3)
- }
- }
-
- // run again up to failure point
- projectionTestKit.runWithTestSink(atLeastOnceProjection(_ == 4)) { sinkProbe =>
- sinkProbe.request(3)
- eventuallyExpectError(sinkProbe).getMessage should startWith(concatHandlerFail4Msg)
- eventually {
- // we re-start from 3, so `e3` is not redelivered
- projectedValueShouldBe("e1|e2|e3")
- offsetShouldBe(3)
- }
- }
- // re-run projection without failing function
- projectionTestKit.run(atLeastOnceProjection()) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
-
- eventually {
- offsetShouldBe(6L)
- }
- }
-
- "restart from previous offset - handler throwing an exception, save after 2" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- offsetStore.readOffset[Long]().futureValue shouldBe empty
-
- def atLeastOnceProjection(failWhenOffset: Long => Boolean = _ => false) =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => new ConcatHandler(failWhenOffset))
- .withSaveOffset(2, 1.minute)
-
- offsetShouldBeEmpty()
- // run again up to safe point
- projectionTestKit.runWithTestSink(atLeastOnceProjection(_ == 4)) { sinkProbe =>
- sinkProbe.request(3)
- eventually {
- projectedValueShouldBe("e1|e2|e3")
- }
- }
-
- // run again up to failure point
- projectionTestKit.runWithTestSink(atLeastOnceProjection(_ == 4)) { sinkProbe =>
- sinkProbe.request(4) // processes elem 3 again and fails when processing 4
- eventuallyExpectError(sinkProbe).getMessage should startWith(concatHandlerFail4Msg)
- eventually {
- // because failures, we may consume 'e1' and 'e2' more then once
- // we check that it at least starts with 'e1|e2|e3'
- projectedValueShould(_.startsWith("e1|e2|e3"))
-
- // note elem 3 is processed twice
- projectedValueShouldIncludeNTimes("e3", 2)
- }
- }
-
- // re-run projection without failing function
- projectionTestKit.run(atLeastOnceProjection()) {
- projectedValueShould(_.startsWith("e1|e2|e3"))
- projectedValueShould(_.endsWith("e3|e4|e5|e6"))
-
- // note elem 3 is should have been seen three times
- projectedValueShouldIncludeNTimes("e3", 3)
-
- offsetShouldBe(6L)
- }
- }
-
- "save offset after number of elements" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- import pekko.actor.typed.scaladsl.adapter._
- val sourceProbe = new AtomicReference[TestPublisher.Probe[Envelope]]()
- val source = TestSource[Envelope]()(system.toClassic).mapMaterializedValue { probe =>
- sourceProbe.set(probe)
- NotUsed
- }
-
- val projection =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = TestSourceProvider[Long, Envelope](source, _.offset),
- handler = () => new ConcatHandler())
- .withSaveOffset(10, 1.minute)
-
- projectionTestKit.runWithTestSink(projection) { sinkProbe =>
- eventually {
- sourceProbe.get should not be null
- }
- sinkProbe.request(1000)
-
- (1 to 15).foreach { n =>
- sourceProbe.get.sendNext(Envelope(entityId, n, s"elem-$n"))
- }
- eventually {
- projectedValueShouldInclude("elem-15")
- offsetShouldBe(10L)
- }
-
- (16 to 22).foreach { n =>
- sourceProbe.get.sendNext(Envelope(entityId, n, s"elem-$n"))
- }
- eventually {
- projectedValueShouldInclude("elem-22")
- offsetShouldBe(20L)
- }
- }
- }
-
- "save offset after idle duration" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- import pekko.actor.typed.scaladsl.adapter._
- val sourceProbe = new AtomicReference[TestPublisher.Probe[Envelope]]()
- val source = TestSource[Envelope]()(system.toClassic).mapMaterializedValue { probe =>
- sourceProbe.set(probe)
- NotUsed
- }
-
- val projection =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = TestSourceProvider[Long, Envelope](source, _.offset),
- handler = () => new ConcatHandler())
- .withSaveOffset(10, 2.seconds)
-
- projectionTestKit.runWithTestSink(projection) { sinkProbe =>
- eventually {
- sourceProbe.get should not be null
- }
- sinkProbe.request(1000)
-
- (1 to 15).foreach { n =>
- sourceProbe.get.sendNext(Envelope(entityId, n, s"elem-$n"))
- }
- eventually {
- projectedValueShouldInclude("elem-15")
- offsetShouldBe(10L)
- }
-
- (16 to 17).foreach { n =>
- sourceProbe.get.sendNext(Envelope(entityId, n, s"elem-$n"))
- }
- eventually {
- projectedValueShouldInclude("elem-17")
- offsetShouldBe(17L)
- }
- }
-
- }
-
- "verify offsets before processing an envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
- val verifiedProbe = createTestProbe[Long]()
-
- val testVerification = (offset: Long) => {
- verifiedProbe.ref ! offset
- VerificationSuccess
- }
-
- val testSourceProvider = sourceProvider(entityId, verifyOffsetFn = testVerification)
-
- val projection =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = testSourceProvider,
- handler = () =>
- R2dbcHandler[Envelope] { (session, envelope) =>
- verifiedProbe.expectMessage(envelope.offset)
- TestRepository(session, settings).concatToText(envelope.id, envelope.message)
- })
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- }
-
- "skip record if offset verification fails before processing envelope" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val testVerification = (offset: Long) => {
- if (offset == 3L)
- VerificationFailure("test")
- else
- VerificationSuccess
- }
-
- val testSourceProvider = sourceProvider(entityId, verifyOffsetFn = testVerification)
-
- val projection =
- R2dbcProjection
- .atLeastOnce(
- projectionId,
- Some(settings),
- sourceProvider = testSourceProvider,
- handler = () => new ConcatHandler())
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e4|e5|e6") // `e3` was skipped
- }
- }
-
- "handle async projection and store offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val result = new StringBuffer()
-
- def handler(): Handler[Envelope] = new Handler[Envelope] {
- override def process(envelope: Envelope): Future[Done] = {
- Future {
- result.append(envelope.message).append("|")
- }.map(_ => Done)
- }
- }
-
- val projection =
- R2dbcProjection.atLeastOnceAsync(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () => handler())
-
- projectionTestKit.run(projection) {
- result.toString shouldBe "e1|e2|e3|e4|e5|e6|"
- }
- eventually {
- offsetShouldBe(6L)
- }
- }
- }
-
- "A R2DBC flow projection" must {
-
- "persist projection and offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- offsetShouldBeEmpty()
-
- val flowHandler =
- FlowWithContext[Envelope, ProjectionContext]
- .mapAsync(1) { env =>
- withRepo(_.concatToText(env.id, env.message))
- }
-
- val projection =
- R2dbcProjection
- .atLeastOnceFlow(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = flowHandler)
- .withSaveOffset(1, 1.minute)
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- eventually {
- offsetShouldBe(6L)
- }
-
- }
- }
-
- "R2dbcProjection lifecycle" must {
-
- class LifecycleHandler(
- probe: ActorRef[String],
- failOnceOnOffset: AtomicInteger = new AtomicInteger(-1),
- alwaysFailOnOffset: Int = -1)
- extends R2dbcHandler[Envelope] {
-
- val createdMessage = "created"
- val startMessage = "start"
- val completedMessage = "completed"
- val failedMessage = "failed"
-
- // stop message can be 'completed' or 'failed'
- // that allows us to assert that the stopHandler is different execution paths were called in test
- private var stopMessage = completedMessage
-
- probe ! createdMessage
-
- override def start(): Future[Done] = {
- // reset stop message to 'completed' on each new start
- stopMessage = completedMessage
- probe ! startMessage
- Future.successful(Done)
- }
-
- override def stop(): Future[Done] = {
- probe ! stopMessage
- Future.successful(Done)
- }
-
- override def process(session: R2dbcSession, envelope: Envelope): Future[Done] = {
- if (envelope.offset == failOnceOnOffset.get()) {
- failOnceOnOffset.set(-1)
- stopMessage = failedMessage
- throw TestException(s"Fail $failOnceOnOffset")
- } else if (envelope.offset == alwaysFailOnOffset) {
- stopMessage = failedMessage
- throw TestException(s"Always Fail $alwaysFailOnOffset")
- } else {
- probe ! envelope.message
- Future.successful(Done)
- }
- }
- }
-
- "call start and stop of the handler" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val handlerProbe = createTestProbe[String]()
- val handler = new LifecycleHandler(handlerProbe.ref)
-
- val statusProbe = createTestProbe[TestStatusObserver.Status]()
- val statusObserver = new TestStatusObserver[Envelope](statusProbe.ref, lifecycle = true)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider = sourceProvider(entityId), handler = () => handler)
- .withStatusObserver(statusObserver)
-
- // not using ProjectionTestKit because want to test restarts
- spawn(ProjectionBehavior(projection))
-
- handlerProbe.expectMessage(handler.createdMessage)
-
- statusProbe.expectMessage(TestStatusObserver.Started)
-
- handlerProbe.expectMessage(handler.startMessage)
- handlerProbe.expectMessage("e1")
- handlerProbe.expectMessage("e2")
- handlerProbe.expectMessage("e3")
- handlerProbe.expectMessage("e4")
- handlerProbe.expectMessage("e5")
- handlerProbe.expectMessage("e6")
- // completed without failure
- handlerProbe.expectMessage(handler.completedMessage)
- handlerProbe.expectNoMessage() // no duplicate stop
-
- statusProbe.expectMessage(TestStatusObserver.Stopped)
- statusProbe.expectNoMessage()
- }
-
- "call start and stop of the handler when using TestKit.runWithTestSink" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val handlerProbe = createTestProbe[String]()
- val handler = new LifecycleHandler(handlerProbe.ref)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider = sourceProvider(entityId), handler = () => handler)
-
- // not using ProjectionTestKit because want to test restarts
- projectionTestKit.runWithTestSink(projection) { sinkProbe =>
- // request all 'strings' (e1 to e6)
-
- handlerProbe.expectMessage(handler.createdMessage)
-
- // the start happens inside runWithTestSink
- handlerProbe.expectMessage(handler.startMessage)
-
- // request the elements
- sinkProbe.request(6)
- handlerProbe.expectMessage("e1")
- handlerProbe.expectMessage("e2")
- handlerProbe.expectMessage("e3")
- handlerProbe.expectMessage("e4")
- handlerProbe.expectMessage("e5")
- handlerProbe.expectMessage("e6")
-
- // all elements should have reached the sink
- sinkProbe.expectNextN(6)
- }
-
- // completed without failure
- handlerProbe.expectMessage(handler.completedMessage)
- handlerProbe.expectNoMessage() // no duplicate stop
- }
-
- "call start and stop of handler when restarted" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val handlerProbe = createTestProbe[String]()
- @volatile var _handler: Option[LifecycleHandler] = None
- val failOnceOnOffset = new AtomicInteger(4)
- val handlerFactory = () => {
- val newHandler = new LifecycleHandler(handlerProbe.ref, failOnceOnOffset)
- _handler = Some(newHandler)
- newHandler
- }
-
- def handler: LifecycleHandler = _handler match {
- case Some(h) => h
- case None =>
- handlerProbe.awaitAssert {
- _handler.get
- }
- }
-
- val statusProbe = createTestProbe[TestStatusObserver.Status]()
- val progressProbe = createTestProbe[TestStatusObserver.OffsetProgress[Envelope]]()
- val statusObserver = new TestStatusObserver[Envelope](statusProbe.ref, lifecycle = true, Some(progressProbe.ref))
-
- val projection =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider = sourceProvider(entityId), handlerFactory)
- .withRestartBackoff(1.second, 2.seconds, 0.0)
- .withStatusObserver(statusObserver)
-
- // not using ProjectionTestKit because want to test restarts
- spawn(ProjectionBehavior(projection))
-
- handlerProbe.expectMessage(handler.createdMessage)
-
- statusProbe.expectMessage(TestStatusObserver.Started)
-
- handlerProbe.expectMessage(handler.startMessage)
- handlerProbe.expectMessage("e1")
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 1, "e1")))
- handlerProbe.expectMessage("e2")
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 2, "e2")))
- handlerProbe.expectMessage("e3")
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 3, "e3")))
- // fail 4
- handlerProbe.expectMessage(handler.failedMessage)
- val someTestException = TestException("err")
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(entityId, 4, "e4"), someTestException))
-
- // backoff will restart
- statusProbe.expectMessage(TestStatusObserver.Failed)
- statusProbe.expectMessage(TestStatusObserver.Stopped)
- handlerProbe.expectMessage(handler.createdMessage)
- handlerProbe.expectMessage(handler.startMessage)
- statusProbe.expectMessage(TestStatusObserver.Started)
- handlerProbe.expectMessage("e4")
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 4, "e4")))
- handlerProbe.expectMessage("e5")
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 5, "e5")))
- handlerProbe.expectMessage("e6")
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(entityId, 6, "e6")))
- // now completed without failure
- handlerProbe.expectMessage(handler.completedMessage)
- handlerProbe.expectNoMessage() // no duplicate stop
-
- statusProbe.expectMessage(TestStatusObserver.Stopped)
- statusProbe.expectNoMessage()
- }
-
- "call start and stop of handler when failed but no restart" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val handlerProbe = createTestProbe[String]()
- val failOnceOnOffset = new AtomicInteger(4)
- val handler = new LifecycleHandler(handlerProbe.ref, failOnceOnOffset)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider = sourceProvider(entityId), handler = () => handler)
- .withRestartBackoff(1.second, 2.seconds, 0.0, maxRestarts = 0)
-
- // not using ProjectionTestKit because want to test restarts
- spawn(ProjectionBehavior(projection))
-
- handlerProbe.expectMessage(handler.createdMessage)
- handlerProbe.expectMessage(handler.startMessage)
- handlerProbe.expectMessage("e1")
- handlerProbe.expectMessage("e2")
- handlerProbe.expectMessage("e3")
- // fail 4, not restarted
- // completed with failure
- handlerProbe.expectMessage(handler.failedMessage)
- handlerProbe.expectNoMessage() // no duplicate stop
- }
-
- "be able to stop when retrying" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val handlerProbe = createTestProbe[String]()
- val handler = new LifecycleHandler(handlerProbe.ref, alwaysFailOnOffset = 4)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider = sourceProvider(entityId), handler = () => handler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.retryAndFail(100, 100.millis))
-
- val ref = spawn(ProjectionBehavior(projection))
-
- handlerProbe.expectMessage(handler.createdMessage)
- handlerProbe.expectMessage(handler.startMessage)
- handlerProbe.expectMessage("e1")
- handlerProbe.expectMessage("e2")
- handlerProbe.expectMessage("e3")
- // fail 4
-
- // let it retry for a while
- Thread.sleep(300)
-
- ref ! ProjectionBehavior.Stop
- createTestProbe().expectTerminated(ref)
- }
- }
-
- "R2dbcProjection management" must {
-
- "restart from beginning when offset is cleared" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () =>
- R2dbcHandler[Envelope] { (session, envelope) =>
- TestRepository(session, settings).concatToText(envelope.id, envelope.message)
- })
-
- offsetShouldBeEmpty()
-
- // not using ProjectionTestKit because want to test ProjectionManagement
- spawn(ProjectionBehavior(projection))
- eventually {
- offsetShouldBe(6L)
- }
- ProjectionManagement(system).getOffset(projectionId).futureValue shouldBe Some(6L)
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
-
- ProjectionManagement(system).clearOffset(projectionId).futureValue shouldBe Done
- eventually {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6|e1|e2|e3|e4|e5|e6")
- }
- }
-
- "restart from updated offset" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () =>
- R2dbcHandler[Envelope] { (session, envelope) =>
- TestRepository(session, settings).concatToText(envelope.id, envelope.message)
- })
-
- offsetShouldBeEmpty()
-
- // not using ProjectionTestKit because want to test ProjectionManagement
- spawn(ProjectionBehavior(projection))
-
- eventually {
- offsetShouldBe(6L)
- }
- ProjectionManagement(system).getOffset(projectionId).futureValue shouldBe Some(6L)
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
-
- ProjectionManagement(system).updateOffset(projectionId, 3L).futureValue shouldBe Done
- eventually {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6|e4|e5|e6")
- }
- }
-
- "pause projection" in {
- implicit val entityId: String = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider = sourceProvider(entityId),
- handler = () =>
- R2dbcHandler[Envelope] { (session, envelope) =>
- TestRepository(session, settings).concatToText(envelope.id, envelope.message)
- })
-
- offsetShouldBeEmpty()
-
- // not using ProjectionTestKit because want to test ProjectionManagement
- spawn(ProjectionBehavior(projection))
-
- val mgmt = ProjectionManagement(system)
-
- mgmt.isPaused(projectionId).futureValue shouldBe false
-
- eventually {
- offsetShouldBe(6L)
- }
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
-
- mgmt.pause(projectionId).futureValue shouldBe Done
- mgmt.clearOffset(projectionId).futureValue shouldBe Done
-
- mgmt.isPaused(projectionId).futureValue shouldBe true
-
- Thread.sleep(500)
- // not updated because paused
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
-
- mgmt.resume(projectionId)
-
- mgmt.isPaused(projectionId).futureValue shouldBe false
-
- eventually {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6|e1|e2|e3|e4|e5|e6")
- }
- }
- }
-
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetProjectionSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetProjectionSpec.scala
deleted file mode 100644
index 3940882..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetProjectionSpec.scala
+++ /dev/null
@@ -1,1460 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.time.Instant
-import java.time.{ Duration => JDuration }
-import java.util.UUID
-import java.util.concurrent.atomic.AtomicBoolean
-import java.util.concurrent.atomic.AtomicInteger
-
-import scala.annotation.tailrec
-import scala.collection.immutable
-import scala.concurrent.Await
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-import scala.concurrent.duration._
-import org.apache.pekko
-import pekko.Done
-import pekko.NotUsed
-import pekko.actor.testkit.typed.TestException
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.typed.ActorRef
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.query.TimestampOffset
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
-import pekko.persistence.query.typed.scaladsl.LoadEventQuery
-import pekko.persistence.typed.PersistenceId
-import pekko.persistence.r2dbc.internal.EnvelopeOrigin
-import pekko.projection.BySlicesSourceProvider
-import pekko.projection.HandlerRecoveryStrategy
-import pekko.projection.ProjectionBehavior
-import pekko.projection.ProjectionContext
-import pekko.projection.ProjectionId
-import pekko.projection.TestStatusObserver
-import pekko.projection.TestStatusObserver.Err
-import pekko.projection.TestStatusObserver.OffsetProgress
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import pekko.projection.scaladsl.Handler
-import pekko.projection.scaladsl.ProjectionManagement
-import pekko.projection.scaladsl.SourceProvider
-import pekko.projection.testkit.scaladsl.ProjectionTestKit
-import pekko.projection.testkit.scaladsl.TestSourceProvider
-import pekko.stream.scaladsl.FlowWithContext
-import pekko.stream.scaladsl.Source
-import pekko.stream.testkit.TestSubscriber
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-object R2dbcTimestampOffsetProjectionSpec {
-
- final case class Envelope(id: String, seqNr: Long, message: String)
-
- /**
- * This variant of TestStatusObserver is useful when the incoming envelope is the original pekko projection
- * EventBySliceEnvelope, but we want to assert on [[Envelope]]. The original [[EventEnvelope]] has too many params
- * that are not so interesting for the test including the offset timestamp that would make the it harder to test.
- */
- class R2dbcTestStatusObserver(
- statusProbe: ActorRef[TestStatusObserver.Status],
- progressProbe: ActorRef[TestStatusObserver.OffsetProgress[Envelope]])
- extends TestStatusObserver[EventEnvelope[String]](statusProbe.ref) {
- override def offsetProgress(projectionId: ProjectionId, envelope: EventEnvelope[String]): Unit =
- progressProbe ! OffsetProgress(
- Envelope(envelope.persistenceId, envelope.sequenceNr, envelope.eventOption.getOrElse("None")))
-
- override def error(
- projectionId: ProjectionId,
- envelope: EventEnvelope[String],
- cause: Throwable,
- recoveryStrategy: HandlerRecoveryStrategy): Unit =
- statusProbe ! Err(
- Envelope(envelope.persistenceId, envelope.sequenceNr, envelope.eventOption.getOrElse("None")),
- cause)
- }
-
- class TestTimestampSourceProvider(
- envelopes: immutable.IndexedSeq[EventEnvelope[String]],
- testSourceProvider: TestSourceProvider[TimestampOffset, EventEnvelope[String]],
- override val maxSlice: Int)
- extends SourceProvider[TimestampOffset, EventEnvelope[String]]
- with BySlicesSourceProvider
- with EventTimestampQuery
- with LoadEventQuery {
-
- override def source(offset: () => Future[Option[TimestampOffset]]): Future[Source[EventEnvelope[String], NotUsed]] =
- testSourceProvider.source(offset)
-
- override def extractOffset(envelope: EventEnvelope[String]): TimestampOffset =
- testSourceProvider.extractOffset(envelope)
-
- override def extractCreationTime(envelope: EventEnvelope[String]): Long =
- testSourceProvider.extractCreationTime(envelope)
-
- override def minSlice: Int = 0
-
- override def timestampOf(persistenceId: String, sequenceNr: Long): Future[Option[Instant]] = {
- Future.successful(envelopes.collectFirst {
- case env
- if env.persistenceId == persistenceId && env.sequenceNr == sequenceNr &&
- env.offset
- .isInstanceOf[TimestampOffset] =>
- env.offset.asInstanceOf[TimestampOffset].timestamp
- })
- }
-
- override def loadEnvelope[Event](persistenceId: String, sequenceNr: Long): Future[EventEnvelope[Event]] = {
- envelopes.collectFirst {
- case env if env.persistenceId == persistenceId && env.sequenceNr == sequenceNr =>
- env.asInstanceOf[EventEnvelope[Event]]
- } match {
- case Some(env) => Future.successful(env)
- case None =>
- Future.failed(
- new NoSuchElementException(
- s"Event with persistenceId [$persistenceId] and sequenceNr [$sequenceNr] not found."))
- }
- }
- }
-
-}
-
-class R2dbcTimestampOffsetProjectionSpec
- extends ScalaTestWithActorTestKit(TestConfig.config)
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
- import R2dbcOffsetStore.Pid
- import R2dbcOffsetStore.SeqNr
- import R2dbcProjectionSpec.ConcatStr
- import R2dbcProjectionSpec.TestRepository
- import R2dbcTimestampOffsetProjectionSpec._
-
- override def typedSystem: ActorSystem[_] = system
- private implicit val ec: ExecutionContext = system.executionContext
-
- private val logger = LoggerFactory.getLogger(getClass)
- private val settings = R2dbcProjectionSettings(testKit.system)
- private def createOffsetStore(
- projectionId: ProjectionId,
- sourceProvider: TestTimestampSourceProvider): R2dbcOffsetStore =
- R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
- private val projectionTestKit = ProjectionTestKit(system)
-
- override protected def beforeAll(): Unit = {
- super.beforeAll()
-
- Await.result(
- r2dbcExecutor.executeDdl("beforeAll createTable") { conn =>
- conn.createStatement(TestRepository.createTableSql)
- },
- 10.seconds)
- Await.result(
- r2dbcExecutor.updateOne("beforeAll delete")(_.createStatement(s"delete from ${TestRepository.table}")),
- 10.seconds)
- }
-
- def createSourceProvider(
- envelopes: immutable.IndexedSeq[EventEnvelope[String]],
- complete: Boolean = true): TestTimestampSourceProvider = {
- val sp = TestSourceProvider[TimestampOffset, EventEnvelope[String]](
- Source(envelopes),
- _.offset.asInstanceOf[TimestampOffset])
- .withStartSourceFrom { (lastProcessedOffset, offset) =>
- offset.timestamp.isBefore(lastProcessedOffset.timestamp) ||
- (offset.timestamp == lastProcessedOffset.timestamp && offset.seen == lastProcessedOffset.seen)
- }
- .withAllowCompletion(complete)
-
- new TestTimestampSourceProvider(envelopes, sp, persistenceExt.numberOfSlices - 1)
- }
-
- def createBacktrackingSourceProvider(
- envelopes: immutable.IndexedSeq[EventEnvelope[String]],
- complete: Boolean = true): TestTimestampSourceProvider = {
- val sp = TestSourceProvider[TimestampOffset, EventEnvelope[String]](
- Source(envelopes),
- _.offset.asInstanceOf[TimestampOffset])
- .withStartSourceFrom { (_, _) => false } // include all
- .withAllowCompletion(complete)
- new TestTimestampSourceProvider(envelopes, sp, persistenceExt.numberOfSlices - 1)
- }
-
- private def offsetShouldBe[Offset](expected: Offset)(implicit offsetStore: R2dbcOffsetStore) = {
- val offset = offsetStore.readOffset[TimestampOffset]().futureValue
- offset shouldBe Some(expected.asInstanceOf[TimestampOffset].copy(readTimestamp = Instant.EPOCH))
- }
-
- private def offsetShouldBeEmpty()(implicit offsetStore: R2dbcOffsetStore) = {
- offsetStore.readOffset[TimestampOffset]().futureValue shouldBe empty
- }
-
- private def projectedValueShouldBe(expected: String)(implicit entityId: String) = {
- val opt = withRepo(_.findById(entityId)).futureValue.map(_.text)
- opt shouldBe Some(expected)
- }
-
- private def projectedValueShouldBeEmpty()(implicit entityId: String) = {
- val opt = withRepo(_.findById(entityId)).futureValue.map(_.text)
- opt shouldBe None
- }
-
- // TODO: extract this to some utility
- @tailrec private def eventuallyExpectError(sinkProbe: TestSubscriber.Probe[_]): Throwable = {
- sinkProbe.expectNextOrError() match {
- case Right(_) => eventuallyExpectError(sinkProbe)
- case Left(exc) => exc
- }
- }
-
- private val concatHandlerFail4Msg = "fail on fourth envelope"
-
- private def withRepo[R](fun: TestRepository => Future[R]): Future[R] = {
- r2dbcExecutor.withConnection("test") { conn =>
- val session = new R2dbcSession(conn)
- fun(TestRepository(session, settings))
- }
- }
-
- class ConcatHandler(failPredicate: EventEnvelope[String] => Boolean = _ => false)
- extends R2dbcHandler[EventEnvelope[String]] {
-
- private val logger = LoggerFactory.getLogger(getClass)
- private val _attempts = new AtomicInteger()
- def attempts: Int = _attempts.get
-
- override def process(session: R2dbcSession, envelope: EventEnvelope[String]): Future[Done] = {
- if (failPredicate(envelope)) {
- _attempts.incrementAndGet()
- throw TestException(concatHandlerFail4Msg + s" after $attempts attempts")
- } else {
- logger.debug(s"handling {}", envelope)
- TestRepository(session, settings).concatToText(envelope.persistenceId, envelope.event)
- }
- }
-
- }
-
- private val clock = TestClock.nowMicros()
- def tick(): TestClock = {
- clock.tick(JDuration.ofMillis(1))
- clock
- }
-
- def createEnvelope(pid: Pid, seqNr: SeqNr, timestamp: Instant, event: String): EventEnvelope[String] = {
- val entityType = PersistenceId.extractEntityType(pid)
- val slice = persistenceExt.sliceForPersistenceId(pid)
- EventEnvelope(
- TimestampOffset(timestamp, timestamp.plusMillis(1000), Map(pid -> seqNr)),
- pid,
- seqNr,
- event,
- timestamp.toEpochMilli,
- entityType,
- slice)
- }
-
- def backtrackingEnvelope(env: EventEnvelope[String]): EventEnvelope[String] =
- new EventEnvelope[String](
- env.offset,
- env.persistenceId,
- env.sequenceNr,
- eventOption = None,
- env.timestamp,
- env.eventMetadata,
- env.entityType,
- env.slice,
- env.filtered,
- source = EnvelopeOrigin.SourceBacktracking)
-
- def createEnvelopes(pid: Pid, numberOfEvents: Int): immutable.IndexedSeq[EventEnvelope[String]] = {
- (1 to numberOfEvents).map { n =>
- createEnvelope(pid, n, tick().instant(), s"e$n")
- }
- }
-
- def now(): Instant = {
- TestClock.nowMicros().instant()
- }
-
- def createEnvelopesWithDuplicates(pid1: Pid, pid2: Pid): Vector[EventEnvelope[String]] = {
- val startTime = now()
-
- Vector(
- createEnvelope(pid1, 1, startTime, "e1-1"),
- createEnvelope(pid1, 2, startTime.plusMillis(1), "e1-2"),
- createEnvelope(pid2, 1, startTime.plusMillis(2), "e2-1"),
- createEnvelope(pid1, 3, startTime.plusMillis(4), "e1-3"),
- // pid1-3 is emitted before pid2-2 even though pid2-2 timestamp is earlier,
- // from backtracking query previous events are emitted again, including the missing pid2-2
- createEnvelope(pid1, 1, startTime, "e1-1"),
- createEnvelope(pid1, 2, startTime.plusMillis(1), "e1-2"),
- createEnvelope(pid2, 1, startTime.plusMillis(2), "e2-1"),
- // now it pid2-2 is included
- createEnvelope(pid2, 2, startTime.plusMillis(3), "e2-2"),
- createEnvelope(pid1, 3, startTime.plusMillis(4), "e1-3"),
- // and then some normal again
- createEnvelope(pid1, 4, startTime.plusMillis(5), "e1-4"),
- createEnvelope(pid2, 3, startTime.plusMillis(6), "e2-3"))
- }
-
- def createEnvelopesUnknownSequenceNumbers(startTime: Instant, pid1: Pid, pid2: Pid): Vector[EventEnvelope[String]] = {
- Vector(
- createEnvelope(pid1, 1, startTime, "e1-1"),
- createEnvelope(pid1, 2, startTime.plusMillis(1), "e1-2"),
- createEnvelope(pid2, 1, startTime.plusMillis(2), "e2-1"),
- // pid2 seqNr 2 missing, will reject 3
- createEnvelope(pid2, 3, startTime.plusMillis(4), "e2-3"),
- createEnvelope(pid1, 3, startTime.plusMillis(5), "e1-3"),
- // pid1 seqNr 4 missing, will reject 5
- createEnvelope(pid1, 5, startTime.plusMillis(7), "e1-5"))
- }
-
- def createEnvelopesBacktrackingUnknownSequenceNumbers(
- startTime: Instant,
- pid1: Pid,
- pid2: Pid): Vector[EventEnvelope[String]] = {
- Vector(
- // may also contain some duplicates
- createEnvelope(pid1, 2, startTime.plusMillis(1), "e1-2"),
- createEnvelope(pid2, 2, startTime.plusMillis(3), "e2-2"),
- createEnvelope(pid2, 3, startTime.plusMillis(4), "e2-3"),
- createEnvelope(pid1, 3, startTime.plusMillis(5), "e1-3"),
- createEnvelope(pid1, 4, startTime.plusMillis(6), "e1-4"),
- createEnvelope(pid1, 5, startTime.plusMillis(7), "e1-5"),
- createEnvelope(pid2, 4, startTime.plusMillis(8), "e2-4"),
- createEnvelope(pid1, 6, startTime.plusMillis(9), "e1-6"))
- }
-
- def groupedHandler(probe: ActorRef[String]): R2dbcHandler[immutable.Seq[EventEnvelope[String]]] = {
- R2dbcHandler[immutable.Seq[EventEnvelope[String]]] { (session, envelopes) =>
- probe ! "called"
- if (envelopes.isEmpty)
- Future.successful(Done)
- else {
- val repo = TestRepository(session, settings)
- val results = envelopes.groupBy(_.persistenceId).map { case (pid, envs) =>
- repo.findById(pid).flatMap { existing =>
- val newConcatStr = envs.foldLeft(existing.getOrElse(ConcatStr(pid, ""))) { (acc, env) =>
- acc.concat(env.event)
- }
- repo.update(pid, newConcatStr.text)
- }
- }
- Future.sequence(results).map(_ => Done)
- }
- }
- }
-
- def markAsFilteredEvent[A](env: EventEnvelope[A]): EventEnvelope[A] = {
- new EventEnvelope[A](
- env.offset,
- env.persistenceId,
- env.sequenceNr,
- env.eventOption,
- env.timestamp,
- env.eventMetadata,
- env.entityType,
- env.slice,
- filtered = true,
- env.source)
- }
-
- "A R2DBC exactly-once projection with TimestampOffset" must {
-
- "persist projection and offset in the same write operation (transactional)" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection.exactlyOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- offsetShouldBe(envelopes.last.offset)
- }
-
- "skip failing events when using RecoveryStrategy.skip" in {
- implicit val pid1 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid1, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val bogusEventHandler = new ConcatHandler(_.sequenceNr == 4)
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider, handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.skip)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- projectedValueShouldBe("e1|e2|e3|e5|e6")
- }
- offsetShouldBe(envelopes.last.offset)
- }
-
- "store offset for failing events when using RecoveryStrategy.skip" in {
- implicit val pid1 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid1, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val bogusEventHandler = new ConcatHandler(_.sequenceNr == 6)
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider, handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.skip)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- projectedValueShouldBe("e1|e2|e3|e4|e5")
- }
- offsetShouldBe(envelopes.last.offset)
- }
-
- "skip failing events after retrying when using RecoveryStrategy.retryAndSkip" in {
- implicit val pid1 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid1, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val statusProbe = createTestProbe[TestStatusObserver.Status]()
- val progressProbe = createTestProbe[TestStatusObserver.OffsetProgress[Envelope]]()
- val statusObserver = new R2dbcTestStatusObserver(statusProbe.ref, progressProbe.ref)
- val bogusEventHandler = new ConcatHandler(_.sequenceNr == 4)
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider, handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.retryAndSkip(3, 10.millis))
- .withStatusObserver(statusObserver)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- projectedValueShouldBe("e1|e2|e3|e5|e6")
- }
-
- // 1 + 3 => 1 original attempt and 3 retries
- bogusEventHandler.attempts shouldBe 1 + 3
-
- val someTestException = TestException("err")
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(pid1, 4, "e4"), someTestException))
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(pid1, 4, "e4"), someTestException))
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(pid1, 4, "e4"), someTestException))
- statusProbe.expectMessage(TestStatusObserver.Err(Envelope(pid1, 4, "e4"), someTestException))
- statusProbe.expectNoMessage()
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(pid1, 1, "e1")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(pid1, 2, "e2")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(pid1, 3, "e3")))
- // Offset 4 is stored even though it failed and was skipped
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(pid1, 4, "e4")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(pid1, 5, "e5")))
- progressProbe.expectMessage(TestStatusObserver.OffsetProgress(Envelope(pid1, 6, "e6")))
-
- offsetShouldBe(envelopes.last.offset)
- }
-
- "fail after retrying when using RecoveryStrategy.retryAndFail" in {
- implicit val pid1 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid1, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val bogusEventHandler = new ConcatHandler(_.sequenceNr == 4)
-
- val projectionFailing =
- R2dbcProjection
- .exactlyOnce(projectionId, Some(settings), sourceProvider, handler = () => bogusEventHandler)
- .withRecoveryStrategy(HandlerRecoveryStrategy.retryAndFail(3, 10.millis))
-
- offsetShouldBeEmpty()
- projectionTestKit.runWithTestSink(projectionFailing) { sinkProbe =>
- sinkProbe.request(1000)
- eventuallyExpectError(sinkProbe).getMessage should startWith(concatHandlerFail4Msg)
- }
- projectedValueShouldBe("e1|e2|e3")
- // 1 + 3 => 1 original attempt and 3 retries
- bogusEventHandler.attempts shouldBe 1 + 3
-
- offsetShouldBe(envelopes(2).offset) // <- offset is from e3
- }
-
- "restart from previous offset - fail with throwing an exception" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- def exactlyOnceProjection(failWhen: EventEnvelope[String] => Boolean = _ => false) = {
- R2dbcProjection.exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider,
- handler = () => new ConcatHandler(failWhen))
- }
-
- offsetShouldBeEmpty()
- projectionTestKit.runWithTestSink(exactlyOnceProjection(_.sequenceNr == 4)) { sinkProbe =>
- sinkProbe.request(1000)
- eventuallyExpectError(sinkProbe).getMessage should startWith(concatHandlerFail4Msg)
- }
- projectedValueShouldBe("e1|e2|e3")
- offsetShouldBe(envelopes(2).offset)
-
- // re-run projection without failing function
- projectionTestKit.run(exactlyOnceProjection()) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- offsetShouldBe(envelopes.last.offset)
- }
-
- "filter out duplicates" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopesWithDuplicates(pid1, pid2)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection.exactlyOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3")(pid2)
- }
- offsetShouldBe(envelopes.last.offset)
- }
-
- "filter out unknown sequence numbers" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val startTime = now()
- val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
- val sourceProvider1 = createSourceProvider(envelopes1)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider1)
-
- val projection1 =
- R2dbcProjection.exactlyOnce(projectionId, Some(settings), sourceProvider1, handler = () => new ConcatHandler)
-
- projectionTestKit.run(projection1) {
- projectedValueShouldBe("e1-1|e1-2|e1-3")(pid1)
- projectedValueShouldBe("e2-1")(pid2)
- }
- offsetShouldBe(envelopes1.collectFirst { case env if env.event == "e1-3" => env.offset }.get)
-
- // simulate backtracking
- logger.debug("Starting backtracking")
- val envelopes2 = createEnvelopesBacktrackingUnknownSequenceNumbers(startTime, pid1, pid2)
- val sourceProvider2 = createBacktrackingSourceProvider(envelopes2)
- val projection2 =
- R2dbcProjection.exactlyOnce(projectionId, Some(settings), sourceProvider2, handler = () => new ConcatHandler)
-
- projectionTestKit.run(projection2) {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4|e1-5|e1-6")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3|e2-4")(pid2)
- }
- offsetShouldBe(envelopes2.last.offset)
- }
-
- "be able to skip envelopes but still store offset" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val envelopes = createEnvelopes(pid, 6).map { env =>
- if (env.event == "e3" || env.event == "e4" || env.event == "e6")
- markAsFilteredEvent(env)
- else
- env
- }
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection.exactlyOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e5")
- }
- offsetShouldBe(envelopes.last.offset)
- }
-
- }
-
- "A R2DBC grouped projection with TimestampOffset" must {
- "persist projection and offset in the same write operation (transactional)" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val handlerProbe = createTestProbe[String]("calls-to-handler")
-
- val projection =
- R2dbcProjection
- .groupedWithin(projectionId, Some(settings), sourceProvider, handler = () => groupedHandler(handlerProbe.ref))
- .withGroup(3, 3.seconds)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
-
- // handler probe is called twice
- handlerProbe.expectMessage("called")
- handlerProbe.expectMessage("called")
-
- offsetShouldBe(envelopes.last.offset)
- }
-
- "filter duplicates" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopesWithDuplicates(pid1, pid2)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val handlerProbe = createTestProbe[String]("calls-to-handler")
-
- val projection =
- R2dbcProjection
- .groupedWithin(projectionId, Some(settings), sourceProvider, handler = () => groupedHandler(handlerProbe.ref))
- .withGroup(3, 3.seconds)
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3")(pid2)
- }
-
- // handler probe is called twice
- handlerProbe.expectMessage("called")
- handlerProbe.expectMessage("called")
-
- offsetShouldBe(envelopes.last.offset)
- }
-
- "filter out unknown sequence numbers" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val startTime = now()
- val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
- val sourceProvider1 = createBacktrackingSourceProvider(envelopes1)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider1)
-
- val handlerProbe = createTestProbe[String]()
-
- val projection1 =
- R2dbcProjection
- .groupedWithin(
- projectionId,
- Some(settings),
- sourceProvider1,
- handler = () => groupedHandler(handlerProbe.ref))
- .withGroup(3, 3.seconds)
-
- projectionTestKit.run(projection1) {
- projectedValueShouldBe("e1-1|e1-2|e1-3")(pid1)
- projectedValueShouldBe("e2-1")(pid2)
- }
- offsetShouldBe(envelopes1.collectFirst { case env if env.event == "e1-3" => env.offset }.get)
-
- // simulate backtracking
- logger.debug("Starting backtracking")
- val envelopes2 = createEnvelopesBacktrackingUnknownSequenceNumbers(startTime, pid1, pid2)
- val sourceProvider2 = createBacktrackingSourceProvider(envelopes2)
- val projection2 =
- R2dbcProjection
- .groupedWithin(
- projectionId,
- Some(settings),
- sourceProvider2,
- handler = () => groupedHandler(handlerProbe.ref))
- .withGroup(3, 3.seconds)
-
- projectionTestKit.run(projection2) {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4|e1-5|e1-6")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3|e2-4")(pid2)
- }
- offsetShouldBe(envelopes2.last.offset)
- }
-
- "be able to skip envelopes but still store offset" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6).map { env =>
- if (env.event == "e3" || env.event == "e4" || env.event == "e6")
- markAsFilteredEvent(env)
- else
- env
- }
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val handlerProbe = createTestProbe[String]("calls-to-handler")
-
- val projection =
- R2dbcProjection
- .groupedWithin(projectionId, Some(settings), sourceProvider, handler = () => groupedHandler(handlerProbe.ref))
- .withGroup(3, 3.seconds)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e5")
- }
-
- // handler probe is called twice
- handlerProbe.expectMessage("called")
- handlerProbe.expectMessage("called")
-
- offsetShouldBe(envelopes.last.offset)
- }
-
- "handle grouped async projection" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val result = new StringBuffer()
-
- def handler(): Handler[immutable.Seq[EventEnvelope[String]]] =
- new Handler[immutable.Seq[EventEnvelope[String]]] {
- override def process(envelopes: immutable.Seq[EventEnvelope[String]]): Future[Done] = {
- Future {
- envelopes.foreach(env => result.append(env.event).append("|"))
- }.map(_ => Done)
- }
- }
-
- val projection =
- R2dbcProjection
- .groupedWithinAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())
- .withGroup(2, 3.seconds)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- result.toString shouldBe "e1|e2|e3|e4|e5|e6|"
- }
-
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter duplicates for grouped async projection" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopesWithDuplicates(pid1, pid2)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val result1 = new StringBuffer()
- val result2 = new StringBuffer()
-
- def handler(): Handler[immutable.Seq[EventEnvelope[String]]] = new Handler[immutable.Seq[EventEnvelope[String]]] {
- override def process(envelopes: immutable.Seq[EventEnvelope[String]]): Future[Done] = {
- Future
- .successful {
- envelopes.foreach { envelope =>
- if (envelope.persistenceId == pid1)
- result1.append(envelope.event).append("|")
- else
- result2.append(envelope.event).append("|")
- }
- }
- .map(_ => Done)
- }
- }
-
- val projection =
- R2dbcProjection.groupedWithinAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())
-
- projectionTestKit.run(projection) {
- result1.toString shouldBe "e1-1|e1-2|e1-3|e1-4|"
- result2.toString shouldBe "e2-1|e2-2|e2-3|"
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter out unknown sequence numbers for grouped async projection" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val startTime = now()
- val sourceProvider = new TestSourceProviderWithInput()
- implicit val offsetStore: R2dbcOffsetStore =
- R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
-
- val result1 = new StringBuffer()
- val result2 = new StringBuffer()
-
- def handler(): Handler[immutable.Seq[EventEnvelope[String]]] = new Handler[immutable.Seq[EventEnvelope[String]]] {
- override def process(envelopes: immutable.Seq[EventEnvelope[String]]): Future[Done] = {
- Future
- .successful {
- envelopes.foreach { envelope =>
- if (envelope.persistenceId == pid1)
- result1.append(envelope.event).append("|")
- else
- result2.append(envelope.event).append("|")
- }
- }
- .map(_ => Done)
- }
- }
-
- val projectionRef = spawn(
- ProjectionBehavior(
- R2dbcProjection.groupedWithinAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())))
- val input = sourceProvider.input.futureValue
-
- val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes1.foreach(input ! _)
-
- eventually {
- result1.toString shouldBe "e1-1|e1-2|e1-3|"
- result2.toString shouldBe "e2-1|"
- }
-
- // simulate backtracking
- logger.debug("Starting backtracking")
- val envelopes2 = createEnvelopesBacktrackingUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes2.foreach(input ! _)
-
- eventually {
- result1.toString shouldBe "e1-1|e1-2|e1-3|e1-4|e1-5|e1-6|"
- result2.toString shouldBe "e2-1|e2-2|e2-3|e2-4|"
- }
-
- eventually {
- offsetShouldBe(envelopes2.last.offset)
- }
- projectionRef ! ProjectionBehavior.Stop
- }
-
- "be able to skip envelopes but still store offset for grouped async projection" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6).map { env =>
- if (env.event == "e3" || env.event == "e4" || env.event == "e6")
- markAsFilteredEvent(env)
- else
- env
- }
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val result = new StringBuffer()
-
- def handler(): Handler[immutable.Seq[EventEnvelope[String]]] =
- new Handler[immutable.Seq[EventEnvelope[String]]] {
- override def process(envelopes: immutable.Seq[EventEnvelope[String]]): Future[Done] = {
- Future {
- envelopes.foreach(env => result.append(env.event).append("|"))
- }.map(_ => Done)
- }
- }
-
- val projection =
- R2dbcProjection
- .groupedWithinAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())
- .withGroup(2, 3.seconds)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- result.toString shouldBe "e1|e2|e5|"
- }
-
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
- }
-
- "A R2DBC at-least-once projection with TimestampOffset" must {
-
- "persist projection and offset" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection.atLeastOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter duplicates" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopesWithDuplicates(pid1, pid2)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection.atLeastOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3")(pid2)
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter out unknown sequence numbers" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val startTime = now()
- val sourceProvider = new TestSourceProviderWithInput()
- implicit val offsetStore: R2dbcOffsetStore =
- R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
-
- val projectionRef = spawn(
- ProjectionBehavior(
- R2dbcProjection.atLeastOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)))
- val input = sourceProvider.input.futureValue
-
- val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes1.foreach(input ! _)
-
- eventually {
- projectedValueShouldBe("e1-1|e1-2|e1-3")(pid1)
- projectedValueShouldBe("e2-1")(pid2)
- }
-
- // simulate backtracking
- logger.debug("Starting backtracking")
- val envelopes2 = createEnvelopesBacktrackingUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes2.foreach(input ! _)
-
- eventually {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4|e1-5|e1-6")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3|e2-4")(pid2)
- }
-
- eventually {
- offsetShouldBe(envelopes2.last.offset)
- }
- projectionRef ! ProjectionBehavior.Stop
- }
-
- "re-delivery inflight events after failure with retry recovery strategy" in {
- implicit val pid1 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid1, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val failOnce = new AtomicBoolean(true)
- val failPredicate = (ev: EventEnvelope[String]) => {
- // fail on first call for event 4, let it pass afterwards
- ev.sequenceNr == 4 && failOnce.compareAndSet(true, false)
- }
- val bogusEventHandler = new ConcatHandler(failPredicate)
-
- val projectionFailing =
- R2dbcProjection
- .atLeastOnce(projectionId, Some(settings), sourceProvider, handler = () => bogusEventHandler)
- .withSaveOffset(afterEnvelopes = 5, afterDuration = 2.seconds)
- .withRecoveryStrategy(HandlerRecoveryStrategy.retryAndSkip(2, 10.millis))
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
-
- bogusEventHandler.attempts shouldBe 1
-
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "be able to skip envelopes but still store offset" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6).map { env =>
- if (env.event == "e3" || env.event == "e4" || env.event == "e6")
- markAsFilteredEvent(env)
- else
- env
- }
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection.atLeastOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e5")
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "handle async projection" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val result = new StringBuffer()
-
- def handler(): Handler[EventEnvelope[String]] = new Handler[EventEnvelope[String]] {
- override def process(envelope: EventEnvelope[String]): Future[Done] = {
- Future
- .successful {
- result.append(envelope.event).append("|")
- }
- .map(_ => Done)
- }
- }
-
- val projection =
- R2dbcProjection.atLeastOnceAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())
-
- projectionTestKit.run(projection) {
- result.toString shouldBe "e1|e2|e3|e4|e5|e6|"
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "re-delivery inflight events after failure with retry recovery strategy for async projection" in {
- implicit val pid1 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid1, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val failOnce = new AtomicBoolean(true)
- val failPredicate = (ev: EventEnvelope[String]) => {
- // fail on first call for event 4, let it pass afterwards
- ev.sequenceNr == 4 && failOnce.compareAndSet(true, false)
- }
-
- val result = new StringBuffer()
- def handler(): Handler[EventEnvelope[String]] = new Handler[EventEnvelope[String]] {
- override def process(envelope: EventEnvelope[String]): Future[Done] = {
- if (failPredicate(envelope)) {
- throw TestException(s"failed to process event '${envelope.sequenceNr}'")
- } else {
- Future
- .successful {
- result.append(envelope.event).append("|")
- }
- .map(_ => Done)
- }
- }
- }
-
- val projectionFailing =
- R2dbcProjection
- .atLeastOnceAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())
- .withSaveOffset(afterEnvelopes = 5, afterDuration = 2.seconds)
- .withRecoveryStrategy(HandlerRecoveryStrategy.retryAndSkip(2, 10.millis))
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projectionFailing) {
- result.toString shouldBe "e1|e2|e3|e4|e5|e6|"
- }
-
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter duplicates for async projection" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopesWithDuplicates(pid1, pid2)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val result1 = new StringBuffer()
- val result2 = new StringBuffer()
-
- def handler(): Handler[EventEnvelope[String]] = new Handler[EventEnvelope[String]] {
- override def process(envelope: EventEnvelope[String]): Future[Done] = {
- Future
- .successful {
- if (envelope.persistenceId == pid1)
- result1.append(envelope.event).append("|")
- else
- result2.append(envelope.event).append("|")
- }
- .map(_ => Done)
- }
- }
-
- val projection =
- R2dbcProjection.atLeastOnceAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())
-
- projectionTestKit.run(projection) {
- result1.toString shouldBe "e1-1|e1-2|e1-3|e1-4|"
- result2.toString shouldBe "e2-1|e2-2|e2-3|"
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter out unknown sequence numbers for async projection" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val startTime = now()
- val sourceProvider = new TestSourceProviderWithInput()
- implicit val offsetStore: R2dbcOffsetStore =
- R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
-
- val result1 = new StringBuffer()
- val result2 = new StringBuffer()
-
- def handler(): Handler[EventEnvelope[String]] = new Handler[EventEnvelope[String]] {
- override def process(envelope: EventEnvelope[String]): Future[Done] = {
- Future
- .successful {
- if (envelope.persistenceId == pid1)
- result1.append(envelope.event).append("|")
- else
- result2.append(envelope.event).append("|")
- }
- .map(_ => Done)
- }
- }
-
- val projectionRef = spawn(
- ProjectionBehavior(
- R2dbcProjection.atLeastOnceAsync(projectionId, Some(settings), sourceProvider, handler = () => handler())))
- val input = sourceProvider.input.futureValue
-
- val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes1.foreach(input ! _)
-
- eventually {
- result1.toString shouldBe "e1-1|e1-2|e1-3|"
- result2.toString shouldBe "e2-1|"
- }
-
- // simulate backtracking
- logger.debug("Starting backtracking")
- val envelopes2 = createEnvelopesBacktrackingUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes2.foreach(input ! _)
-
- eventually {
- result1.toString shouldBe "e1-1|e1-2|e1-3|e1-4|e1-5|e1-6|"
- result2.toString shouldBe "e2-1|e2-2|e2-3|e2-4|"
- }
-
- eventually {
- offsetShouldBe(envelopes2.last.offset)
- }
- projectionRef ! ProjectionBehavior.Stop
- }
-
- "be able to skip envelopes but still store offset for async projection" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val envelopes = createEnvelopes(pid, 6).map { env =>
- if (env.event == "e3" || env.event == "e4" || env.event == "e6")
- markAsFilteredEvent(env)
- else
- env
- }
-
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection.atLeastOnce(projectionId, Some(settings), sourceProvider, handler = () => new ConcatHandler)
-
- offsetShouldBeEmpty()
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e5")
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
- }
-
- "A R2DBC flow projection with TimestampOffset" must {
-
- "persist projection and offset" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- offsetShouldBeEmpty()
-
- val flowHandler =
- FlowWithContext[EventEnvelope[String], ProjectionContext]
- .mapAsync(1) { env =>
- withRepo(_.concatToText(env.persistenceId, env.event))
- }
-
- val projection =
- R2dbcProjection
- .atLeastOnceFlow(projectionId, Some(settings), sourceProvider, handler = flowHandler)
- .withSaveOffset(1, 1.minute)
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter duplicates" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopesWithDuplicates(pid1, pid2)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- info(s"pid1 [$pid1], pid2 [$pid2]")
-
- val flowHandler =
- FlowWithContext[EventEnvelope[String], ProjectionContext]
- .mapAsync(1) { env =>
- withRepo(_.concatToText(env.persistenceId, env.event))
- }
-
- val projection =
- R2dbcProjection
- .atLeastOnceFlow(projectionId, Some(settings), sourceProvider, handler = flowHandler)
- .withSaveOffset(2, 1.minute)
-
- projectionTestKit.run(projection) {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3")(pid2)
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
-
- "filter out unknown sequence numbers" in {
- val pid1 = UUID.randomUUID().toString
- val pid2 = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
-
- val startTime = now()
- val sourceProvider = new TestSourceProviderWithInput()
- implicit val offsetStore: R2dbcOffsetStore =
- R2dbcOffsetStore.fromConfig(projectionId, Some(sourceProvider), system, settings, r2dbcExecutor)
-
- val flowHandler =
- FlowWithContext[EventEnvelope[String], ProjectionContext]
- .mapAsync(1) { env =>
- withRepo(_.concatToText(env.persistenceId, env.event))
- }
-
- val projectionRef = spawn(
- ProjectionBehavior(
- R2dbcProjection
- .atLeastOnceFlow(projectionId, Some(settings), sourceProvider, handler = flowHandler)
- .withSaveOffset(2, 1.minute)))
- val input = sourceProvider.input.futureValue
-
- val envelopes1 = createEnvelopesUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes1.foreach(input ! _)
-
- eventually {
- projectedValueShouldBe("e1-1|e1-2|e1-3")(pid1)
- projectedValueShouldBe("e2-1")(pid2)
- }
-
- // simulate backtracking
- logger.debug("Starting backtracking")
- val envelopes2 = createEnvelopesBacktrackingUnknownSequenceNumbers(startTime, pid1, pid2)
- envelopes2.foreach(input ! _)
-
- eventually {
- projectedValueShouldBe("e1-1|e1-2|e1-3|e1-4|e1-5|e1-6")(pid1)
- projectedValueShouldBe("e2-1|e2-2|e2-3|e2-4")(pid2)
- }
-
- eventually {
- offsetShouldBe(envelopes2.last.offset)
- }
- projectionRef ! ProjectionBehavior.Stop
- }
-
- "not support skipping envelopes but still store offset" in {
- // This is a limitation for atLeastOnceFlow and this test is just
- // capturing current behavior of not supporting the feature of skipping
- // envelopes that are marked with NotUsed in the eventMetadata.
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6).map { env =>
- if (env.event == "e3" || env.event == "e4" || env.event == "e6")
- markAsFilteredEvent(env)
- else
- env
- }
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- offsetShouldBeEmpty()
-
- val flowHandler =
- FlowWithContext[EventEnvelope[String], ProjectionContext]
- .mapAsync(1) { env =>
- withRepo(_.concatToText(env.persistenceId, env.event))
- }
-
- val projection =
- R2dbcProjection
- .atLeastOnceFlow(projectionId, Some(settings), sourceProvider, handler = flowHandler)
- .withSaveOffset(1, 1.minute)
-
- projectionTestKit.run(projection) {
- // e3, e4, e6 are still included
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
- }
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- }
- }
-
- "R2dbcProjection management with TimestampOffset" must {
-
- "restart from beginning when offset is cleared" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider,
- handler = () =>
- R2dbcHandler[EventEnvelope[String]] { (session, envelope) =>
- TestRepository(session, settings).concatToText(envelope.persistenceId, envelope.event)
- })
-
- offsetShouldBeEmpty()
-
- // not using ProjectionTestKit because want to test ProjectionManagement
- spawn(ProjectionBehavior(projection))
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- ProjectionManagement(system).getOffset(projectionId).futureValue shouldBe Some(envelopes.last.offset)
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
-
- ProjectionManagement(system).clearOffset(projectionId).futureValue shouldBe Done
- eventually {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6|e1|e2|e3|e4|e5|e6")
- }
- }
-
- "restart from updated offset" in {
- implicit val pid = UUID.randomUUID().toString
- val projectionId = genRandomProjectionId()
- val envelopes = createEnvelopes(pid, 6)
- val sourceProvider = createSourceProvider(envelopes)
- implicit val offsetStore: R2dbcOffsetStore = createOffsetStore(projectionId, sourceProvider)
-
- val projection =
- R2dbcProjection
- .exactlyOnce(
- projectionId,
- Some(settings),
- sourceProvider,
- handler = () =>
- R2dbcHandler[EventEnvelope[String]] { (session, envelope) =>
- TestRepository(session, settings).concatToText(envelope.persistenceId, envelope.event)
- })
-
- offsetShouldBeEmpty()
-
- // not using ProjectionTestKit because want to test ProjectionManagement
- spawn(ProjectionBehavior(projection))
-
- eventually {
- offsetShouldBe(envelopes.last.offset)
- }
- ProjectionManagement(system).getOffset(projectionId).futureValue shouldBe Some(envelopes.last.offset)
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6")
-
- ProjectionManagement(system).updateOffset(projectionId, envelopes(2).offset).futureValue shouldBe Done
- eventually {
- projectedValueShouldBe("e1|e2|e3|e4|e5|e6|e4|e5|e6")
- }
- }
-
- }
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetStoreSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetStoreSpec.scala
deleted file mode 100644
index d56e6dc..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/R2dbcTimestampOffsetStoreSpec.scala
+++ /dev/null
@@ -1,863 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * license agreements; and to You under the Apache License, version 2.0:
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * This file is part of the Apache Pekko project, which was derived from Akka.
- */
-
-/*
- * Copyright (C) 2021 Lightbend Inc. <https://www.lightbend.com>
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.time.Instant
-import java.time.{ Duration => JDuration }
-import java.util.UUID
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-
-import org.apache.pekko
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.typed.ActorSystem
-import pekko.persistence.query.TimestampOffset
-import pekko.persistence.query.UpdatedDurableState
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.query.typed.scaladsl.EventTimestampQuery
-import pekko.persistence.query.typed.scaladsl.LoadEventQuery
-import pekko.persistence.typed.PersistenceId
-import pekko.persistence.r2dbc.internal.EnvelopeOrigin
-import pekko.projection.BySlicesSourceProvider
-import pekko.projection.ProjectionId
-import pekko.projection.internal.ManagementState
-import pekko.projection.r2dbc.internal.OffsetPidSeqNr
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore.Pid
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore.Record
-import pekko.projection.r2dbc.internal.R2dbcOffsetStore.SeqNr
-import com.typesafe.config.ConfigFactory
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-object R2dbcTimestampOffsetStoreSpec {
- class TestTimestampSourceProvider(override val minSlice: Int, override val maxSlice: Int, clock: TestClock)
- extends BySlicesSourceProvider
- with EventTimestampQuery
- with LoadEventQuery {
-
- override def timestampOf(persistenceId: String, sequenceNr: SeqNr): Future[Option[Instant]] =
- Future.successful(Some(clock.instant()))
-
- override def loadEnvelope[Event](persistenceId: String, sequenceNr: SeqNr): Future[EventEnvelope[Event]] =
- throw new IllegalStateException("loadEvent shouldn't be used here")
- }
-}
-
-class R2dbcTimestampOffsetStoreSpec
- extends ScalaTestWithActorTestKit(
- ConfigFactory
- .parseString("""
- # to be able to test eviction
- pekko.projection.r2dbc.offset-store.keep-number-of-entries = 0
- """).withFallback(TestConfig.config))
- with AnyWordSpecLike
- with TestDbLifecycle
- with TestData
- with LogCapturing {
- import R2dbcTimestampOffsetStoreSpec.TestTimestampSourceProvider
-
- override def typedSystem: ActorSystem[_] = system
-
- private val clock = TestClock.nowMicros()
- def tick(): Unit = clock.tick(JDuration.ofMillis(1))
-
- private val log = LoggerFactory.getLogger(getClass)
-
- private val settings = R2dbcProjectionSettings(testKit.system)
-
- private def createOffsetStore(
- projectionId: ProjectionId,
- customSettings: R2dbcProjectionSettings = settings,
- eventTimestampQueryClock: TestClock = clock) =
- R2dbcOffsetStore.fromConfig(
- projectionId,
- Some(new TestTimestampSourceProvider(0, persistenceExt.numberOfSlices - 1, eventTimestampQueryClock)),
- system,
- customSettings,
- r2dbcExecutor)
-
- def createEnvelope(pid: Pid, seqNr: SeqNr, timestamp: Instant, event: String): EventEnvelope[String] = {
- val entityType = PersistenceId.extractEntityType(pid)
- val slice = persistenceExt.sliceForPersistenceId(pid)
- EventEnvelope(
- TimestampOffset(timestamp, timestamp.plusMillis(1000), Map(pid -> seqNr)),
- pid,
- seqNr,
- event,
- timestamp.toEpochMilli,
- entityType,
- slice)
- }
-
- def backtrackingEnvelope(env: EventEnvelope[String]): EventEnvelope[String] =
- new EventEnvelope[String](
- env.offset,
- env.persistenceId,
- env.sequenceNr,
- eventOption = None,
- env.timestamp,
- env.eventMetadata,
- env.entityType,
- env.slice,
- env.filtered,
- source = EnvelopeOrigin.SourceBacktracking)
-
- def filteredEnvelope(env: EventEnvelope[String]): EventEnvelope[String] =
- new EventEnvelope[String](
- env.offset,
- env.persistenceId,
- env.sequenceNr,
- env.eventOption,
- env.timestamp,
- env.eventMetadata,
- env.entityType,
- env.slice,
- filtered = true,
- env.source)
-
- def createUpdatedDurableState(
- pid: Pid,
- revision: SeqNr,
- timestamp: Instant,
- state: String): UpdatedDurableState[String] =
- new UpdatedDurableState(
- pid,
- revision,
- state,
- TimestampOffset(timestamp, timestamp.plusMillis(1000), Map(pid -> revision)),
- timestamp.toEpochMilli)
-
- private implicit val ec: ExecutionContext = system.executionContext
-
- "The R2dbcOffsetStore for TimestampOffset" must {
-
- "save TimestampOffset with one entry" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 3L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- val readOffset1 = offsetStore.readOffset[TimestampOffset]()
- readOffset1.futureValue shouldBe Some(offset1)
-
- tick()
- val offset2 = TimestampOffset(clock.instant(), Map("p1" -> 4L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p1", 4L)).futureValue
- val readOffset2 = offsetStore.readOffset[TimestampOffset]()
- readOffset2.futureValue shouldBe Some(offset2) // yep, saveOffset overwrites previous
- }
-
- "save TimestampOffset with several seen entries" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 3L, "p2" -> 1L, "p3" -> 5L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- val readOffset1 = offsetStore.readOffset[TimestampOffset]()
- val expectedOffset1 = offset1.copy(seen = Map("p1" -> 3L))
- readOffset1.futureValue shouldBe Some(expectedOffset1)
-
- tick()
- val offset2 = TimestampOffset(clock.instant(), Map("p1" -> 4L, "p3" -> 6L, "p4" -> 9L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p3", 6L)).futureValue
- val readOffset2 = offsetStore.readOffset[TimestampOffset]()
- val expectedOffset2 = offset2.copy(seen = Map("p3" -> 6L))
- readOffset2.futureValue shouldBe Some(expectedOffset2)
- }
-
- "save TimestampOffset when same timestamp" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 3L, "p2" -> 1L, "p3" -> 5L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p2", 1L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p3", 5L)).futureValue
- val readOffset1 = offsetStore.readOffset[TimestampOffset]()
- readOffset1.futureValue shouldBe Some(offset1)
-
- // not tick, same timestamp
- val offset2 = TimestampOffset(clock.instant(), Map("p2" -> 2L, "p4" -> 9L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p2", 2L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p4", 9L)).futureValue
- val readOffset2 = offsetStore.readOffset[TimestampOffset]()
- // all should be included since same timestamp
- val expectedOffset2 = TimestampOffset(clock.instant(), Map("p1" -> 3L, "p2" -> 2L, "p3" -> 5L, "p4" -> 9L))
- readOffset2.futureValue shouldBe Some(expectedOffset2)
-
- // saving new with later timestamp
- tick()
- val offset3 = TimestampOffset(clock.instant(), Map("p1" -> 4L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset3, "p1", 4L)).futureValue
- val readOffset3 = offsetStore.readOffset[TimestampOffset]()
- // then it should only contain that entry
- readOffset3.futureValue shouldBe Some(offset3)
- }
-
- "save batch of TimestampOffsets" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 3L, "p2" -> 1L, "p3" -> 5L))
- tick()
- val offset2 = TimestampOffset(clock.instant(), Map("p5" -> 1L))
- tick()
- val offset3 = TimestampOffset(clock.instant(), Map("p6" -> 6L))
- tick()
- val offset4 = TimestampOffset(clock.instant(), Map("p1" -> 4L, "p3" -> 6L, "p4" -> 9L))
- val offsetsBatch1 = Vector(
- OffsetPidSeqNr(offset1, "p1", 3L),
- OffsetPidSeqNr(offset1, "p2", 1L),
- OffsetPidSeqNr(offset1, "p3", 5L),
- OffsetPidSeqNr(offset2, "p5", 1L),
- OffsetPidSeqNr(offset3, "p6", 6L),
- OffsetPidSeqNr(offset4, "p1", 4L),
- OffsetPidSeqNr(offset4, "p3", 6L),
- OffsetPidSeqNr(offset4, "p4", 9L))
-
- offsetStore.saveOffsets(offsetsBatch1).futureValue
- val readOffset1 = offsetStore.readOffset[TimestampOffset]()
- readOffset1.futureValue shouldBe Some(offsetsBatch1.last.offset)
- offsetStore.getState().byPid("p1").seqNr shouldBe 4L
- offsetStore.getState().byPid("p2").seqNr shouldBe 1L
- offsetStore.getState().byPid("p3").seqNr shouldBe 6L
- offsetStore.getState().byPid("p4").seqNr shouldBe 9L
- offsetStore.getState().byPid("p5").seqNr shouldBe 1L
- offsetStore.getState().byPid("p6").seqNr shouldBe 6L
-
- tick()
- val offset5 = TimestampOffset(clock.instant(), Map("p1" -> 5L))
- offsetStore.saveOffsets(Vector(OffsetPidSeqNr(offset5, "p1", 5L))).futureValue
-
- tick()
- // duplicate
- val offset6 = TimestampOffset(clock.instant(), Map("p2" -> 1L))
- offsetStore.saveOffsets(Vector(OffsetPidSeqNr(offset6, "p2", 1L))).futureValue
-
- tick()
- val offset7 = TimestampOffset(clock.instant(), Map("p1" -> 6L))
- tick()
- val offset8 = TimestampOffset(clock.instant(), Map("p1" -> 7L))
- tick()
- val offset9 = TimestampOffset(clock.instant(), Map("p1" -> 8L))
- val offsetsBatch2 =
- Vector(OffsetPidSeqNr(offset7, "p1", 6L), OffsetPidSeqNr(offset8, "p1", 7L), OffsetPidSeqNr(offset9, "p1", 8L))
-
- offsetStore.saveOffsets(offsetsBatch2).futureValue
- val readOffset2 = offsetStore.readOffset[TimestampOffset]()
- readOffset2.futureValue shouldBe Some(offsetsBatch2.last.offset)
- offsetStore.getState().byPid("p1").seqNr shouldBe 8L
- offsetStore.getState().byPid("p2").seqNr shouldBe 1L
- offsetStore.getState().byPid("p3").seqNr shouldBe 6L
- offsetStore.getState().byPid("p4").seqNr shouldBe 9L
- offsetStore.getState().byPid("p5").seqNr shouldBe 1L
- offsetStore.getState().byPid("p6").seqNr shouldBe 6L
- }
-
- "not update when earlier seqNr" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 3L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- val readOffset1 = offsetStore.readOffset[TimestampOffset]()
- readOffset1.futureValue shouldBe Some(offset1)
-
- clock.setInstant(clock.instant().minusMillis(1))
- val offset2 = TimestampOffset(clock.instant(), Map("p1" -> 2L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p1", 2L)).futureValue
- val readOffset2 = offsetStore.readOffset[TimestampOffset]()
- readOffset2.futureValue shouldBe Some(offset1) // keeping offset1
- }
-
- "readOffset from given slices" in {
- val projectionId0 = ProjectionId(UUID.randomUUID().toString, "0-1023")
- val projectionId1 = ProjectionId(projectionId0.name, "0-511")
- val projectionId2 = ProjectionId(projectionId0.name, "512-1023")
-
- val p1 = "p1"
- val slice1 = persistenceExt.sliceForPersistenceId(p1)
- slice1 shouldBe 449
-
- val p2 = "p2"
- val slice2 = persistenceExt.sliceForPersistenceId(p2)
- slice2 shouldBe 450
-
- val p3 = "p10"
- val slice3 = persistenceExt.sliceForPersistenceId(p3)
- slice3 shouldBe 655
-
- val p4 = "p11"
- val slice4 = persistenceExt.sliceForPersistenceId(p4)
- slice4 shouldBe 656
-
- val offsetStore0 =
- R2dbcOffsetStore.fromConfig(
- projectionId0,
- Some(new TestTimestampSourceProvider(0, persistenceExt.numberOfSlices - 1, clock)),
- system,
- settings,
- r2dbcExecutor)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map(p1 -> 3L))
- offsetStore0.saveOffset(OffsetPidSeqNr(offset1, p1, 3L)).futureValue
- tick()
- val offset2 = TimestampOffset(clock.instant(), Map(p2 -> 4L))
- offsetStore0.saveOffset(OffsetPidSeqNr(offset2, p2, 4L)).futureValue
- tick()
- val offset3 = TimestampOffset(clock.instant(), Map(p3 -> 7L))
- offsetStore0.saveOffset(OffsetPidSeqNr(offset3, p3, 7L)).futureValue
- tick()
- val offset4 = TimestampOffset(clock.instant(), Map(p4 -> 5L))
- offsetStore0.saveOffset(OffsetPidSeqNr(offset4, p4, 5L)).futureValue
-
- val offsetStore1 =
- R2dbcOffsetStore.fromConfig(
- projectionId1,
- Some(new TestTimestampSourceProvider(0, 511, clock)),
- system,
- settings,
- r2dbcExecutor)
- offsetStore1.readOffset().futureValue
- offsetStore1.getState().byPid.keySet shouldBe Set(p1, p2)
-
- val offsetStore2 =
- R2dbcOffsetStore.fromConfig(
- projectionId2,
- Some(new TestTimestampSourceProvider(512, 1023, clock)),
- system,
- settings,
- r2dbcExecutor)
- offsetStore2.readOffset().futureValue
- offsetStore2.getState().byPid.keySet shouldBe Set(p3, p4)
- }
-
- "filter duplicates" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 3L, "p2" -> 1L, "p3" -> 5L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p2", 1L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p3", 5L)).futureValue
- tick()
- val offset2 = TimestampOffset(clock.instant(), Map("p1" -> 4L, "p3" -> 6L, "p4" -> 9L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p1", 4L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p3", 6L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset2, "p4", 9L)).futureValue
- tick()
- val offset3 = TimestampOffset(clock.instant(), Map("p5" -> 10L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset3, "p5", 10L)).futureValue
-
- offsetStore.isDuplicate(Record("p5", 10, offset3.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p1", 4, offset2.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p3", 6, offset2.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p4", 9, offset2.timestamp)) shouldBe true
-
- offsetStore.isDuplicate(Record("p1", 3, offset1.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p2", 1, offset1.timestamp)) shouldBe true
- offsetStore.isDuplicate(Record("p3", 5, offset1.timestamp)) shouldBe true
-
- offsetStore.isDuplicate(Record("p1", 2, offset1.timestamp.minusMillis(1))) shouldBe true
- offsetStore.isDuplicate(Record("p5", 9, offset3.timestamp.minusMillis(1))) shouldBe true
-
- offsetStore.isDuplicate(Record("p5", 11, offset3.timestamp)) shouldBe false
- offsetStore.isDuplicate(Record("p5", 12, offset3.timestamp.plusMillis(1))) shouldBe false
-
- offsetStore.isDuplicate(Record("p6", 1, offset3.timestamp.plusMillis(2))) shouldBe false
- offsetStore.isDuplicate(Record("p7", 1, offset3.timestamp.minusMillis(1))) shouldBe false
- }
-
- "accept known sequence numbers and reject unknown" in {
- val projectionId = genRandomProjectionId()
- val eventTimestampQueryClock = TestClock.nowMicros()
- val offsetStore = createOffsetStore(projectionId, eventTimestampQueryClock = eventTimestampQueryClock)
-
- val startTime = TestClock.nowMicros().instant()
- val offset1 = TimestampOffset(startTime, Map("p1" -> 3L, "p2" -> 1L, "p3" -> 5L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p2", 1L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p3", 5L)).futureValue
-
- // seqNr 1 is always accepted
- val env1 = createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")
- offsetStore.isAccepted(env1).futureValue shouldBe true
- offsetStore.isAccepted(backtrackingEnvelope(env1)).futureValue shouldBe true
- // but not if already inflight, seqNr 1 was accepted
- offsetStore.addInflight(env1)
- val env1Later = createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")
- offsetStore.isAccepted(env1Later).futureValue shouldBe false
- offsetStore.isAccepted(backtrackingEnvelope(env1Later)).futureValue shouldBe false
- // subsequent seqNr is accepted
- val env2 = createEnvelope("p4", 2L, startTime.plusMillis(2), "e4-2")
- offsetStore.isAccepted(env2).futureValue shouldBe true
- offsetStore.isAccepted(backtrackingEnvelope(env2)).futureValue shouldBe true
- offsetStore.addInflight(env2)
- // but not when gap
- val envP4SeqNr4 = createEnvelope("p4", 4L, startTime.plusMillis(3), "e4-4")
- offsetStore.isAccepted(envP4SeqNr4).futureValue shouldBe false
- // hard reject when gap from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(envP4SeqNr4))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unexpected sequence number.*")
- // reject filtered event when gap
- offsetStore.isAccepted(filteredEnvelope(envP4SeqNr4)).futureValue shouldBe false
- // hard reject when filtered event with gap from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(filteredEnvelope(envP4SeqNr4)))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unexpected sequence number.*")
- // and not if later already inflight, seqNr 2 was accepted
- offsetStore.isAccepted(createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")).futureValue shouldBe false
-
- // +1 to known is accepted
- val env3 = createEnvelope("p1", 4L, startTime.plusMillis(4), "e1-4")
- offsetStore.isAccepted(env3).futureValue shouldBe true
- // but not same
- offsetStore.isAccepted(createEnvelope("p3", 5L, startTime, "e3-5")).futureValue shouldBe false
- // but not same, even if it's 1
- offsetStore.isAccepted(createEnvelope("p2", 1L, startTime, "e2-1")).futureValue shouldBe false
- // and not less
- offsetStore.isAccepted(createEnvelope("p3", 4L, startTime, "e3-4")).futureValue shouldBe false
- offsetStore.addInflight(env3)
- // and then it's not accepted again
- offsetStore.isAccepted(env3).futureValue shouldBe false
- offsetStore.isAccepted(backtrackingEnvelope(env3)).futureValue shouldBe false
- // and not when later seqNr is inflight
- offsetStore.isAccepted(env2).futureValue shouldBe false
- offsetStore.isAccepted(backtrackingEnvelope(env2)).futureValue shouldBe false
-
- // +1 to known, and then also subsequent are accepted (needed for grouped)
- val env4 = createEnvelope("p3", 6L, startTime.plusMillis(5), "e3-6")
- offsetStore.isAccepted(env4).futureValue shouldBe true
- offsetStore.addInflight(env4)
- val env5 = createEnvelope("p3", 7L, startTime.plusMillis(6), "e3-7")
- offsetStore.isAccepted(env5).futureValue shouldBe true
- offsetStore.addInflight(env5)
- val env6 = createEnvelope("p3", 8L, startTime.plusMillis(7), "e3-8")
- offsetStore.isAccepted(env6).futureValue shouldBe true
- offsetStore.addInflight(env6)
-
- // reject unknown
- val env7 = createEnvelope("p5", 7L, startTime.plusMillis(8), "e5-7")
- offsetStore.isAccepted(env7).futureValue shouldBe false
- (offsetStore.isAccepted(backtrackingEnvelope(env7)).failed.futureValue.getMessage should fullyMatch)
- .regex("Rejected envelope from backtracking.*unknown sequence number.*")
- // but ok when previous is old
- eventTimestampQueryClock.setInstant(startTime.minusSeconds(3600))
- val env8 = createEnvelope("p5", 7L, startTime.plusMillis(5), "e5-7")
- offsetStore.isAccepted(env8).futureValue shouldBe true
- eventTimestampQueryClock.setInstant(startTime)
- offsetStore.addInflight(env8)
- // and subsequent seqNr is accepted
- val env9 = createEnvelope("p5", 8L, startTime.plusMillis(9), "e5-8")
- offsetStore.isAccepted(env9).futureValue shouldBe true
- offsetStore.addInflight(env9)
-
- // reject unknown filtered
- val env10 = filteredEnvelope(createEnvelope("p6", 7L, startTime.plusMillis(10), "e6-7"))
- offsetStore.isAccepted(env10).futureValue shouldBe false
- // hard reject when unknown from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(env10))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unknown sequence number.*")
- // hard reject when unknown filtered event from backtracking
- (offsetStore
- .isAccepted(backtrackingEnvelope(filteredEnvelope(env10)))
- .failed
- .futureValue
- .getMessage should fullyMatch).regex("Rejected envelope from backtracking.*unknown sequence number.*")
-
- // it's keeping the inflight that are not in the "stored" state
- offsetStore.getInflight() shouldBe Map("p1" -> 4L, "p3" -> 8, "p4" -> 2L, "p5" -> 8)
- // and they are removed from inflight once they have been stored
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plusMillis(2), Map("p4" -> 2L)), "p4", 2L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plusMillis(9), Map("p5" -> 8L)), "p5", 8L))
- .futureValue
- offsetStore.getInflight() shouldBe Map("p1" -> 4L, "p3" -> 8)
- }
-
- "update inflight on error and re-accept element" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- val startTime = TestClock.nowMicros().instant()
-
- val envelope1 = createEnvelope("p1", 1L, startTime.plusMillis(1), "e1-1")
- val envelope2 = createEnvelope("p1", 2L, startTime.plusMillis(2), "e1-2")
- val envelope3 = createEnvelope("p1", 3L, startTime.plusMillis(2), "e1-2")
-
- // seqNr 1 is always accepted
- offsetStore.isAccepted(envelope1).futureValue shouldBe true
- offsetStore.addInflight(envelope1)
- offsetStore.getInflight() shouldBe Map("p1" -> 1L)
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plusMillis(1), Map("p1" -> 1L)), "p1", 1L))
- .futureValue
- offsetStore.getInflight() shouldBe empty
-
- // seqNr 2 is accepts since it follows seqNr 1 that is stored in state
- offsetStore.isAccepted(envelope2).futureValue shouldBe true
- // simulate envelope processing error by not adding envelope2 to inflight
-
- // seqNr 3 is not accepted, still waiting for seqNr 2
- offsetStore.isAccepted(envelope3).futureValue shouldBe false
-
- // offer seqNr 2 once again
- offsetStore.isAccepted(envelope2).futureValue shouldBe true
- offsetStore.addInflight(envelope2)
- offsetStore.getInflight() shouldBe Map("p1" -> 2L)
-
- // offer seqNr 3 once more
- offsetStore.isAccepted(envelope3).futureValue shouldBe true
- offsetStore.addInflight(envelope3)
- offsetStore.getInflight() shouldBe Map("p1" -> 3L)
-
- // and they are removed from inflight once they have been stored
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plusMillis(2), Map("p1" -> 3L)), "p1", 3L))
- .futureValue
- offsetStore.getInflight() shouldBe empty
- }
-
- "filter accepted" in {
- val projectionId = genRandomProjectionId()
- val startTime = TestClock.nowMicros().instant()
- val offsetStore = createOffsetStore(projectionId)
-
- val offset1 = TimestampOffset(startTime, Map("p1" -> 3L, "p2" -> 1L, "p3" -> 5L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p2", 1L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p3", 5L)).futureValue
-
- // seqNr 1 is always accepted
- val env1 = createEnvelope("p4", 1L, startTime.plusMillis(1), "e4-1")
- // subsequent seqNr is accepted
- val env2 = createEnvelope("p4", 2L, startTime.plusMillis(2), "e4-2")
- // but not when gap
- val env3 = createEnvelope("p4", 4L, startTime.plusMillis(3), "e4-4")
- // ok when previous is known
- val env4 = createEnvelope("p1", 4L, startTime.plusMillis(5), "e1-4")
- // but not when previous is unknown
- val env5 = createEnvelope("p3", 7L, startTime.plusMillis(5), "e3-7")
-
- offsetStore.filterAccepted(List(env1, env2, env3, env4, env5)).futureValue shouldBe List(env1, env2, env4)
-
- }
-
- "accept new revisions for durable state" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- val startTime = TestClock.nowMicros().instant()
- val offset1 = TimestampOffset(startTime, Map("p1" -> 3L, "p2" -> 1L, "p3" -> 5L))
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p1", 3L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p2", 1L)).futureValue
- offsetStore.saveOffset(OffsetPidSeqNr(offset1, "p3", 5L)).futureValue
-
- // seqNr 1 is always accepted
- val env1 = createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1")
- offsetStore.isAccepted(env1).futureValue shouldBe true
- // but not if already inflight, seqNr 1 was accepted
- offsetStore.addInflight(env1)
- offsetStore
- .isAccepted(createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1"))
- .futureValue shouldBe false
- // subsequent seqNr is accepted
- val env2 = createUpdatedDurableState("p4", 2L, startTime.plusMillis(2), "s4-2")
- offsetStore.isAccepted(env2).futureValue shouldBe true
- offsetStore.addInflight(env2)
- // and also ok with gap
- offsetStore
- .isAccepted(createUpdatedDurableState("p4", 4L, startTime.plusMillis(3), "s4-4"))
- .futureValue shouldBe true
- // and not if later already inflight, seqNr 2 was accepted
- offsetStore
- .isAccepted(createUpdatedDurableState("p4", 1L, startTime.plusMillis(1), "s4-1"))
- .futureValue shouldBe false
-
- // greater than known is accepted
- val env3 = createUpdatedDurableState("p1", 4L, startTime.plusMillis(4), "s1-4")
- offsetStore.isAccepted(env3).futureValue shouldBe true
- // but not same
- offsetStore.isAccepted(createUpdatedDurableState("p3", 5L, startTime, "s3-5")).futureValue shouldBe false
- // but not same, even if it's 1
- offsetStore.isAccepted(createUpdatedDurableState("p2", 1L, startTime, "s2-1")).futureValue shouldBe false
- // and not less
- offsetStore.isAccepted(createUpdatedDurableState("p3", 4L, startTime, "s3-4")).futureValue shouldBe false
- offsetStore.addInflight(env3)
-
- // greater than known, and then also subsequent are accepted (needed for grouped)
- val env4 = createUpdatedDurableState("p3", 8L, startTime.plusMillis(5), "s3-6")
- offsetStore.isAccepted(env4).futureValue shouldBe true
- offsetStore.addInflight(env4)
- val env5 = createUpdatedDurableState("p3", 9L, startTime.plusMillis(6), "s3-7")
- offsetStore.isAccepted(env5).futureValue shouldBe true
- offsetStore.addInflight(env5)
- val env6 = createUpdatedDurableState("p3", 20L, startTime.plusMillis(7), "s3-8")
- offsetStore.isAccepted(env6).futureValue shouldBe true
- offsetStore.addInflight(env6)
-
- // accept unknown
- val env7 = createUpdatedDurableState("p5", 7L, startTime.plusMillis(8), "s5-7")
- offsetStore.isAccepted(env7).futureValue shouldBe true
- offsetStore.addInflight(env7)
-
- // it's keeping the inflight that are not in the "stored" state
- offsetStore.getInflight() shouldBe Map("p1" -> 4L, "p3" -> 20, "p4" -> 2L, "p5" -> 7)
- // and they are removed from inflight once they have been stored
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plusMillis(2), Map("p4" -> 2L)), "p4", 2L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plusMillis(9), Map("p5" -> 8L)), "p5", 8L))
- .futureValue
- offsetStore.getInflight() shouldBe Map("p1" -> 4L, "p3" -> 20)
- }
-
- "evict old records" in {
- val projectionId = genRandomProjectionId()
- val evictSettings = settings.withTimeWindow(JDuration.ofSeconds(100)).withEvictInterval(JDuration.ofSeconds(10))
- import evictSettings._
- val offsetStore = createOffsetStore(projectionId, evictSettings)
-
- val startTime = TestClock.nowMicros().instant()
- log.debug("Start time [{}]", startTime)
-
- offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map("p1" -> 1L)), "p1", 1L)).futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map("p2" -> 1L)), "p2", 1L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map("p3" -> 1L)), "p3", 1L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(evictInterval), Map("p4" -> 1L)), "p4", 1L))
- .futureValue
- offsetStore
- .saveOffset(
- OffsetPidSeqNr(
- TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(1)), Map("p4" -> 1L)),
- "p4",
- 1L))
- .futureValue
- offsetStore
- .saveOffset(
- OffsetPidSeqNr(
- TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(2)), Map("p5" -> 1L)),
- "p5",
- 1L))
- .futureValue
- offsetStore
- .saveOffset(
- OffsetPidSeqNr(
- TimestampOffset(startTime.plus(evictInterval).plus(JDuration.ofSeconds(3)), Map("p6" -> 1L)),
- "p6",
- 1L))
- .futureValue
- offsetStore.getState().size shouldBe 6
-
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(10)), Map("p7" -> 1L)), "p7",
- 1L))
- .futureValue
- offsetStore.getState().size shouldBe 7 // nothing evicted yet
-
- offsetStore
- .saveOffset(
- OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).minusSeconds(3)), Map("p8" -> 1L)),
- "p8",
- 1L))
- .futureValue
- offsetStore.getState().size shouldBe 8 // still nothing evicted yet
-
- offsetStore
- .saveOffset(
- OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(1)), Map("p8" -> 2L)),
- "p8",
- 2L))
- .futureValue
- offsetStore.getState().byPid.keySet shouldBe Set("p5", "p6", "p7", "p8")
-
- offsetStore
- .saveOffset(
- OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.plus(evictInterval).plusSeconds(20)), Map("p8" -> 3L)),
- "p8",
- 3L))
- .futureValue
- offsetStore.getState().byPid.keySet shouldBe Set("p7", "p8")
- }
-
- "delete old records" in {
- val projectionId = genRandomProjectionId()
- val deleteSettings = settings.withTimeWindow(JDuration.ofSeconds(100))
- import deleteSettings._
- val offsetStore = createOffsetStore(projectionId, deleteSettings)
-
- val startTime = TestClock.nowMicros().instant()
- log.debug("Start time [{}]", startTime)
-
- offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map("p1" -> 1L)), "p1", 1L)).futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(1)), Map("p2" -> 1L)), "p2", 1L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(2)), Map("p3" -> 1L)), "p3", 1L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(JDuration.ofSeconds(3)), Map("p4" -> 1L)), "p4", 1L))
- .futureValue
- offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 0
- offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().size shouldBe 4
-
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(2)), Map("p5" -> 1L)), "p5",
- 1L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.minusSeconds(1)), Map("p6" -> 1L)), "p6",
- 1L))
- .futureValue
- // nothing deleted yet
- offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 0
- offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().size shouldBe 6
-
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(1)), Map("p7" -> 1L)), "p7",
- 1L))
- .futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(2)), Map("p8" -> 1L)), "p8",
- 1L))
- .futureValue
- offsetStore.deleteOldTimestampOffsets().futureValue shouldBe 2
- offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().byPid.keySet shouldBe Set("p3", "p4", "p5", "p6", "p7", "p8")
- }
-
- "periodically delete old records" in {
- val projectionId = genRandomProjectionId()
- val deleteSettings = settings.withTimeWindow(JDuration.ofSeconds(100)).withDeleteInterval(JDuration.ofMillis(500))
- import deleteSettings._
- val offsetStore = createOffsetStore(projectionId, deleteSettings)
-
- val startTime = TestClock.nowMicros().instant()
- log.debug("Start time [{}]", startTime)
-
- offsetStore.saveOffset(OffsetPidSeqNr(TimestampOffset(startTime, Map("p1" -> 1L)), "p1", 1L)).futureValue
- offsetStore
- .saveOffset(OffsetPidSeqNr(TimestampOffset(startTime.plus(timeWindow.plusSeconds(1)), Map("p2" -> 1L)), "p2",
- 1L))
- .futureValue
- eventually {
- offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().byPid.keySet shouldBe Set("p2")
- }
-
- offsetStore
- .saveOffset(
- OffsetPidSeqNr(
- TimestampOffset(startTime.plus(timeWindow.multipliedBy(2).plusSeconds(2)), Map("p3" -> 1L)),
- "p3",
- 1L))
- .futureValue
- eventually {
- offsetStore.readOffset().futureValue // this will load from database
- offsetStore.getState().byPid.keySet shouldBe Set("p3")
- }
- }
-
- "set offset" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 10L))
- tick()
- val t2 = clock.instant()
- tick()
- val t3 = clock.instant()
- tick()
- val offset4 = TimestampOffset(clock.instant(), Map("p4" -> 40L))
-
- offsetStore
- .saveOffsets(Vector(OffsetPidSeqNr(offset1, "p1", 10L), OffsetPidSeqNr(offset4, "p4", 40L)))
- .futureValue
-
- // offset without any seen pid/seqNr
- offsetStore.managementSetOffset(TimestampOffset(t2, seen = Map.empty)).futureValue
- offsetStore.readOffset[TimestampOffset]().futureValue.get.timestamp shouldBe t2
- offsetStore.getState().latestTimestamp shouldBe t2
- offsetStore.getState().byPid("p1").seqNr shouldBe 10L
-
- // offset with seen pid/seqNr
- offsetStore.managementSetOffset(TimestampOffset(t3, seen = Map("p3" -> 30L))).futureValue
- offsetStore.readOffset[TimestampOffset]().futureValue.get.timestamp shouldBe t3
- offsetStore.getState().latestTimestamp shouldBe t3
- offsetStore.getState().byPid("p1").seqNr shouldBe 10L
- offsetStore.getState().byPid("p3").seqNr shouldBe 30L
- }
-
- "clear offset" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- tick()
- val offset1 = TimestampOffset(clock.instant(), Map("p1" -> 3L))
- tick()
- val offset2 = TimestampOffset(clock.instant(), Map("p2" -> 4L))
-
- offsetStore.saveOffsets(Vector(OffsetPidSeqNr(offset1, "p1", 3L), OffsetPidSeqNr(offset2, "p2", 4L))).futureValue
-
- offsetStore.managementClearOffset().futureValue
- offsetStore.readOffset[TimestampOffset]().futureValue shouldBe None
- }
-
- "read and save paused" in {
- val projectionId = genRandomProjectionId()
- val offsetStore = createOffsetStore(projectionId)
-
- offsetStore.readManagementState().futureValue shouldBe None
-
- offsetStore.savePaused(paused = true).futureValue
- offsetStore.readManagementState().futureValue shouldBe Some(ManagementState(paused = true))
-
- offsetStore.savePaused(paused = false).futureValue
- offsetStore.readManagementState().futureValue shouldBe Some(ManagementState(paused = false))
- }
- }
-}
diff --git a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/RuntimePluginConfigSpec.scala b/projection/src/test/scala/org/apache/pekko/projection/r2dbc/RuntimePluginConfigSpec.scala
deleted file mode 100644
index 58a8e85..0000000
--- a/projection/src/test/scala/org/apache/pekko/projection/r2dbc/RuntimePluginConfigSpec.scala
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pekko.projection.r2dbc
-
-import java.util.UUID
-
-import scala.collection.immutable.ListSet
-import scala.concurrent.Await
-import scala.concurrent.Future
-import scala.concurrent.duration.DurationInt
-import com.typesafe.config.Config
-import com.typesafe.config.ConfigFactory
-import org.apache.pekko
-import pekko.Done
-import pekko.actor.testkit.typed.scaladsl.LogCapturing
-import pekko.actor.testkit.typed.scaladsl.ScalaTestWithActorTestKit
-import pekko.actor.testkit.typed.scaladsl.TestProbe
-import pekko.actor.typed.ActorRef
-import pekko.persistence.Persistence
-import pekko.persistence.query.typed.EventEnvelope
-import pekko.persistence.r2dbc.ConnectionFactoryProvider
-import pekko.persistence.r2dbc.JournalSettings
-import pekko.persistence.r2dbc.SnapshotSettings
-import pekko.persistence.r2dbc.internal.R2dbcExecutor
-import pekko.persistence.r2dbc.journal.RuntimePluginConfigSpec._
-import pekko.projection.ProjectionBehavior
-import pekko.projection.ProjectionContext
-import pekko.projection.ProjectionId
-import pekko.projection.eventsourced.scaladsl.EventSourcedProvider
-import pekko.projection.r2dbc.scaladsl.R2dbcHandler
-import pekko.projection.r2dbc.scaladsl.R2dbcProjection
-import pekko.projection.r2dbc.scaladsl.R2dbcSession
-import pekko.projection.scaladsl.Handler
-import pekko.stream.scaladsl.FlowWithContext
-import org.scalatest.BeforeAndAfterEach
-import org.scalatest.Inside
-import org.scalatest.wordspec.AnyWordSpecLike
-import org.slf4j.LoggerFactory
-
-class RuntimePluginConfigSpec extends ScalaTestWithActorTestKit(TestConfig.config)
- with AnyWordSpecLike
- with BeforeAndAfterEach
- with LogCapturing
- with Inside {
-
- private lazy val eventSourced1 = new EventSourced {
- override def configKey: String = "plugin1"
- override def database: String = "database1"
- }
- private lazy val eventSourced2 = new EventSourced {
- override def configKey: String = "plugin2"
- override def database: String = "database2"
- }
-
- override protected def beforeEach(): Unit = {
- super.beforeAll()
... 603 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pekko.apache.org
For additional commands, e-mail: commits-help@pekko.apache.org