You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by pi...@apache.org on 2026/01/19 11:37:56 UTC
(iggy) branch master updated: feat(connectors): extend Postgres sink & source connectors, add integration tests (#2579)
This is an automated email from the ASF dual-hosted git repository.
piotr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 9319220e4 feat(connectors): extend Postgres sink & source connectors, add integration tests (#2579)
9319220e4 is described below
commit 9319220e45d731399daf948b90115e3fad0b696c
Author: Piotr Gankiewicz <pi...@gmail.com>
AuthorDate: Mon Jan 19 12:37:52 2026 +0100
feat(connectors): extend Postgres sink & source connectors, add integration tests (#2579)
- Extended Postgres sink/source connectors with JSON and raw message
format support, multiple processing modes (mark/delete), and bytea column handling
- Added comprehensive integration tests for sink and source connectors
- Improved shared configuration loader with env overrides for array and strings
- Updated Rust dependencies, while locking reqwest and a few others until fully compatible with other crates
- Added optional verbose flag for sink and source configuration to toggle logging about processed messages
---
Cargo.lock | 843 ++++++-------
Cargo.toml | 36 +-
DEPENDENCIES.md | 186 ++-
core/ai/mcp/Cargo.toml | 2 +-
core/bench/dashboard/server/Cargo.toml | 4 +-
core/cli/Cargo.toml | 2 +-
core/common/src/configs/mod.rs | 113 +-
core/connectors/runtime/src/configs/connectors.rs | 10 +
.../src/configs/connectors/local_provider.rs | 75 +-
core/connectors/runtime/src/main.rs | 2 +
core/connectors/runtime/src/sink.rs | 36 +-
core/connectors/runtime/src/source.rs | 26 +-
core/connectors/sdk/Cargo.toml | 4 +-
core/connectors/sdk/src/lib.rs | 2 +
core/connectors/sinks/README.md | 2 +
core/connectors/sinks/postgres_sink/Cargo.toml | 2 +
core/connectors/sinks/postgres_sink/README.md | 298 +++--
.../sinks/postgres_sink/config.toml} | 12 +-
core/connectors/sinks/postgres_sink/src/lib.rs | 741 +++++++++--
core/connectors/sources/README.md | 2 +
core/connectors/sources/postgres_source/Cargo.toml | 1 +
core/connectors/sources/postgres_source/README.md | 399 +++++-
.../sources/postgres_source/config.toml} | 27 +-
core/connectors/sources/postgres_source/src/lib.rs | 1299 +++++++++++++-------
core/integration/Cargo.toml | 8 +-
core/integration/tests/connectors/mod.rs | 158 ++-
core/integration/tests/connectors/postgres/mod.rs | 511 +++++++-
.../tests/connectors/postgres/postgres_sink.rs | 141 ++-
.../tests/connectors/postgres/postgres_source.rs | 209 ++++
.../connectors/postgres/{config.toml => sink.toml} | 2 +-
.../postgres/{config.toml => source.toml} | 2 +-
.../tests/connectors/random/random_source.rs | 5 +-
core/integration/tests/mcp/mod.rs | 1 +
core/server/Cargo.toml | 6 +-
34 files changed, 3679 insertions(+), 1488 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index eab223eaf..dd5bc7d89 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -103,7 +103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -220,7 +220,7 @@ dependencies = [
"actix-router",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -276,7 +276,7 @@ version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"once_cell",
"version_check",
]
@@ -568,7 +568,7 @@ dependencies = [
"arrow-schema",
"chrono",
"half",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"lexical-core",
"memchr",
"num",
@@ -657,7 +657,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"synstructure",
]
@@ -669,14 +669,14 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "assert_cmd"
-version = "2.1.1"
+version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcbb6924530aa9e0432442af08bbcafdad182db80d2e560da42a6d442535bf85"
+checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514"
dependencies = [
"anstyle",
"bstr",
@@ -729,16 +729,20 @@ dependencies = [
[[package]]
name = "async-compression"
-version = "0.4.36"
+version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37"
+checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c"
dependencies = [
- "compression-codecs",
- "compression-core",
+ "bzip2 0.5.2",
+ "flate2",
"futures-core",
"futures-io",
+ "memchr",
"pin-project-lite",
"tokio",
+ "xz2",
+ "zstd",
+ "zstd-safe",
]
[[package]]
@@ -763,7 +767,7 @@ dependencies = [
"async-trait",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"tokio",
]
@@ -821,7 +825,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -838,7 +842,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -906,9 +910,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.15.2"
+version = "1.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288"
+checksum = "e84ce723ab67259cfeb9877c6a639ee9eb7a27b28123abd71db7f0d5d0cc9d86"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -916,9 +920,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.35.0"
+version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1"
+checksum = "43a442ece363113bd4bd4c8b18977a7798dd4d3c3383f34fb61936960e8f4ad8"
dependencies = [
"cc",
"cmake",
@@ -987,7 +991,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1043,9 +1047,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64ct"
-version = "1.8.1"
+version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a"
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bdd"
@@ -1106,7 +1110,7 @@ dependencies = [
"rand 0.9.2",
"serde",
"serde_json",
- "sysinfo 0.37.2",
+ "sysinfo",
"tracing",
"uuid",
]
@@ -1207,15 +1211,16 @@ dependencies = [
[[package]]
name = "blake3"
-version = "1.8.2"
+version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0"
+checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
- "constant_time_eq",
+ "constant_time_eq 0.4.2",
+ "cpufeatures",
]
[[package]]
@@ -1315,9 +1320,9 @@ dependencies = [
[[package]]
name = "bon"
-version = "3.8.1"
+version = "3.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1"
+checksum = "234655ec178edd82b891e262ea7cf71f6584bcd09eff94db786be23f1821825c"
dependencies = [
"bon-macros",
"rustversion",
@@ -1325,17 +1330,17 @@ dependencies = [
[[package]]
name = "bon-macros"
-version = "3.8.1"
+version = "3.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645"
+checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365"
dependencies = [
- "darling 0.21.3",
+ "darling 0.23.0",
"ident_case",
"prettyplease",
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1358,7 +1363,7 @@ dependencies = [
"proc-macro-crate 3.4.0",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1466,6 +1471,15 @@ dependencies = [
"bytes",
]
+[[package]]
+name = "bzip2"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47"
+dependencies = [
+ "bzip2-sys",
+]
+
[[package]]
name = "bzip2"
version = "0.6.1"
@@ -1476,21 +1490,22 @@ dependencies = [
]
[[package]]
-name = "camino"
-version = "1.2.2"
+name = "bzip2-sys"
+version = "0.1.13+1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48"
+checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14"
dependencies = [
- "serde_core",
+ "cc",
+ "pkg-config",
]
[[package]]
-name = "cargo-platform"
-version = "0.1.9"
+name = "camino"
+version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea"
+checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48"
dependencies = [
- "serde",
+ "serde_core",
]
[[package]]
@@ -1503,20 +1518,6 @@ dependencies = [
"serde_core",
]
-[[package]]
-name = "cargo_metadata"
-version = "0.19.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba"
-dependencies = [
- "camino",
- "cargo-platform 0.1.9",
- "semver",
- "serde",
- "serde_json",
- "thiserror 2.0.17",
-]
-
[[package]]
name = "cargo_metadata"
version = "0.23.1"
@@ -1524,7 +1525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9"
dependencies = [
"camino",
- "cargo-platform 0.3.2",
+ "cargo-platform",
"semver",
"serde",
"serde_json",
@@ -1533,9 +1534,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.51"
+version = "1.2.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203"
+checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1586,14 +1587,14 @@ checksum = "5ec2398273c047c67d69794a924b1a2a5c14a5fab6bcbe8b24e86a0df9328e5e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "chrono"
-version = "0.4.42"
+version = "0.4.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2"
+checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118"
dependencies = [
"iana-time-zone",
"js-sys",
@@ -1615,9 +1616,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.53"
+version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8"
+checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1625,9 +1626,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.53"
+version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00"
+checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00"
dependencies = [
"anstream",
"anstyle",
@@ -1638,9 +1639,9 @@ dependencies = [
[[package]]
name = "clap_complete"
-version = "4.5.63"
+version = "4.5.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ef1fcbbf16b486489d0df91725ccc653c07115dd61f46363162535b74c6bc3"
+checksum = "430b4dc2b5e3861848de79627b2bedc9f3342c7da5173a14eaa5d0f8dc18ae5d"
dependencies = [
"clap",
]
@@ -1654,14 +1655,14 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "clap_lex"
-version = "0.7.6"
+version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
+checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32"
[[package]]
name = "clock"
@@ -1696,11 +1697,11 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
[[package]]
name = "colored"
-version = "3.0.0"
+version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e"
+checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -1715,9 +1716,9 @@ dependencies = [
[[package]]
name = "comfy-table"
-version = "7.2.1"
+version = "7.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b"
+checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"
dependencies = [
"crossterm",
"unicode-segmentation",
@@ -1825,7 +1826,7 @@ dependencies = [
"proc-macro-crate 3.4.0",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -1919,27 +1920,6 @@ dependencies = [
"tungstenite",
]
-[[package]]
-name = "compression-codecs"
-version = "0.4.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2"
-dependencies = [
- "bzip2",
- "compression-core",
- "flate2",
- "liblzma",
- "memchr",
- "zstd",
- "zstd-safe",
-]
-
-[[package]]
-name = "compression-core"
-version = "0.4.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d"
-
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -2004,7 +1984,7 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"once_cell",
"tiny-keccak",
]
@@ -2015,6 +1995,12 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+[[package]]
+name = "constant_time_eq"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
+
[[package]]
name = "convert_case"
version = "0.6.0"
@@ -2311,7 +2297,7 @@ dependencies = [
"proc-macro2",
"quote",
"regex",
- "syn 2.0.111",
+ "syn 2.0.114",
"synthez",
]
@@ -2353,7 +2339,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2457,7 +2443,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2471,7 +2457,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2484,7 +2470,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2495,7 +2481,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core 0.20.11",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2506,7 +2492,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81"
dependencies = [
"darling_core 0.21.3",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2517,7 +2503,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core 0.23.0",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2542,9 +2528,9 @@ dependencies = [
[[package]]
name = "data-encoding"
-version = "2.9.0"
+version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
+checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
[[package]]
name = "dbus"
@@ -2617,7 +2603,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2628,7 +2614,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2649,7 +2635,7 @@ dependencies = [
"darling 0.20.11",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2659,7 +2645,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
dependencies = [
"derive_builder_core",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2681,7 +2667,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustc_version",
- "syn 2.0.111",
+ "syn 2.0.114",
"unicode-xid",
]
@@ -2755,7 +2741,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2784,7 +2770,7 @@ checksum = "0fbbb781877580993a8707ec48672673ec7b81eeba04cfd2310bd28c08e47c8f"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -2988,7 +2974,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3057,7 +3043,7 @@ dependencies = [
"proc-macro2",
"quote",
"regex",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3149,9 +3135,9 @@ checksum = "b877ceff2e3d2922823bab7960826ce198181f9c25f75d67b913fde9237e2da6"
[[package]]
name = "fastbloom"
-version = "0.14.0"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18c1ddb9231d8554c2d6bdf4cfaabf0c59251658c68b6c95cd52dd0c513a912a"
+checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4"
dependencies = [
"getrandom 0.3.4",
"libm",
@@ -3165,14 +3151,14 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
]
[[package]]
name = "ferroid"
-version = "0.8.8"
+version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce161062fb044bd629c2393590efd47cab8d0241faf15704ffb0d47b7b4e4a35"
+checksum = "bb330bbd4cb7a5b9f559427f06f98a4f853a137c8298f3bd3f8ca57663e21986"
dependencies = [
"portable-atomic",
"rand 0.9.2",
@@ -3218,9 +3204,9 @@ dependencies = [
[[package]]
name = "file-operation"
-version = "0.8.8"
+version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34664772b984b86dbccb14ab56722778cca458b38f6f324db3331141ef441e05"
+checksum = "42c2d0511147e9c5aebb94162f9c90aaeb1c232dcd8f864654d30edfbf0385bc"
dependencies = [
"tokio",
]
@@ -3239,9 +3225,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
-version = "0.1.6"
+version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff"
+checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
[[package]]
name = "flatbuffers"
@@ -3255,13 +3241,13 @@ dependencies = [
[[package]]
name = "flate2"
-version = "1.1.5"
+version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
+checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369"
dependencies = [
"crc32fast",
- "libz-rs-sys",
"miniz_oxide",
+ "zlib-rs",
]
[[package]]
@@ -3455,7 +3441,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3533,9 +3519,9 @@ dependencies = [
[[package]]
name = "getrandom"
-version = "0.2.16"
+version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"js-sys",
@@ -3579,7 +3565,7 @@ dependencies = [
"quote",
"serde",
"serde_json",
- "syn 2.0.111",
+ "syn 2.0.114",
"textwrap",
"thiserror 2.0.17",
"typed-builder 0.23.2",
@@ -3693,7 +3679,7 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "903f432be5ba34427eac5e16048ef65604a82061fe93789f2212afc73d8617d6"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"gloo-events",
"gloo-utils",
"serde",
@@ -3803,7 +3789,7 @@ dependencies = [
"proc-macro-crate 1.3.1",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -3852,7 +3838,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http 0.2.12",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"slab",
"tokio",
"tokio-util",
@@ -3861,9 +3847,9 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.4.12"
+version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386"
+checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
dependencies = [
"atomic-waker",
"bytes",
@@ -3871,7 +3857,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http 1.4.0",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"slab",
"tokio",
"tokio-util",
@@ -3902,9 +3888,9 @@ dependencies = [
[[package]]
name = "handlebars"
-version = "6.3.2"
+version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "759e2d5aea3287cb1190c8ec394f42866cb5bf74fcbf213f354e3c856ea26098"
+checksum = "9b3f9296c208515b87bd915a2f5d1163d4b3f863ba83337d7713cf478055948e"
dependencies = [
"derive_builder",
"log",
@@ -4165,7 +4151,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-core",
- "h2 0.4.12",
+ "h2 0.4.13",
"http 1.4.0",
"http-body",
"httparse",
@@ -4209,7 +4195,7 @@ dependencies = [
"tokio",
"tokio-rustls",
"tower-service",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -4502,7 +4488,7 @@ dependencies = [
"tracing",
"trait-variant",
"tungstenite",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -4526,9 +4512,9 @@ dependencies = [
"rand 0.9.2",
"rayon",
"serde",
- "sysinfo 0.37.2",
+ "sysinfo",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tracing",
"tracing-appender",
"tracing-subscriber",
@@ -4617,7 +4603,7 @@ dependencies = [
"tempfile",
"thiserror 2.0.17",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tower-http",
"tracing",
"tracing-subscriber",
@@ -4664,7 +4650,7 @@ dependencies = [
"serde",
"serde_json",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tracing",
]
@@ -4706,7 +4692,7 @@ dependencies = [
"strum 0.27.2",
"thiserror 2.0.17",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tracing",
"tungstenite",
"twox-hash",
@@ -4775,9 +4761,11 @@ dependencies = [
"chrono",
"dashmap",
"futures",
+ "humantime",
"iggy_connector_sdk",
"once_cell",
"serde",
+ "serde_json",
"simd-json",
"sqlx",
"tokio",
@@ -4789,6 +4777,7 @@ name = "iggy_connector_postgres_source"
version = "0.1.0"
dependencies = [
"async-trait",
+ "base64 0.22.1",
"chrono",
"dashmap",
"futures",
@@ -4926,7 +4915,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1689b939ee35e3a075b0834b5672efd43aec8a6e81a1c6002b76a5ca2f211ae0"
dependencies = [
"implicit-clone-derive",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
]
[[package]]
@@ -4936,7 +4925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "699c1b6d335e63d0ba5c1e1c7f647371ce989c3bcbe1f7ed2b85fa56e3bd1a21"
dependencies = [
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -4952,9 +4941,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.12.1"
+version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2"
+checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
@@ -5036,9 +5025,11 @@ dependencies = [
"rcgen",
"reqwest",
"rmcp",
+ "serde",
"serde_json",
"serial_test",
"server",
+ "sqlx",
"strum 0.27.2",
"strum_macros 0.27.2",
"tempfile",
@@ -5129,9 +5120,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]]
name = "jiff"
-version = "0.2.17"
+version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a87d9b8105c23642f50cbbae03d1f75d8422c5cb98ce7ee9271f7ff7505be6b8"
+checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50"
dependencies = [
"jiff-static",
"log",
@@ -5142,13 +5133,13 @@ dependencies = [
[[package]]
name = "jiff-static"
-version = "0.2.17"
+version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b787bebb543f8969132630c51fd0afab173a86c6abae56ff3b9e5e3e3f9f6e58"
+checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -5189,9 +5180,9 @@ version = "0.1.0"
[[package]]
name = "js-sys"
-version = "0.3.83"
+version = "0.3.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8"
+checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3"
dependencies = [
"once_cell",
"wasm-bindgen",
@@ -5205,7 +5196,7 @@ checksum = "c76e1c7d7df3e34443b3621b459b066a7b79644f059fc8b2db7070c825fd417e"
dependencies = [
"base64 0.22.1",
"ed25519-dalek",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"hmac",
"js-sys",
"p256",
@@ -5387,9 +5378,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
[[package]]
name = "libc"
-version = "0.2.178"
+version = "0.2.180"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
+checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
[[package]]
name = "libdbus-sys"
@@ -5437,26 +5428,6 @@ dependencies = [
"pkg-config",
]
-[[package]]
-name = "liblzma"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648"
-dependencies = [
- "liblzma-sys",
-]
-
-[[package]]
-name = "liblzma-sys"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
-]
-
[[package]]
name = "libm"
version = "0.2.15"
@@ -5494,15 +5465,6 @@ dependencies = [
"vcpkg",
]
-[[package]]
-name = "libz-rs-sys"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415"
-dependencies = [
- "zlib-rs",
-]
-
[[package]]
name = "libz-sys"
version = "1.1.23"
@@ -5593,7 +5555,7 @@ dependencies = [
"quote",
"regex-syntax",
"rustc_version",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -5643,6 +5605,17 @@ dependencies = [
"sha2",
]
+[[package]]
+name = "lzma-sys"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
[[package]]
name = "macro_rules_attribute"
version = "0.1.3"
@@ -5731,7 +5704,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -5810,7 +5783,7 @@ dependencies = [
"cfg-if",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6135,21 +6108,21 @@ dependencies = [
[[package]]
name = "octocrab"
-version = "0.49.4"
+version = "0.49.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "add74bcdaf5b0a0c3edb0f8c0a952d7a52d9e9a76ac9c690a3e477c4e8c343b9"
+checksum = "89f6f72d7084a80bf261bb6b6f83bd633323d5633d5ec7988c6c95b20448b2b5"
dependencies = [
"arc-swap",
"async-trait",
"base64 0.22.1",
"bytes",
- "cargo_metadata 0.23.1",
+ "cargo_metadata",
"cfg-if",
"chrono",
"either",
"futures",
"futures-util",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"http-body",
"http-body-util",
@@ -6215,7 +6188,7 @@ dependencies = [
"chrono",
"crc32c",
"futures",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"http-body",
"log",
@@ -6253,7 +6226,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6542,7 +6515,7 @@ dependencies = [
"regex",
"regex-syntax",
"structmeta",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6615,7 +6588,7 @@ dependencies = [
"proc-macro2",
"proc-macro2-diagnostics",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6672,9 +6645,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pest"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22"
+checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7"
dependencies = [
"memchr",
"ucd-trie",
@@ -6682,9 +6655,9 @@ dependencies = [
[[package]]
name = "pest_derive"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51f72981ade67b1ca6adc26ec221be9f463f2b5839c7508998daa17c23d94d7f"
+checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed"
dependencies = [
"pest",
"pest_generator",
@@ -6692,22 +6665,22 @@ dependencies = [
[[package]]
name = "pest_generator"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dee9efd8cdb50d719a80088b76f81aec7c41ed6d522ee750178f83883d271625"
+checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "pest_meta"
-version = "2.8.4"
+version = "2.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf1d70880e76bdc13ba52eafa6239ce793d85c8e43896507e43dd8984ff05b82"
+checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365"
dependencies = [
"pest",
"sha2",
@@ -6730,7 +6703,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6910,7 +6883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -6967,9 +6940,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.104"
+version = "1.0.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
+checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7"
dependencies = [
"unicode-ident",
]
@@ -6982,7 +6955,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"version_check",
"yansi",
]
@@ -7007,14 +6980,14 @@ checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "prost"
-version = "0.14.1"
+version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d"
+checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
dependencies = [
"bytes",
"prost-derive",
@@ -7022,15 +6995,15 @@ dependencies = [
[[package]]
name = "prost-derive"
-version = "0.14.1"
+version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
+checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools 0.14.0",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -7047,9 +7020,9 @@ dependencies = [
[[package]]
name = "prost-types"
-version = "0.14.1"
+version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72"
+checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
dependencies = [
"prost",
]
@@ -7201,9 +7174,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.42"
+version = "1.0.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
+checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a"
dependencies = [
"proc-macro2",
]
@@ -7238,7 +7211,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [
"rand_chacha 0.9.0",
- "rand_core 0.9.3",
+ "rand_core 0.9.5",
]
[[package]]
@@ -7258,7 +7231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core 0.9.3",
+ "rand_core 0.9.5",
]
[[package]]
@@ -7267,14 +7240,14 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
]
[[package]]
name = "rand_core"
-version = "0.9.3"
+version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
"getrandom 0.3.4",
]
@@ -7285,7 +7258,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
dependencies = [
- "rand_core 0.9.3",
+ "rand_core 0.9.5",
]
[[package]]
@@ -7355,7 +7328,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
dependencies = [
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"libredox",
"thiserror 2.0.17",
]
@@ -7377,7 +7350,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -7435,7 +7408,7 @@ dependencies = [
"base64 0.22.1",
"chrono",
"form_urlencoded",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"hex",
"hmac",
"home",
@@ -7492,7 +7465,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -7519,7 +7492,7 @@ dependencies = [
"anyhow",
"async-trait",
"futures",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"hyper",
"reqwest",
@@ -7539,7 +7512,7 @@ checksum = "d70ea85f131b2ee9874f0b160ac5976f8af75f3c9badfe0d955880257d10bd83"
dependencies = [
"anyhow",
"async-trait",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"http 1.4.0",
"matchit",
"reqwest",
@@ -7574,7 +7547,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
- "getrandom 0.2.16",
+ "getrandom 0.2.17",
"libc",
"untrusted",
"windows-sys 0.52.0",
@@ -7588,9 +7561,9 @@ checksum = "57b0b88a509053cbfd535726dcaaceee631313cef981266119527a1d110f6d2b"
[[package]]
name = "rkyv"
-version = "0.7.45"
+version = "0.7.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b"
+checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
dependencies = [
"bitvec",
"bytecheck",
@@ -7606,9 +7579,9 @@ dependencies = [
[[package]]
name = "rkyv_derive"
-version = "0.7.45"
+version = "0.7.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0"
+checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
dependencies = [
"proc-macro2",
"quote",
@@ -7623,11 +7596,12 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
[[package]]
name = "rmcp"
-version = "0.12.0"
+version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528d42f8176e6e5e71ea69182b17d1d0a19a6b3b894b564678b74cd7cab13cfa"
+checksum = "d1815dbc06c414d720f8bc1951eccd66bc99efc6376331f1e7093a119b3eb508"
dependencies = [
"async-trait",
+ "axum",
"base64 0.22.1",
"bytes",
"chrono",
@@ -7655,15 +7629,15 @@ dependencies = [
[[package]]
name = "rmcp-macros"
-version = "0.12.0"
+version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3f81daaa494eb8e985c9462f7d6ce1ab05e5299f48aafd76cdd3d8b060e6f59"
+checksum = "11f0bc7008fa102e771a76c6d2c9b253be3f2baa5964e060464d038ae1cbc573"
dependencies = [
"darling 0.23.0",
"proc-macro2",
"quote",
"serde_json",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -7703,9 +7677,9 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746"
[[package]]
name = "rsa"
-version = "0.9.9"
+version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88"
+checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d"
dependencies = [
"const-oid",
"digest",
@@ -7723,9 +7697,9 @@ dependencies = [
[[package]]
name = "rust-embed"
-version = "8.9.0"
+version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "947d7f3fad52b283d261c4c99a084937e2fe492248cb9a68a8435a861b8798ca"
+checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
@@ -7734,22 +7708,22 @@ dependencies = [
[[package]]
name = "rust-embed-impl"
-version = "8.9.0"
+version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fa2c8c9e8711e10f9c4fd2d64317ef13feaab820a4c51541f1a8c8e2e851ab2"
+checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
- "syn 2.0.111",
+ "syn 2.0.114",
"walkdir",
]
[[package]]
name = "rust-embed-utils"
-version = "8.9.0"
+version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b161f275cb337fe0a44d924a5f4df0ed69c2c39519858f931ce61c779d3475"
+checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
dependencies = [
"sha2",
"walkdir",
@@ -7767,9 +7741,9 @@ dependencies = [
[[package]]
name = "rust_decimal"
-version = "1.39.0"
+version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282"
+checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0"
dependencies = [
"arrayvec",
"borsh",
@@ -7820,9 +7794,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.35"
+version = "0.23.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
+checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b"
dependencies = [
"aws-lc-rs",
"log",
@@ -7857,9 +7831,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
-version = "1.13.2"
+version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282"
+checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
dependencies = [
"web-time",
"zeroize",
@@ -7894,9 +7868,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
-version = "0.103.8"
+version = "0.103.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52"
+checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
dependencies = [
"aws-lc-rs",
"ring",
@@ -7978,7 +7952,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8013,7 +7987,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8145,7 +8119,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8156,14 +8130,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
name = "serde_json"
-version = "1.0.148"
+version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [
"itoa",
"memchr",
@@ -8191,7 +8165,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8234,7 +8208,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"schemars 0.9.0",
"schemars 1.2.0",
"serde_core",
@@ -8252,7 +8226,7 @@ dependencies = [
"darling 0.21.3",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8261,7 +8235,7 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"itoa",
"ryu",
"serde",
@@ -8270,11 +8244,12 @@ dependencies = [
[[package]]
name = "serial_test"
-version = "3.2.0"
+version = "3.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9"
+checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555"
dependencies = [
- "futures",
+ "futures-executor",
+ "futures-util",
"log",
"once_cell",
"parking_lot",
@@ -8284,13 +8259,13 @@ dependencies = [
[[package]]
name = "serial_test_derive"
-version = "3.2.0"
+version = "3.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef"
+checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8359,11 +8334,11 @@ dependencies = [
"socket2 0.6.1",
"static-toml",
"strum 0.27.2",
- "sysinfo 0.37.2",
+ "sysinfo",
"tempfile",
"thiserror 2.0.17",
"tokio",
- "toml 0.9.10+spec-1.1.0",
+ "toml 0.9.11+spec-1.1.0",
"tower-http",
"tracing",
"tracing-appender",
@@ -8509,7 +8484,7 @@ checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8536,7 +8511,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8625,7 +8600,7 @@ dependencies = [
"futures-util",
"hashbrown 0.15.5",
"hashlink",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"log",
"memchr",
"once_cell",
@@ -8654,7 +8629,7 @@ dependencies = [
"quote",
"sqlx-core",
"sqlx-macros-core",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8677,7 +8652,7 @@ dependencies = [
"sqlx-mysql",
"sqlx-postgres",
"sqlx-sqlite",
- "syn 2.0.111",
+ "syn 2.0.114",
"tokio",
"url",
]
@@ -8820,7 +8795,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"toml 0.8.23",
]
@@ -8850,7 +8825,7 @@ dependencies = [
"proc-macro2",
"quote",
"structmeta-derive",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8861,7 +8836,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8889,7 +8864,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8901,7 +8876,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8923,9 +8898,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.111"
+version = "2.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87"
+checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a"
dependencies = [
"proc-macro2",
"quote",
@@ -8949,7 +8924,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -8958,7 +8933,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d8a928f38f1bc873f28e0d2ba8298ad65374a6ac2241dabd297271531a736cd"
dependencies = [
- "syn 2.0.111",
+ "syn 2.0.114",
"synthez-codegen",
"synthez-core",
]
@@ -8969,7 +8944,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fb83b8df4238e11746984dfb3819b155cd270de0e25847f45abad56b3671047"
dependencies = [
- "syn 2.0.111",
+ "syn 2.0.114",
"synthez-core",
]
@@ -8982,20 +8957,7 @@ dependencies = [
"proc-macro2",
"quote",
"sealed",
- "syn 2.0.111",
-]
-
-[[package]]
-name = "sysinfo"
-version = "0.34.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4b93974b3d3aeaa036504b8eefd4c039dced109171c1ae973f1dc63b2c7e4b2"
-dependencies = [
- "libc",
- "memchr",
- "ntapi",
- "objc2-core-foundation",
- "windows 0.57.0",
+ "syn 2.0.114",
]
[[package]]
@@ -9009,7 +8971,7 @@ dependencies = [
"ntapi",
"objc2-core-foundation",
"objc2-io-kit",
- "windows 0.61.3",
+ "windows",
]
[[package]]
@@ -9082,7 +9044,7 @@ dependencies = [
"cfg-if",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9093,7 +9055,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"test-case-core",
]
@@ -9173,7 +9135,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9184,7 +9146,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9220,9 +9182,9 @@ dependencies = [
[[package]]
name = "time"
-version = "0.3.44"
+version = "0.3.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d"
+checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd"
dependencies = [
"deranged",
"itoa",
@@ -9230,22 +9192,22 @@ dependencies = [
"num-conv",
"num_threads",
"powerfmt",
- "serde",
+ "serde_core",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b"
+checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca"
[[package]]
name = "time-macros"
-version = "0.2.24"
+version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3"
+checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd"
dependencies = [
"num-conv",
"time-core",
@@ -9287,9 +9249,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.48.0"
+version = "1.49.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408"
+checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86"
dependencies = [
"bytes",
"libc",
@@ -9310,7 +9272,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9325,9 +9287,9 @@ dependencies = [
[[package]]
name = "tokio-stream"
-version = "0.1.17"
+version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
+checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
dependencies = [
"futures-core",
"pin-project-lite",
@@ -9352,9 +9314,9 @@ dependencies = [
[[package]]
name = "tokio-util"
-version = "0.7.17"
+version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594"
+checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
dependencies = [
"bytes",
"futures-core",
@@ -9395,11 +9357,11 @@ dependencies = [
[[package]]
name = "toml"
-version = "0.9.10+spec-1.1.0"
+version = "0.9.11+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0825052159284a1a8b4d6c0c86cbc801f2da5afd2b225fa548c72f2e74002f48"
+checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"serde_core",
"serde_spanned 1.0.4",
"toml_datetime 0.7.5+spec-1.1.0",
@@ -9432,7 +9394,7 @@ version = "0.19.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"toml_datetime 0.6.11",
"winnow 0.5.40",
]
@@ -9443,7 +9405,7 @@ version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.11",
@@ -9457,7 +9419,7 @@ version = "0.23.10+spec-1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
dependencies = [
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"winnow 0.7.14",
@@ -9494,7 +9456,7 @@ dependencies = [
"axum",
"base64 0.22.1",
"bytes",
- "h2 0.4.12",
+ "h2 0.4.13",
"http 1.4.0",
"http-body",
"http-body-util",
@@ -9539,13 +9501,13 @@ dependencies = [
[[package]]
name = "tower"
-version = "0.5.2"
+version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
+checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [
"futures-core",
"futures-util",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"pin-project-lite",
"slab",
"sync_wrapper",
@@ -9624,7 +9586,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9650,16 +9612,13 @@ dependencies = [
[[package]]
name = "tracing-opentelemetry"
-version = "0.32.0"
+version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e6e5658463dd88089aba75c7791e1d3120633b1bfde22478b28f625a9bb1b8e"
+checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc"
dependencies = [
"js-sys",
"opentelemetry",
- "opentelemetry_sdk",
- "rustversion",
"smallvec",
- "thiserror 2.0.17",
"tracing",
"tracing-core",
"tracing-log",
@@ -9693,7 +9652,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9765,7 +9724,7 @@ checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9776,7 +9735,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9787,7 +9746,7 @@ checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -9823,9 +9782,9 @@ dependencies = [
[[package]]
name = "unicase"
-version = "2.8.1"
+version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
+checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-bidi"
@@ -9920,7 +9879,7 @@ dependencies = [
"rustls-pki-types",
"ureq-proto",
"utf-8",
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
@@ -9937,14 +9896,15 @@ dependencies = [
[[package]]
name = "url"
-version = "2.5.7"
+version = "2.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b"
+checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
"serde",
+ "serde_derive",
]
[[package]]
@@ -10023,26 +9983,26 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vergen"
-version = "9.0.6"
+version = "9.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b2bf58be11fc9414104c6d3a2e464163db5ef74b12296bda593cac37b6e4777"
+checksum = "b849a1f6d8639e8de261e81ee0fc881e3e3620db1af9f2e0da015d4382ceaf75"
dependencies = [
"anyhow",
- "cargo_metadata 0.19.2",
+ "cargo_metadata",
"derive_builder",
"regex",
"rustc_version",
"rustversion",
- "sysinfo 0.34.2",
+ "sysinfo",
"time",
"vergen-lib",
]
[[package]]
name = "vergen-git2"
-version = "1.0.7"
+version = "9.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f6ee511ec45098eabade8a0750e76eec671e7fb2d9360c563911336bea9cac1"
+checksum = "d51ab55ddf1188c8d679f349775362b0fa9e90bd7a4ac69838b2a087623f0d57"
dependencies = [
"anyhow",
"derive_builder",
@@ -10055,9 +10015,9 @@ dependencies = [
[[package]]
name = "vergen-lib"
-version = "0.1.6"
+version = "9.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b07e6010c0f3e59fcb164e0163834597da68d1f864e2b8ca49f74de01e9c166"
+checksum = "b34a29ba7e9c59e62f229ae1932fb1b8fb8a6fdcc99215a641913f5f5a59a569"
dependencies = [
"anyhow",
"derive_builder",
@@ -10112,9 +10072,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasip2"
-version = "1.0.1+wasi-0.2.4"
+version = "1.0.2+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
+checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
dependencies = [
"wit-bindgen",
]
@@ -10127,9 +10087,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
[[package]]
name = "wasm-bindgen"
-version = "0.2.106"
+version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd"
+checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566"
dependencies = [
"cfg-if",
"once_cell",
@@ -10140,11 +10100,12 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.56"
+version = "0.4.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c"
+checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f"
dependencies = [
"cfg-if",
+ "futures-util",
"js-sys",
"once_cell",
"wasm-bindgen",
@@ -10153,9 +10114,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.106"
+version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"
+checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -10163,22 +10124,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.106"
+version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"
+checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.106"
+version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4"
+checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12"
dependencies = [
"unicode-ident",
]
@@ -10212,9 +10173,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.83"
+version = "0.3.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac"
+checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -10233,9 +10194,9 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
-version = "1.0.4"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b"
+checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc"
dependencies = [
"rustls-pki-types",
]
@@ -10246,14 +10207,14 @@ version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
- "webpki-roots 1.0.4",
+ "webpki-roots 1.0.5",
]
[[package]]
name = "webpki-roots"
-version = "1.0.4"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e"
+checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c"
dependencies = [
"rustls-pki-types",
]
@@ -10305,16 +10266,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-[[package]]
-name = "windows"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
-dependencies = [
- "windows-core 0.57.0",
- "windows-targets 0.52.6",
-]
-
[[package]]
name = "windows"
version = "0.61.3"
@@ -10337,26 +10288,14 @@ dependencies = [
"windows-core 0.61.2",
]
-[[package]]
-name = "windows-core"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
-dependencies = [
- "windows-implement 0.57.0",
- "windows-interface 0.57.0",
- "windows-result 0.1.2",
- "windows-targets 0.52.6",
-]
-
[[package]]
name = "windows-core"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
dependencies = [
- "windows-implement 0.60.2",
- "windows-interface 0.59.3",
+ "windows-implement",
+ "windows-interface",
"windows-link 0.1.3",
"windows-result 0.3.4",
"windows-strings 0.4.2",
@@ -10368,8 +10307,8 @@ version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
- "windows-implement 0.60.2",
- "windows-interface 0.59.3",
+ "windows-implement",
+ "windows-interface",
"windows-link 0.2.1",
"windows-result 0.4.1",
"windows-strings 0.5.1",
@@ -10386,17 +10325,6 @@ dependencies = [
"windows-threading",
]
-[[package]]
-name = "windows-implement"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.111",
-]
-
[[package]]
name = "windows-implement"
version = "0.60.2"
@@ -10405,18 +10333,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
-]
-
-[[package]]
-name = "windows-interface"
-version = "0.57.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10427,7 +10344,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10452,15 +10369,6 @@ dependencies = [
"windows-link 0.1.3",
]
-[[package]]
-name = "windows-result"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
[[package]]
name = "windows-result"
version = "0.3.4"
@@ -10823,9 +10731,9 @@ dependencies = [
[[package]]
name = "wit-bindgen"
-version = "0.46.0"
+version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
+checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
[[package]]
name = "writeable"
@@ -10870,6 +10778,15 @@ dependencies = [
"rustix",
]
+[[package]]
+name = "xz2"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
+dependencies = [
+ "lzma-sys",
+]
+
[[package]]
name = "yansi"
version = "1.0.1"
@@ -10895,7 +10812,7 @@ dependencies = [
"futures",
"gloo",
"implicit-clone",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"js-sys",
"rustversion",
"serde",
@@ -10922,7 +10839,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10952,7 +10869,7 @@ checksum = "9e87a3ce33434ab66a700edbaf2cc8a417d9b89f00a6fd8216fd6ac83b0e7b1c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -10974,28 +10891,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"synstructure",
]
[[package]]
name = "zerocopy"
-version = "0.8.31"
+version = "0.8.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3"
+checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.31"
+version = "0.8.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a"
+checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -11015,7 +10932,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
"synstructure",
]
@@ -11036,7 +10953,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -11069,7 +10986,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.111",
+ "syn 2.0.114",
]
[[package]]
@@ -11080,15 +10997,15 @@ checksum = "bdd8a47718a4ee5fe78e07667cd36f3de80e7c2bfe727c7074245ffc7303c037"
dependencies = [
"aes",
"arbitrary",
- "bzip2",
- "constant_time_eq",
+ "bzip2 0.6.1",
+ "constant_time_eq 0.3.1",
"crc32fast",
"deflate64",
"flate2",
"generic-array",
"getrandom 0.3.4",
"hmac",
- "indexmap 2.12.1",
+ "indexmap 2.13.0",
"lzma-rust2",
"memchr",
"pbkdf2",
@@ -11108,9 +11025,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
[[package]]
name = "zmij"
-version = "1.0.3"
+version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06"
+checksum = "bd8f3f50b848df28f887acb68e41201b5aea6bc8a8dacc00fb40635ff9a72fea"
[[package]]
name = "zopfli"
diff --git a/Cargo.toml b/Cargo.toml
index 3db1cac62..dbe9f6a98 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -82,8 +82,8 @@ bench-dashboard-shared = { path = "core/bench/dashboard/shared" }
bench-report = { path = "core/bench/report" }
bench-runner = { path = "core/bench/runner" }
bit-set = "0.8.0"
-blake3 = "1.8.2"
-bon = "3.8.1"
+blake3 = "1.8.3"
+bon = "3.8.2"
byte-unit = { version = "5.2.0", default-features = false, features = [
"serde",
"byte",
@@ -92,10 +92,10 @@ byte-unit = { version = "5.2.0", default-features = false, features = [
bytemuck = { version = "1.24" }
bytes = "1.11.0"
charming = "0.6.0"
-chrono = { version = "0.4.42", features = ["serde"] }
-clap = { version = "4.5.53", features = ["derive", "wrap_help"] }
-colored = "3.0.0"
-comfy-table = "7.2.1"
+chrono = { version = "0.4.43", features = ["serde"] }
+clap = { version = "4.5.54", features = ["derive", "wrap_help"] }
+colored = "3.1.1"
+comfy-table = "7.2.2"
compio = { version = "0.17.0", features = [
"runtime",
"macros",
@@ -157,24 +157,24 @@ quinn = "0.11.9"
rand = "0.9.2"
rand_xoshiro = "0.7.0"
regex = "1.12.2"
-reqwest = { version = "0.12.28", default-features = false, features = [
+reqwest = { version = "=0.12.28", default-features = false, features = [
"json",
"rustls-tls",
] }
-reqwest-middleware = { version = "0.4.2", features = ["json"] }
-reqwest-retry = "0.8.0"
-reqwest-tracing = "0.5.8"
+reqwest-middleware = { version = "=0.4.2", features = ["json"] }
+reqwest-retry = "=0.8.0"
+reqwest-tracing = "=0.5.8"
rmp-serde = "1.3.1"
rust-s3 = { version = "0.37.1", default-features = false, features = [
"tokio-rustls-tls",
"tags",
] }
-rustls = { version = "0.23.35", features = ["ring"] }
+rustls = { version = "0.23.36", features = ["ring"] }
serde = { version = "1.0.228", features = ["derive", "rc"] }
-serde_json = "1.0.148"
+serde_json = "1.0.149"
serde_with = { version = "3.16.1", features = ["base64", "macros"] }
serde_yaml_ng = "0.10.0"
-serial_test = "3.2.0"
+serial_test = "3.3.1"
server = { path = "core/server" }
simd-json = { version = "0.17.0", features = ["serde_impl"] }
strum = { version = "0.27.2", features = ["derive"] }
@@ -183,11 +183,11 @@ sysinfo = "0.37.2"
tempfile = "3.24.0"
test-case = "3.3.1"
thiserror = "2.0.17"
-tokio = { version = "1.48.0", features = ["full"] }
+tokio = { version = "1.49.0", features = ["full"] }
tokio-rustls = "0.26.4"
tokio-tungstenite = { version = "0.28", features = ["rustls-tls-webpki-roots"] }
-tokio-util = { version = "0.7.17", features = ["compat"] }
-toml = "0.9.10"
+tokio-util = { version = "0.7.18", features = ["compat"] }
+toml = "0.9.11"
tower-http = { version = "0.6.8", features = [
"add-extension",
"cors",
@@ -210,8 +210,8 @@ uuid = { version = "1.19.0", features = [
"serde",
"zerocopy",
] }
-webpki-roots = "1.0.4"
-zip = "7.0.0"
+webpki-roots = "1.0.5"
+zip = "=7.0.0"
[profile.release]
lto = true
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index 1468dcbb0..3ea9bbf8b 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -53,11 +53,11 @@ as-any: 0.3.2, "Apache-2.0 OR MIT",
asn1-rs: 0.7.1, "Apache-2.0 OR MIT",
asn1-rs-derive: 0.6.0, "Apache-2.0 OR MIT",
asn1-rs-impl: 0.2.0, "Apache-2.0 OR MIT",
-assert_cmd: 2.1.1, "Apache-2.0 OR MIT",
+assert_cmd: 2.1.2, "Apache-2.0 OR MIT",
astral-tokio-tar: 0.5.6, "Apache-2.0 OR MIT",
async-broadcast: 0.7.2, "Apache-2.0 OR MIT",
async-channel: 2.5.0, "Apache-2.0 OR MIT",
-async-compression: 0.4.36, "Apache-2.0 OR MIT",
+async-compression: 0.4.19, "Apache-2.0 OR MIT",
async-dropper: 0.3.1, "MIT",
async-dropper-derive: 0.3.1, "MIT",
async-dropper-simple: 0.2.6, "MIT",
@@ -74,8 +74,8 @@ atomic-polyfill: 1.0.3, "Apache-2.0 OR MIT",
atomic-waker: 1.1.2, "Apache-2.0 OR MIT",
autocfg: 1.5.0, "Apache-2.0 OR MIT",
autotools: 0.2.7, "MIT",
-aws-lc-rs: 1.15.2, "(Apache-2.0 OR ISC) AND ISC",
-aws-lc-sys: 0.35.0, "(Apache-2.0 OR ISC) AND ISC AND OpenSSL",
+aws-lc-rs: 1.15.3, "(Apache-2.0 OR ISC) AND ISC",
+aws-lc-sys: 0.36.0, "(Apache-2.0 OR ISC) AND ISC AND OpenSSL",
axum: 0.8.8, "MIT",
axum-core: 0.5.6, "MIT",
axum-macros: 0.5.0, "MIT",
@@ -84,7 +84,7 @@ backon: 1.6.0, "Apache-2.0",
base16ct: 0.2.0, "Apache-2.0 OR MIT",
base64: 0.21.7, "Apache-2.0 OR MIT",
base64: 0.22.1, "Apache-2.0 OR MIT",
-base64ct: 1.8.1, "Apache-2.0 OR MIT",
+base64ct: 1.8.3, "Apache-2.0 OR MIT",
bdd: 0.0.1, "Apache-2.0",
beef: 0.5.2, "Apache-2.0 OR MIT",
bench-dashboard-frontend: 0.4.1, "Apache-2.0",
@@ -100,14 +100,14 @@ bitflags: 1.3.2, "Apache-2.0 OR MIT",
bitflags: 2.10.0, "Apache-2.0 OR MIT",
bitvec: 1.0.1, "MIT",
blake2: 0.10.6, "Apache-2.0 OR MIT",
-blake3: 1.8.2, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR CC0-1.0",
+blake3: 1.8.3, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR CC0-1.0",
block-buffer: 0.10.4, "Apache-2.0 OR MIT",
block2: 0.6.2, "MIT",
bollard: 0.19.4, "Apache-2.0",
bollard-buildkit-proto: 0.7.0, "Apache-2.0",
bollard-stubs: 1.49.1-rc.28.4.0, "Apache-2.0",
-bon: 3.8.1, "Apache-2.0 OR MIT",
-bon-macros: 3.8.1, "Apache-2.0 OR MIT",
+bon: 3.8.2, "Apache-2.0 OR MIT",
+bon-macros: 3.8.2, "Apache-2.0 OR MIT",
borsh: 1.6.0, "Apache-2.0 OR MIT",
borsh-derive: 1.6.0, "Apache-2.0",
brotli: 8.0.2, "BSD-3-Clause AND MIT",
@@ -122,32 +122,32 @@ bytemuck: 1.24.0, "Apache-2.0 OR MIT OR Zlib",
byteorder: 1.5.0, "MIT OR Unlicense",
bytes: 1.11.0, "MIT",
bytestring: 1.5.0, "Apache-2.0 OR MIT",
+bzip2: 0.5.2, "Apache-2.0 OR MIT",
bzip2: 0.6.1, "Apache-2.0 OR MIT",
+bzip2-sys: 0.1.13+1.0.8, "Apache-2.0 OR MIT",
camino: 1.2.2, "Apache-2.0 OR MIT",
-cargo-platform: 0.1.9, "Apache-2.0 OR MIT",
cargo-platform: 0.3.2, "Apache-2.0 OR MIT",
-cargo_metadata: 0.19.2, "MIT",
cargo_metadata: 0.23.1, "MIT",
-cc: 1.2.51, "Apache-2.0 OR MIT",
+cc: 1.2.53, "Apache-2.0 OR MIT",
cesu8: 1.1.0, "Apache-2.0 OR MIT",
cfg-if: 1.0.4, "Apache-2.0 OR MIT",
cfg_aliases: 0.2.1, "MIT",
charming: 0.6.0, "Apache-2.0 OR MIT",
charming_macros: 0.1.0, "Apache-2.0 OR MIT",
-chrono: 0.4.42, "Apache-2.0 OR MIT",
+chrono: 0.4.43, "Apache-2.0 OR MIT",
cipher: 0.4.4, "Apache-2.0 OR MIT",
-clap: 4.5.53, "Apache-2.0 OR MIT",
-clap_builder: 4.5.53, "Apache-2.0 OR MIT",
-clap_complete: 4.5.63, "Apache-2.0 OR MIT",
+clap: 4.5.54, "Apache-2.0 OR MIT",
+clap_builder: 4.5.54, "Apache-2.0 OR MIT",
+clap_complete: 4.5.65, "Apache-2.0 OR MIT",
clap_derive: 4.5.49, "Apache-2.0 OR MIT",
-clap_lex: 0.7.6, "Apache-2.0 OR MIT",
+clap_lex: 0.7.7, "Apache-2.0 OR MIT",
clock: 0.1.0, "N/A",
cmake: 0.1.57, "Apache-2.0 OR MIT",
cobs: 0.3.0, "Apache-2.0 OR MIT",
colorchoice: 1.0.4, "Apache-2.0 OR MIT",
-colored: 3.0.0, "MPL-2.0",
+colored: 3.1.1, "MPL-2.0",
combine: 4.6.7, "MIT",
-comfy-table: 7.2.1, "MIT",
+comfy-table: 7.2.2, "MIT",
compio: 0.17.0, "MIT",
compio-buf: 0.7.2, "MIT",
compio-driver: 0.10.0, "MIT",
@@ -160,8 +160,6 @@ compio-quic: 0.6.0, "MIT",
compio-runtime: 0.10.1, "MIT",
compio-tls: 0.8.0, "MIT",
compio-ws: 0.2.0, "MIT",
-compression-codecs: 0.4.35, "Apache-2.0 OR MIT",
-compression-core: 0.4.31, "Apache-2.0 OR MIT",
concurrent-queue: 2.5.0, "Apache-2.0 OR MIT",
consensus: 0.1.0, "Apache-2.0",
console: 0.16.2, "MIT",
@@ -170,6 +168,7 @@ const-oid: 0.9.6, "Apache-2.0 OR MIT",
const-random: 0.1.18, "Apache-2.0 OR MIT",
const-random-macro: 0.1.16, "Apache-2.0 OR MIT",
constant_time_eq: 0.3.1, "Apache-2.0 OR CC0-1.0 OR MIT-0",
+constant_time_eq: 0.4.2, "Apache-2.0 OR CC0-1.0 OR MIT-0",
convert_case: 0.6.0, "MIT",
convert_case: 0.10.0, "MIT",
cookie: 0.16.2, "Apache-2.0 OR MIT",
@@ -217,7 +216,7 @@ darling_macro: 0.21.3, "MIT",
darling_macro: 0.23.0, "MIT",
dary_heap: 0.3.8, "Apache-2.0 OR MIT",
dashmap: 6.1.0, "MIT",
-data-encoding: 2.9.0, "MIT",
+data-encoding: 2.10.0, "MIT",
dbus: 0.9.10, "Apache-2.0 OR MIT",
dbus-secret-service: 4.1.0, "Apache-2.0 OR MIT",
deflate64: 0.1.10, "MIT",
@@ -278,18 +277,18 @@ ext-trait: 1.0.1, "Apache-2.0 OR MIT OR Zlib",
ext-trait-proc_macros: 1.0.1, "Apache-2.0 OR MIT OR Zlib",
extension-traits: 1.0.1, "Apache-2.0 OR MIT OR Zlib",
fast-async-mutex: 0.6.7, "Apache-2.0 OR MIT",
-fastbloom: 0.14.0, "Apache-2.0 OR MIT",
+fastbloom: 0.14.1, "Apache-2.0 OR MIT",
fastrand: 2.3.0, "Apache-2.0 OR MIT",
-ferroid: 0.8.8, "Apache-2.0 OR MIT",
+ferroid: 0.8.9, "Apache-2.0 OR MIT",
ff: 0.13.1, "Apache-2.0 OR MIT",
fiat-crypto: 0.2.9, "Apache-2.0 OR BSD-1-Clause OR MIT",
figlet-rs: 0.1.5, "Apache-2.0",
figment: 0.10.19, "Apache-2.0 OR MIT",
-file-operation: 0.8.8, "MIT",
+file-operation: 0.8.11, "MIT",
filetime: 0.2.26, "Apache-2.0 OR MIT",
-find-msvc-tools: 0.1.6, "Apache-2.0 OR MIT",
+find-msvc-tools: 0.1.8, "Apache-2.0 OR MIT",
flatbuffers: 25.12.19, "Apache-2.0",
-flate2: 1.1.5, "Apache-2.0 OR MIT",
+flate2: 1.1.8, "Apache-2.0 OR MIT",
float-cmp: 0.10.0, "MIT",
flume: 0.11.1, "Apache-2.0 OR MIT",
flume: 0.12.0, "Apache-2.0 OR MIT",
@@ -319,7 +318,7 @@ futures-timer: 3.0.3, "Apache-2.0 OR MIT",
futures-util: 0.3.31, "Apache-2.0 OR MIT",
generator: 0.8.8, "Apache-2.0 OR MIT",
generic-array: 0.14.7, "MIT",
-getrandom: 0.2.16, "Apache-2.0 OR MIT",
+getrandom: 0.2.17, "Apache-2.0 OR MIT",
getrandom: 0.3.4, "Apache-2.0 OR MIT",
ghash: 0.5.1, "Apache-2.0 OR MIT",
gherkin: 0.15.0, "Apache-2.0 OR MIT",
@@ -342,10 +341,10 @@ gloo-worker-macros: 0.1.0, "Apache-2.0 OR MIT",
governor: 0.10.4, "MIT",
group: 0.13.0, "Apache-2.0 OR MIT",
h2: 0.3.27, "MIT",
-h2: 0.4.12, "MIT",
+h2: 0.4.13, "MIT",
half: 2.7.1, "Apache-2.0 OR MIT",
halfbrown: 0.4.0, "Apache-2.0 OR MIT",
-handlebars: 6.3.2, "MIT",
+handlebars: 6.4.0, "MIT",
hash32: 0.2.1, "Apache-2.0 OR MIT",
hash32: 1.0.0, "Apache-2.0 OR MIT",
hashbrown: 0.12.3, "Apache-2.0 OR MIT",
@@ -415,7 +414,7 @@ impl-more: 0.1.9, "Apache-2.0 OR MIT",
implicit-clone: 0.6.0, "Apache-2.0 OR MIT",
implicit-clone-derive: 0.1.2, "Apache-2.0 OR MIT",
indexmap: 1.9.3, "Apache-2.0 OR MIT",
-indexmap: 2.12.1, "Apache-2.0 OR MIT",
+indexmap: 2.13.0, "Apache-2.0 OR MIT",
inflections: 1.1.1, "MIT",
inlinable_string: 0.1.15, "Apache-2.0 OR MIT",
inotify: 0.11.0, "ISC",
@@ -432,13 +431,13 @@ is_terminal_polyfill: 1.70.2, "Apache-2.0 OR MIT",
itertools: 0.13.0, "Apache-2.0 OR MIT",
itertools: 0.14.0, "Apache-2.0 OR MIT",
itoa: 1.0.17, "Apache-2.0 OR MIT",
-jiff: 0.2.17, "MIT OR Unlicense",
-jiff-static: 0.2.17, "MIT OR Unlicense",
+jiff: 0.2.18, "MIT OR Unlicense",
+jiff-static: 0.2.18, "MIT OR Unlicense",
jni: 0.21.1, "Apache-2.0 OR MIT",
jni-sys: 0.3.0, "Apache-2.0 OR MIT",
jobserver: 0.1.34, "Apache-2.0 OR MIT",
journal: 0.1.0, "Apache-2.0",
-js-sys: 0.3.83, "Apache-2.0 OR MIT",
+js-sys: 0.3.85, "Apache-2.0 OR MIT",
jsonwebtoken: 10.2.0, "MIT",
jwalk: 0.8.1, "MIT",
keccak: 0.1.5, "Apache-2.0 OR MIT",
@@ -457,18 +456,15 @@ lexical-util: 1.0.7, "Apache-2.0 OR MIT",
lexical-write-float: 1.0.6, "Apache-2.0 OR MIT",
lexical-write-integer: 1.0.6, "Apache-2.0 OR MIT",
libbz2-rs-sys: 0.2.2, "bzip2-1.0.6",
-libc: 0.2.178, "Apache-2.0 OR MIT",
+libc: 0.2.180, "Apache-2.0 OR MIT",
libdbus-sys: 0.2.7, "Apache-2.0 OR MIT",
libflate: 2.2.1, "MIT",
libflate_lz77: 2.2.0, "MIT",
libgit2-sys: 0.18.3+1.9.2, "Apache-2.0 OR MIT",
-liblzma: 0.4.5, "Apache-2.0 OR MIT",
-liblzma-sys: 0.4.4, "Apache-2.0 OR MIT",
libm: 0.2.15, "MIT",
libmimalloc-sys: 0.1.44, "MIT",
libredox: 0.1.12, "MIT",
libsqlite3-sys: 0.30.1, "MIT",
-libz-rs-sys: 0.5.5, "Zlib",
libz-sys: 1.1.23, "Apache-2.0 OR MIT",
linked-hash-map: 0.5.6, "Apache-2.0 OR MIT",
linux-raw-sys: 0.11.0, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
@@ -485,6 +481,7 @@ loom: 0.7.2, "MIT",
lru-slab: 0.1.2, "Apache-2.0 OR MIT OR Zlib",
lz4_flex: 0.11.5, "MIT",
lzma-rust2: 0.15.4, "Apache-2.0",
+lzma-sys: 0.1.20, "Apache-2.0 OR MIT",
macro_rules_attribute: 0.1.3, "MIT",
macro_rules_attribute-proc_macro: 0.1.3, "MIT",
matchers: 0.2.0, "MIT",
@@ -536,7 +533,7 @@ objc2: 0.6.3, "MIT",
objc2-core-foundation: 0.3.2, "Apache-2.0 OR MIT OR Zlib",
objc2-encode: 4.1.0, "MIT",
objc2-io-kit: 0.3.2, "Apache-2.0 OR MIT OR Zlib",
-octocrab: 0.49.4, "Apache-2.0 OR MIT",
+octocrab: 0.49.5, "Apache-2.0 OR MIT",
oid-registry: 0.8.1, "Apache-2.0 OR MIT",
once_cell: 1.21.3, "Apache-2.0 OR MIT",
once_cell_polyfill: 1.70.2, "Apache-2.0 OR MIT",
@@ -582,10 +579,10 @@ peg-runtime: 0.6.3, "MIT",
pem: 3.0.6, "MIT",
pem-rfc7468: 0.7.0, "Apache-2.0 OR MIT",
percent-encoding: 2.3.2, "Apache-2.0 OR MIT",
-pest: 2.8.4, "Apache-2.0 OR MIT",
-pest_derive: 2.8.4, "Apache-2.0 OR MIT",
-pest_generator: 2.8.4, "Apache-2.0 OR MIT",
-pest_meta: 2.8.4, "Apache-2.0 OR MIT",
+pest: 2.8.5, "Apache-2.0 OR MIT",
+pest_derive: 2.8.5, "Apache-2.0 OR MIT",
+pest_generator: 2.8.5, "Apache-2.0 OR MIT",
+pest_meta: 2.8.5, "Apache-2.0 OR MIT",
pin-project: 1.1.10, "Apache-2.0 OR MIT",
pin-project-internal: 1.1.10, "Apache-2.0 OR MIT",
pin-project-lite: 0.2.16, "Apache-2.0 OR MIT",
@@ -613,14 +610,14 @@ proc-macro-crate: 1.3.1, "Apache-2.0 OR MIT",
proc-macro-crate: 3.4.0, "Apache-2.0 OR MIT",
proc-macro-error: 1.0.4, "Apache-2.0 OR MIT",
proc-macro-error-attr: 1.0.4, "Apache-2.0 OR MIT",
-proc-macro2: 1.0.104, "Apache-2.0 OR MIT",
+proc-macro2: 1.0.105, "Apache-2.0 OR MIT",
proc-macro2-diagnostics: 0.10.1, "Apache-2.0 OR MIT",
prometheus-client: 0.24.0, "Apache-2.0 OR MIT",
prometheus-client-derive-encode: 0.5.0, "Apache-2.0 OR MIT",
-prost: 0.14.1, "Apache-2.0",
-prost-derive: 0.14.1, "Apache-2.0",
+prost: 0.14.3, "Apache-2.0",
+prost-derive: 0.14.3, "Apache-2.0",
prost-reflect: 0.16.3, "Apache-2.0 OR MIT",
-prost-types: 0.14.1, "Apache-2.0",
+prost-types: 0.14.3, "Apache-2.0",
protox: 0.9.1, "Apache-2.0 OR MIT",
protox-parse: 0.9.0, "Apache-2.0 OR MIT",
ptr_meta: 0.1.4, "MIT",
@@ -632,7 +629,7 @@ quick-xml: 0.38.4, "MIT",
quinn: 0.11.9, "Apache-2.0 OR MIT",
quinn-proto: 0.11.13, "Apache-2.0 OR MIT",
quinn-udp: 0.5.14, "Apache-2.0 OR MIT",
-quote: 1.0.42, "Apache-2.0 OR MIT",
+quote: 1.0.43, "Apache-2.0 OR MIT",
r-efi: 5.3.0, "Apache-2.0 OR LGPL-2.1-or-later OR MIT",
radium: 0.7.0, "MIT",
rand: 0.8.5, "Apache-2.0 OR MIT",
@@ -640,7 +637,7 @@ rand: 0.9.2, "Apache-2.0 OR MIT",
rand_chacha: 0.3.1, "Apache-2.0 OR MIT",
rand_chacha: 0.9.0, "Apache-2.0 OR MIT",
rand_core: 0.6.4, "Apache-2.0 OR MIT",
-rand_core: 0.9.3, "Apache-2.0 OR MIT",
+rand_core: 0.9.5, "Apache-2.0 OR MIT",
rand_xoshiro: 0.7.0, "Apache-2.0 OR MIT",
raw-cpuid: 11.6.0, "MIT",
rayon: 1.11.0, "Apache-2.0 OR MIT",
@@ -665,32 +662,32 @@ retry-policies: 0.5.1, "Apache-2.0 OR MIT",
rfc6979: 0.4.0, "Apache-2.0 OR MIT",
ring: 0.17.14, "Apache-2.0 AND ISC",
ringbuffer: 0.16.0, "MIT",
-rkyv: 0.7.45, "MIT",
-rkyv_derive: 0.7.45, "MIT",
+rkyv: 0.7.46, "MIT",
+rkyv_derive: 0.7.46, "MIT",
rle-decode-fast: 1.0.3, "Apache-2.0 OR MIT",
-rmcp: 0.12.0, "MIT",
-rmcp-macros: 0.12.0, "MIT",
+rmcp: 0.13.0, "Apache-2.0",
+rmcp-macros: 0.13.0, "Apache-2.0",
rmp: 0.8.15, "MIT",
rmp-serde: 1.3.1, "MIT",
roaring: 0.10.12, "Apache-2.0 OR MIT",
route-recognizer: 0.3.1, "MIT",
-rsa: 0.9.9, "Apache-2.0 OR MIT",
-rust-embed: 8.9.0, "MIT",
-rust-embed-impl: 8.9.0, "MIT",
-rust-embed-utils: 8.9.0, "MIT",
+rsa: 0.9.10, "Apache-2.0 OR MIT",
+rust-embed: 8.11.0, "MIT",
+rust-embed-impl: 8.11.0, "MIT",
+rust-embed-utils: 8.11.0, "MIT",
rust-ini: 0.21.3, "MIT",
-rust_decimal: 1.39.0, "MIT",
+rust_decimal: 1.40.0, "MIT",
rustc-hash: 2.1.1, "Apache-2.0 OR MIT",
rustc_version: 0.4.1, "Apache-2.0 OR MIT",
rusticata-macros: 4.1.0, "Apache-2.0 OR MIT",
rustix: 1.1.3, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
-rustls: 0.23.35, "Apache-2.0 OR ISC OR MIT",
+rustls: 0.23.36, "Apache-2.0 OR ISC OR MIT",
rustls-native-certs: 0.8.3, "Apache-2.0 OR ISC OR MIT",
rustls-pemfile: 2.2.0, "Apache-2.0 OR ISC OR MIT",
-rustls-pki-types: 1.13.2, "Apache-2.0 OR MIT",
+rustls-pki-types: 1.14.0, "Apache-2.0 OR MIT",
rustls-platform-verifier: 0.6.2, "Apache-2.0 OR MIT",
rustls-platform-verifier-android: 0.1.1, "Apache-2.0 OR MIT",
-rustls-webpki: 0.103.8, "ISC",
+rustls-webpki: 0.103.9, "ISC",
rustversion: 1.0.22, "Apache-2.0 OR MIT",
ryu: 1.0.22, "Apache-2.0 OR BSL-1.0",
same-file: 1.0.6, "MIT OR Unlicense",
@@ -718,7 +715,7 @@ serde_bytes: 0.11.19, "Apache-2.0 OR MIT",
serde_core: 1.0.228, "Apache-2.0 OR MIT",
serde_derive: 1.0.228, "Apache-2.0 OR MIT",
serde_derive_internals: 0.29.1, "Apache-2.0 OR MIT",
-serde_json: 1.0.148, "Apache-2.0 OR MIT",
+serde_json: 1.0.149, "Apache-2.0 OR MIT",
serde_path_to_error: 0.1.20, "Apache-2.0 OR MIT",
serde_repr: 0.1.20, "Apache-2.0 OR MIT",
serde_spanned: 0.6.9, "Apache-2.0 OR MIT",
@@ -727,8 +724,8 @@ serde_urlencoded: 0.7.1, "Apache-2.0 OR MIT",
serde_with: 3.16.1, "Apache-2.0 OR MIT",
serde_with_macros: 3.16.1, "Apache-2.0 OR MIT",
serde_yaml_ng: 0.10.0, "MIT",
-serial_test: 3.2.0, "MIT",
-serial_test_derive: 3.2.0, "MIT",
+serial_test: 3.3.1, "MIT",
+serial_test_derive: 3.3.1, "MIT",
server: 0.6.1-edge.4, "Apache-2.0",
sha1: 0.10.6, "Apache-2.0 OR MIT",
sha2: 0.10.9, "Apache-2.0 OR MIT",
@@ -774,13 +771,12 @@ strum_macros: 0.26.4, "MIT",
strum_macros: 0.27.2, "MIT",
subtle: 2.6.1, "BSD-3-Clause",
syn: 1.0.109, "Apache-2.0 OR MIT",
-syn: 2.0.111, "Apache-2.0 OR MIT",
+syn: 2.0.114, "Apache-2.0 OR MIT",
sync_wrapper: 1.0.2, "Apache-2.0",
synstructure: 0.13.2, "MIT",
synthez: 0.4.0, "BlueOak-1.0.0",
synthez-codegen: 0.4.0, "BlueOak-1.0.0",
synthez-core: 0.4.0, "BlueOak-1.0.0",
-sysinfo: 0.34.2, "MIT",
sysinfo: 0.37.2, "MIT",
tagptr: 0.2.0, "Apache-2.0 OR MIT",
tap: 1.0.1, "MIT",
@@ -801,22 +797,22 @@ thiserror-impl: 2.0.17, "Apache-2.0 OR MIT",
thread_local: 1.1.9, "Apache-2.0 OR MIT",
threadpool: 1.8.1, "Apache-2.0 OR MIT",
thrift: 0.17.0, "Apache-2.0",
-time: 0.3.44, "Apache-2.0 OR MIT",
-time-core: 0.1.6, "Apache-2.0 OR MIT",
-time-macros: 0.2.24, "Apache-2.0 OR MIT",
+time: 0.3.45, "Apache-2.0 OR MIT",
+time-core: 0.1.7, "Apache-2.0 OR MIT",
+time-macros: 0.2.25, "Apache-2.0 OR MIT",
tiny-keccak: 2.0.2, "CC0-1.0",
tinystr: 0.8.2, "Unicode-3.0",
tinyvec: 1.10.0, "Apache-2.0 OR MIT OR Zlib",
tinyvec_macros: 0.1.1, "Apache-2.0 OR MIT OR Zlib",
-tokio: 1.48.0, "MIT",
+tokio: 1.49.0, "MIT",
tokio-macros: 2.6.0, "MIT",
tokio-rustls: 0.26.4, "Apache-2.0 OR MIT",
-tokio-stream: 0.1.17, "MIT",
+tokio-stream: 0.1.18, "MIT",
tokio-tungstenite: 0.28.0, "MIT",
-tokio-util: 0.7.17, "MIT",
+tokio-util: 0.7.18, "MIT",
tokise: 0.2.0, "Apache-2.0 OR MIT",
toml: 0.8.23, "Apache-2.0 OR MIT",
-toml: 0.9.10+spec-1.1.0, "Apache-2.0 OR MIT",
+toml: 0.9.11+spec-1.1.0, "Apache-2.0 OR MIT",
toml_datetime: 0.6.11, "Apache-2.0 OR MIT",
toml_datetime: 0.7.5+spec-1.1.0, "Apache-2.0 OR MIT",
toml_edit: 0.19.15, "Apache-2.0 OR MIT",
@@ -828,7 +824,7 @@ toml_writer: 1.0.6+spec-1.1.0, "Apache-2.0 OR MIT",
tonic: 0.14.2, "MIT",
tonic-prost: 0.14.2, "MIT",
tools: 0.1.0, "Apache-2.0",
-tower: 0.5.2, "MIT",
+tower: 0.5.3, "MIT",
tower-http: 0.6.8, "MIT",
tower-layer: 0.3.3, "MIT",
tower-service: 0.3.3, "MIT",
@@ -837,7 +833,7 @@ tracing-appender: 0.2.4, "MIT",
tracing-attributes: 0.1.31, "MIT",
tracing-core: 0.1.36, "MIT",
tracing-log: 0.2.0, "MIT",
-tracing-opentelemetry: 0.32.0, "MIT",
+tracing-opentelemetry: 0.32.1, "MIT",
tracing-subscriber: 0.3.22, "MIT",
trait-variant: 0.1.2, "Apache-2.0 OR MIT",
try-lock: 0.2.5, "MIT",
@@ -853,7 +849,7 @@ typenum: 1.19.0, "Apache-2.0 OR MIT",
ucd-trie: 0.1.7, "Apache-2.0 OR MIT",
ulid: 1.2.1, "MIT",
uncased: 0.9.10, "Apache-2.0 OR MIT",
-unicase: 2.8.1, "Apache-2.0 OR MIT",
+unicase: 2.9.0, "Apache-2.0 OR MIT",
unicode-bidi: 0.3.18, "Apache-2.0 OR MIT",
unicode-ident: 1.0.22, "(Apache-2.0 OR MIT) AND Unicode-3.0",
unicode-linebreak: 0.1.5, "Apache-2.0",
@@ -868,7 +864,7 @@ unsafe-libyaml: 0.2.11, "MIT",
untrusted: 0.9.0, "ISC",
ureq: 3.1.4, "Apache-2.0 OR MIT",
ureq-proto: 0.5.3, "Apache-2.0 OR MIT",
-url: 2.5.7, "Apache-2.0 OR MIT",
+url: 2.5.8, "Apache-2.0 OR MIT",
urlencoding: 2.1.3, "MIT",
utf-8: 0.7.6, "Apache-2.0 OR MIT",
utf8-width: 0.1.8, "MIT",
@@ -879,50 +875,45 @@ v_htmlescape: 0.15.8, "Apache-2.0 OR MIT",
valuable: 0.1.1, "MIT",
value-trait: 0.12.1, "Apache-2.0 OR MIT",
vcpkg: 0.2.15, "Apache-2.0 OR MIT",
-vergen: 9.0.6, "Apache-2.0 OR MIT",
-vergen-git2: 1.0.7, "Apache-2.0 OR MIT",
-vergen-lib: 0.1.6, "Apache-2.0 OR MIT",
+vergen: 9.1.0, "Apache-2.0 OR MIT",
+vergen-git2: 9.1.0, "Apache-2.0 OR MIT",
+vergen-lib: 9.1.0, "Apache-2.0 OR MIT",
version_check: 0.9.5, "Apache-2.0 OR MIT",
void: 1.0.2, "MIT",
wait-timeout: 0.2.1, "Apache-2.0 OR MIT",
walkdir: 2.5.0, "MIT OR Unlicense",
want: 0.3.1, "MIT",
wasi: 0.11.1+wasi-snapshot-preview1, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
-wasip2: 1.0.1+wasi-0.2.4, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
+wasip2: 1.0.2+wasi-0.2.9, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
wasite: 0.1.0, "Apache-2.0 OR BSL-1.0 OR MIT",
-wasm-bindgen: 0.2.106, "Apache-2.0 OR MIT",
-wasm-bindgen-futures: 0.4.56, "Apache-2.0 OR MIT",
-wasm-bindgen-macro: 0.2.106, "Apache-2.0 OR MIT",
-wasm-bindgen-macro-support: 0.2.106, "Apache-2.0 OR MIT",
-wasm-bindgen-shared: 0.2.106, "Apache-2.0 OR MIT",
+wasm-bindgen: 0.2.108, "Apache-2.0 OR MIT",
+wasm-bindgen-futures: 0.4.58, "Apache-2.0 OR MIT",
+wasm-bindgen-macro: 0.2.108, "Apache-2.0 OR MIT",
+wasm-bindgen-macro-support: 0.2.108, "Apache-2.0 OR MIT",
+wasm-bindgen-shared: 0.2.108, "Apache-2.0 OR MIT",
wasm-streams: 0.4.2, "Apache-2.0 OR MIT",
wasmtimer: 0.4.3, "MIT",
-web-sys: 0.3.83, "Apache-2.0 OR MIT",
+web-sys: 0.3.85, "Apache-2.0 OR MIT",
web-time: 1.1.0, "Apache-2.0 OR MIT",
-webpki-root-certs: 1.0.4, "CDLA-Permissive-2.0",
+webpki-root-certs: 1.0.5, "CDLA-Permissive-2.0",
webpki-roots: 0.26.11, "CDLA-Permissive-2.0",
-webpki-roots: 1.0.4, "CDLA-Permissive-2.0",
+webpki-roots: 1.0.5, "CDLA-Permissive-2.0",
whoami: 1.6.1, "Apache-2.0 OR BSL-1.0 OR MIT",
widestring: 1.2.1, "Apache-2.0 OR MIT",
winapi: 0.3.9, "Apache-2.0 OR MIT",
winapi-i686-pc-windows-gnu: 0.4.0, "Apache-2.0 OR MIT",
winapi-util: 0.1.11, "MIT OR Unlicense",
winapi-x86_64-pc-windows-gnu: 0.4.0, "Apache-2.0 OR MIT",
-windows: 0.57.0, "Apache-2.0 OR MIT",
windows: 0.61.3, "Apache-2.0 OR MIT",
windows-collections: 0.2.0, "Apache-2.0 OR MIT",
-windows-core: 0.57.0, "Apache-2.0 OR MIT",
windows-core: 0.61.2, "Apache-2.0 OR MIT",
windows-core: 0.62.2, "Apache-2.0 OR MIT",
windows-future: 0.2.1, "Apache-2.0 OR MIT",
-windows-implement: 0.57.0, "Apache-2.0 OR MIT",
windows-implement: 0.60.2, "Apache-2.0 OR MIT",
-windows-interface: 0.57.0, "Apache-2.0 OR MIT",
windows-interface: 0.59.3, "Apache-2.0 OR MIT",
windows-link: 0.1.3, "Apache-2.0 OR MIT",
windows-link: 0.2.1, "Apache-2.0 OR MIT",
windows-numerics: 0.2.0, "Apache-2.0 OR MIT",
-windows-result: 0.1.2, "Apache-2.0 OR MIT",
windows-result: 0.3.4, "Apache-2.0 OR MIT",
windows-result: 0.4.1, "Apache-2.0 OR MIT",
windows-strings: 0.4.2, "Apache-2.0 OR MIT",
@@ -970,11 +961,12 @@ windows_x86_64_msvc: 0.52.6, "Apache-2.0 OR MIT",
windows_x86_64_msvc: 0.53.1, "Apache-2.0 OR MIT",
winnow: 0.5.40, "MIT",
winnow: 0.7.14, "MIT",
-wit-bindgen: 0.46.0, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
+wit-bindgen: 0.51.0, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
writeable: 0.6.2, "Unicode-3.0",
wyz: 0.5.1, "MIT",
x509-parser: 0.18.0, "Apache-2.0 OR MIT",
xattr: 1.6.1, "Apache-2.0 OR MIT",
+xz2: 0.1.7, "Apache-2.0 OR MIT",
yansi: 1.0.1, "Apache-2.0 OR MIT",
yasna: 0.5.2, "Apache-2.0 OR MIT",
yew: 0.22.0, "Apache-2.0 OR MIT",
@@ -983,8 +975,8 @@ yew-router: 0.19.0, "Apache-2.0 OR MIT",
yew-router-macro: 0.19.0, "Apache-2.0 OR MIT",
yoke: 0.8.1, "Unicode-3.0",
yoke-derive: 0.8.1, "Unicode-3.0",
-zerocopy: 0.8.31, "Apache-2.0 OR BSD-2-Clause OR MIT",
-zerocopy-derive: 0.8.31, "Apache-2.0 OR BSD-2-Clause OR MIT",
+zerocopy: 0.8.33, "Apache-2.0 OR BSD-2-Clause OR MIT",
+zerocopy-derive: 0.8.33, "Apache-2.0 OR BSD-2-Clause OR MIT",
zerofrom: 0.1.6, "Unicode-3.0",
zerofrom-derive: 0.1.6, "Unicode-3.0",
zeroize: 1.8.2, "Apache-2.0 OR MIT",
@@ -994,7 +986,7 @@ zerovec: 0.11.5, "Unicode-3.0",
zerovec-derive: 0.11.2, "Unicode-3.0",
zip: 7.0.0, "MIT",
zlib-rs: 0.5.5, "Zlib",
-zmij: 1.0.3, "MIT",
+zmij: 1.0.14, "MIT",
zopfli: 0.8.3, "Apache-2.0",
zstd: 0.13.3, "MIT",
zstd-safe: 7.2.4, "Apache-2.0 OR MIT",
diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml
index 7f1f56380..ca729329d 100644
--- a/core/ai/mcp/Cargo.toml
+++ b/core/ai/mcp/Cargo.toml
@@ -35,7 +35,7 @@ figlet-rs = { workspace = true }
figment = { workspace = true }
iggy = { workspace = true }
iggy_common = { workspace = true }
-rmcp = { version = "0.12.0", features = [
+rmcp = { version = "0.13.0", features = [
"server",
"transport-io",
"transport-streamable-http-server",
diff --git a/core/bench/dashboard/server/Cargo.toml b/core/bench/dashboard/server/Cargo.toml
index 63c3449c3..6303a663a 100644
--- a/core/bench/dashboard/server/Cargo.toml
+++ b/core/bench/dashboard/server/Cargo.toml
@@ -30,9 +30,9 @@ bench-report = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
clap = { workspace = true }
dashmap = { workspace = true }
-file-operation = "0.8.8"
+file-operation = "0.8.11"
notify = "8.2.0"
-octocrab = "0.49.4"
+octocrab = "0.49.5"
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
tempfile = { workspace = true }
diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml
index 22b728ab2..98fd9b7f1 100644
--- a/core/cli/Cargo.toml
+++ b/core/cli/Cargo.toml
@@ -43,7 +43,7 @@ login-session = ["dep:keyring"]
ahash = { workspace = true }
anyhow = { workspace = true }
clap = { workspace = true }
-clap_complete = "4.5.63"
+clap_complete = "4.5.65"
figlet-rs = { workspace = true }
iggy = { workspace = true }
iggy_binary_protocol = { workspace = true }
diff --git a/core/common/src/configs/mod.rs b/core/common/src/configs/mod.rs
index bf578b38c..fc440820f 100644
--- a/core/common/src/configs/mod.rs
+++ b/core/common/src/configs/mod.rs
@@ -157,7 +157,8 @@ impl<T: ConfigurationType> CustomEnvProvider<T> {
.map(|k| k.to_lowercase())
.collect();
- let env_var_value = Self::parse_environment_value(&value);
+ let expected_type = Self::find_expected_type(&source_dict, &keys);
+ let env_var_value = Self::parse_environment_value_typed(&value, expected_type);
if self.secret_keys.contains(&env_key) {
value = SECRET_MASK.to_string();
@@ -686,45 +687,131 @@ impl<T: ConfigurationType> CustomEnvProvider<T> {
}
}
- /// Parse environment variable value with type inference
- fn parse_environment_value(value: &str) -> FigmentValue {
- // Handle array syntax
+ fn parse_environment_value_typed(
+ value: &str,
+ expected_type: Option<&FigmentValue>,
+ ) -> FigmentValue {
if value.starts_with('[') && value.ends_with(']') {
let inner_value = value.trim_start_matches('[').trim_end_matches(']');
- let values: Vec<FigmentValue> = inner_value
- .split(',')
- .map(|s| Self::parse_environment_value(s.trim()))
+ let element_type = match expected_type {
+ Some(FigmentValue::Array(_, arr)) => arr.first(),
+ _ => None,
+ };
+ let values: Vec<FigmentValue> = split_array_elements(inner_value)
+ .into_iter()
+ .map(|s| Self::parse_environment_value_typed(strip_quotes(s), element_type))
.collect();
return FigmentValue::from(values);
}
- // Handle boolean values
+ if let Some(FigmentValue::String(..)) = expected_type {
+ return FigmentValue::from(value);
+ }
+
match value.to_lowercase().as_str() {
"true" => return FigmentValue::from(true),
"false" => return FigmentValue::from(false),
_ => {}
}
- // Try u64 first for most numeric values (ports, byte sizes, etc.)
if let Ok(uint_val) = value.parse::<u64>() {
return FigmentValue::from(uint_val);
}
- // Fall back to i64 for signed integers
if let Ok(int_val) = value.parse::<i64>() {
return FigmentValue::from(int_val);
}
- // Try parsing as float
if let Ok(float_val) = value.parse::<f64>() {
return FigmentValue::from(float_val);
}
- // Default to string
FigmentValue::from(value)
}
+
+ fn find_expected_type<'a>(source: &'a Dict, keys: &[String]) -> Option<&'a FigmentValue> {
+ if keys.is_empty() {
+ return None;
+ }
+
+ let mut current = source;
+ let mut combined_keys = Vec::new();
+
+ for (i, key) in keys.iter().enumerate() {
+ combined_keys.push(key.clone());
+ let key_to_check = combined_keys.join("_");
+
+ match current.get(&key_to_check) {
+ Some(FigmentValue::Dict(_, inner)) => {
+ current = inner;
+ combined_keys.clear();
+ }
+ Some(value) if i == keys.len() - 1 => {
+ return Some(value);
+ }
+ _ => continue,
+ }
+ }
+
+ let final_key = combined_keys.join("_");
+ current.get(&final_key)
+ }
+}
+
+fn split_array_elements(s: &str) -> Vec<&str> {
+ let mut elements = Vec::new();
+ let mut start = 0;
+ let mut in_quotes = false;
+ for (i, c) in s.char_indices() {
+ match c {
+ '"' => in_quotes = !in_quotes,
+ ',' if !in_quotes => {
+ elements.push(s[start..i].trim());
+ start = i + 1;
+ }
+ _ => {}
+ }
+ }
+ if start < s.len() {
+ elements.push(s[start..].trim());
+ }
+ elements
+}
+
+fn strip_quotes(s: &str) -> &str {
+ s.trim_matches('"')
+}
+
+pub fn parse_env_value_to_json(value: &str) -> serde_json::Value {
+ if value.starts_with('[') && value.ends_with(']') {
+ let inner = value.trim_start_matches('[').trim_end_matches(']');
+ let values: Vec<serde_json::Value> = split_array_elements(inner)
+ .into_iter()
+ .map(|s| parse_env_value_to_json(strip_quotes(s)))
+ .collect();
+ return serde_json::Value::Array(values);
+ }
+
+ match value.to_lowercase().as_str() {
+ "true" => return serde_json::Value::Bool(true),
+ "false" => return serde_json::Value::Bool(false),
+ _ => {}
+ }
+
+ if let Ok(n) = value.parse::<u64>() {
+ return serde_json::Value::Number(n.into());
+ }
+ if let Ok(n) = value.parse::<i64>() {
+ return serde_json::Value::Number(n.into());
+ }
+ if let Ok(n) = value.parse::<f64>()
+ && let Some(num) = serde_json::Number::from_f64(n)
+ {
+ return serde_json::Value::Number(num);
+ }
+
+ serde_json::Value::String(value.to_owned())
}
-/// Check if a file exists using the same logic as Figment
fn file_exists<P: AsRef<Path>>(path: P) -> bool {
let path = path.as_ref();
diff --git a/core/connectors/runtime/src/configs/connectors.rs b/core/connectors/runtime/src/configs/connectors.rs
index 6bacdc7e4..14dd0bdf8 100644
--- a/core/connectors/runtime/src/configs/connectors.rs
+++ b/core/connectors/runtime/src/configs/connectors.rs
@@ -80,6 +80,8 @@ pub struct CreateSinkConfig {
pub streams: Vec<StreamConsumerConfig>,
pub plugin_config_format: Option<ConfigFormat>,
pub plugin_config: Option<serde_json::Value>,
+ #[serde(default)]
+ pub verbose: bool,
}
impl CreateSinkConfig {
@@ -94,6 +96,7 @@ impl CreateSinkConfig {
streams: self.streams.clone(),
plugin_config_format: self.plugin_config_format,
plugin_config: self.plugin_config.clone(),
+ verbose: self.verbose,
}
}
}
@@ -109,6 +112,8 @@ pub struct SinkConfig {
pub streams: Vec<StreamConsumerConfig>,
pub plugin_config_format: Option<ConfigFormat>,
pub plugin_config: Option<serde_json::Value>,
+ #[serde(default)]
+ pub verbose: bool,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -120,6 +125,8 @@ pub struct CreateSourceConfig {
pub streams: Vec<StreamProducerConfig>,
pub plugin_config_format: Option<ConfigFormat>,
pub plugin_config: Option<serde_json::Value>,
+ #[serde(default)]
+ pub verbose: bool,
}
impl CreateSourceConfig {
@@ -134,6 +141,7 @@ impl CreateSourceConfig {
streams: self.streams.clone(),
plugin_config_format: self.plugin_config_format,
plugin_config: self.plugin_config.clone(),
+ verbose: self.verbose,
}
}
}
@@ -149,6 +157,8 @@ pub struct SourceConfig {
pub streams: Vec<StreamProducerConfig>,
pub plugin_config_format: Option<ConfigFormat>,
pub plugin_config: Option<serde_json::Value>,
+ #[serde(default)]
+ pub verbose: bool,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
diff --git a/core/connectors/runtime/src/configs/connectors/local_provider.rs b/core/connectors/runtime/src/configs/connectors/local_provider.rs
index f1c138961..fb363d9c5 100644
--- a/core/connectors/runtime/src/configs/connectors/local_provider.rs
+++ b/core/connectors/runtime/src/configs/connectors/local_provider.rs
@@ -164,24 +164,36 @@ impl LocalConnectorsConfigProvider<Created> {
let sinks: DashMap<ConnectorId, SinkConfigFile> = DashMap::new();
let sources: DashMap<ConnectorId, SourceConfigFile> = DashMap::new();
- info!("Loading connectors configuration from: {}", self.config_dir);
+ let cwd = match std::env::current_dir() {
+ Ok(path) => path.display().to_string(),
+ Err(_) => "unknown".to_string(),
+ };
+ info!(
+ "Loading connectors configuration from: {}, current directory: {cwd}",
+ self.config_dir
+ );
let entries = std::fs::read_dir(&self.config_dir)?;
for entry in entries.flatten() {
let path = entry.path();
if path.is_file() {
if path.extension().and_then(|ext| ext.to_str()) != Some("toml") {
- debug!("Skipping non-TOML file: {:?}", path);
+ debug!("Skipping non-TOML file: {}", path.display());
continue;
}
- if let Some(file_name) = path.file_name().and_then(|n| n.to_str())
- && file_name.starts_with('.')
- {
- debug!("Skipping hidden file: {:?}", path);
- continue;
+ if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) {
+ if file_name.starts_with('.') {
+ debug!("Skipping hidden file: {}", path.display());
+ continue;
+ }
+ let file_name_lower = file_name.to_lowercase();
+ if file_name_lower == "cargo.toml" {
+ debug!("Skipping Cargo.toml: {}", path.display());
+ continue;
+ }
}
- debug!("Loading connector configuration from: {:?}", path);
+ info!("Loading connector configuration from: {}", path.display());
let base_config = Self::read_base_config(&path)?;
debug!("Loaded base configuration: {:?}", base_config);
let path = path
@@ -196,8 +208,14 @@ impl LocalConnectorsConfigProvider<Created> {
let created_at: DateTime<Utc> = entry.metadata()?.created()?.into();
let connector_id: ConnectorId = (&connector_config).into();
- match connector_config.clone() {
- ConnectorConfig::Sink(sink_config) => {
+ let version = connector_config.version();
+
+ match connector_config {
+ ConnectorConfig::Sink(mut sink_config) => {
+ Self::apply_plugin_config_env_overrides(
+ &mut sink_config.plugin_config,
+ &base_config,
+ );
sinks.insert(
connector_id,
SinkConfigFile {
@@ -207,7 +225,11 @@ impl LocalConnectorsConfigProvider<Created> {
},
);
}
- ConnectorConfig::Source(source_config) => {
+ ConnectorConfig::Source(mut source_config) => {
+ Self::apply_plugin_config_env_overrides(
+ &mut source_config.plugin_config,
+ &base_config,
+ );
sources.insert(
connector_id,
SourceConfigFile {
@@ -219,10 +241,10 @@ impl LocalConnectorsConfigProvider<Created> {
}
}
- debug!(
- "Loaded connector configuration with key {}, version {}, created at {}",
+ info!(
+ "Loaded connector configuration with key: {}, version: {}, created at {}",
base_config.key(),
- connector_config.version(),
+ version,
created_at.to_rfc3339()
);
}
@@ -299,6 +321,31 @@ impl<S: ProviderState> LocalConnectorsConfigProvider<S> {
))
})
}
+
+ fn apply_plugin_config_env_overrides(
+ plugin_config: &mut Option<serde_json::Value>,
+ base_config: &BaseConnectorConfig,
+ ) {
+ let connector_type = base_config.connector_type().to_uppercase();
+ let key = base_config.key().to_uppercase();
+ let prefix = format!("IGGY_CONNECTORS_{connector_type}_{key}_PLUGIN_CONFIG_");
+
+ for (env_key, env_value) in std::env::vars() {
+ let env_key_upper = env_key.to_uppercase();
+ if !env_key_upper.starts_with(&prefix) {
+ continue;
+ }
+
+ let field_path = &env_key_upper[prefix.len()..];
+ let field_name = field_path.to_lowercase();
+ let parsed_value = iggy_common::parse_env_value_to_json(&env_value);
+
+ let config = plugin_config.get_or_insert_with(|| serde_json::json!({}));
+ if let serde_json::Value::Object(map) = config {
+ map.insert(field_name, parsed_value);
+ }
+ }
+ }
}
#[derive(Debug, Deserialize)]
diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs
index da0ac4214..d600fc7eb 100644
--- a/core/connectors/runtime/src/main.rs
+++ b/core/connectors/runtime/src/main.rs
@@ -276,6 +276,7 @@ struct SinkConnectorPlugin {
config_format: Option<ConfigFormat>,
consumers: Vec<SinkConnectorConsumer>,
error: Option<String>,
+ verbose: bool,
}
struct SinkConnectorConsumer {
@@ -310,6 +311,7 @@ struct SourceConnectorPlugin {
producer: Option<SourceConnectorProducer>,
state_storage: StateStorage,
error: Option<String>,
+ verbose: bool,
}
struct SourceConnectorProducer {
diff --git a/core/connectors/runtime/src/sink.rs b/core/connectors/runtime/src/sink.rs
index abd04971e..c9d8bd3da 100644
--- a/core/connectors/runtime/src/sink.rs
+++ b/core/connectors/runtime/src/sink.rs
@@ -40,7 +40,7 @@ use std::{
sync::{Arc, atomic::Ordering},
time::Instant,
};
-use tracing::{error, info, warn};
+use tracing::{debug, error, info, warn};
pub async fn init(
sink_configs: HashMap<String, SinkConfig>,
@@ -78,6 +78,7 @@ pub async fn init(
config_format: config.plugin_config_format,
consumers: vec![],
error: init_error.clone(),
+ verbose: config.verbose,
});
} else {
let container: Container<SinkApi> =
@@ -102,6 +103,7 @@ pub async fn init(
config_format: config.plugin_config_format,
consumers: vec![],
error: init_error.clone(),
+ verbose: config.verbose,
}],
},
);
@@ -205,6 +207,7 @@ pub fn consume(sinks: Vec<SinkConnectorWrapper>, context: Arc<RuntimeContext>) {
sink.callback,
consumer.transforms,
consumer.consumer,
+ plugin.verbose,
)
.await
{
@@ -233,6 +236,7 @@ async fn consume_messages(
consume: ConsumeCallback,
transforms: Vec<Arc<dyn Transform>>,
mut consumer: IggyConsumer,
+ verbose: bool,
) -> Result<(), RuntimeError> {
info!("Started consuming messages for sink connector with ID: {plugin_id}");
let batch_size = batch_size as usize;
@@ -263,10 +267,17 @@ async fn consume_messages(
current_offset,
schema: decoder.schema(),
};
- info!(
- "Processing {messages_count} messages for sink connector with ID: {}",
- plugin_id
- );
+ if verbose {
+ info!(
+ "Processing {messages_count} messages for sink connector with ID: {}",
+ plugin_id
+ );
+ } else {
+ debug!(
+ "Processing {messages_count} messages for sink connector with ID: {}",
+ plugin_id
+ );
+ }
let start = Instant::now();
if let Err(error) = process_messages(
plugin_id,
@@ -286,10 +297,17 @@ async fn consume_messages(
}
let elapsed = start.elapsed();
- info!(
- "Consumed {messages_count} messages in {:#?} for sink connector with ID: {plugin_id}",
- elapsed
- );
+ if verbose {
+ info!(
+ "Consumed {messages_count} messages in {:#?} for sink connector with ID: {plugin_id}",
+ elapsed
+ );
+ } else {
+ debug!(
+ "Consumed {messages_count} messages in {:#?} for sink connector with ID: {plugin_id}",
+ elapsed
+ );
+ }
}
info!("Stopped consuming messages for sink connector with ID: {plugin_id}");
Ok(())
diff --git a/core/connectors/runtime/src/source.rs b/core/connectors/runtime/src/source.rs
index 26af0eaff..5d6d88744 100644
--- a/core/connectors/runtime/src/source.rs
+++ b/core/connectors/runtime/src/source.rs
@@ -91,6 +91,7 @@ pub async fn init(
transforms: vec![],
state_storage,
error: init_error.clone(),
+ verbose: config.verbose,
});
} else {
let container: Container<SourceApi> =
@@ -118,6 +119,7 @@ pub async fn init(
transforms: vec![],
state_storage,
error: init_error.clone(),
+ verbose: config.verbose,
}],
},
);
@@ -267,7 +269,11 @@ pub fn handle(sources: Vec<SourceConnectorWrapper>, context: Arc<RuntimeContext>
while let Ok(produced_messages) = receiver.recv_async().await {
let count = produced_messages.messages.len();
- info!("Source connector with ID: {plugin_id} received {count} messages",);
+ if plugin.verbose {
+ info!("Source connector with ID: {plugin_id} received {count} messages",);
+ } else {
+ debug!("Source connector with ID: {plugin_id} received {count} messages",);
+ }
let schema = produced_messages.schema;
let mut messages: Vec<DecodedMessage> = Vec::with_capacity(count);
for message in produced_messages.messages {
@@ -322,11 +328,19 @@ pub fn handle(sources: Vec<SourceConnectorWrapper>, context: Arc<RuntimeContext>
continue;
}
- info!(
- "Sent {count} messages to stream: {}, topic: {} by source connector with ID: {plugin_id}",
- producer.stream(),
- producer.topic()
- );
+ if plugin.verbose {
+ info!(
+ "Sent {count} messages to stream: {}, topic: {} by source connector with ID: {plugin_id}",
+ producer.stream(),
+ producer.topic()
+ );
+ } else {
+ debug!(
+ "Sent {count} messages to stream: {}, topic: {} by source connector with ID: {plugin_id}",
+ producer.stream(),
+ producer.topic()
+ );
+ }
let Some(state) = produced_messages.state else {
debug!("No state provided for source connector with ID: {plugin_id}");
diff --git a/core/connectors/sdk/Cargo.toml b/core/connectors/sdk/Cargo.toml
index 42081e8a1..94c5d6192 100644
--- a/core/connectors/sdk/Cargo.toml
+++ b/core/connectors/sdk/Cargo.toml
@@ -40,8 +40,8 @@ flatbuffers = "25.12.19"
iggy = { workspace = true }
once_cell = { workspace = true }
postcard = { workspace = true }
-prost = "0.14.1"
-prost-types = "0.14.1"
+prost = "0.14.3"
+prost-types = "0.14.3"
protox = "0.9.1"
protox-parse = "0.9.0"
regex = { workspace = true }
diff --git a/core/connectors/sdk/src/lib.rs b/core/connectors/sdk/src/lib.rs
index 98066f33d..b91ade3dc 100644
--- a/core/connectors/sdk/src/lib.rs
+++ b/core/connectors/sdk/src/lib.rs
@@ -324,4 +324,6 @@ pub enum Error {
InvalidState,
#[error("Connection error: {0}")]
Connection(String),
+ #[error("Cannot store data: {0}")]
+ CannotStoreData(String),
}
diff --git a/core/connectors/sinks/README.md b/core/connectors/sinks/README.md
index f29b4b92f..a2782c8e6 100644
--- a/core/connectors/sinks/README.md
+++ b/core/connectors/sinks/README.md
@@ -40,6 +40,7 @@ pub struct SinkConfig {
pub streams: Vec<StreamConsumerConfig>,
pub plugin_config_format: Option<ConfigFormat>,
pub plugin_config: Option<serde_json::Value>,
+ pub verbose: bool, // Log message processing at info level instead of debug (default: false)
}
```
@@ -64,6 +65,7 @@ version = 0
name = "Stdout sink"
path = "target/release/libiggy_connector_stdout_sink"
plugin_config_format = "toml"
+verbose = false # Log message processing at info level instead of debug
# Collection of the streams from which messages are consumed
[[streams]]
diff --git a/core/connectors/sinks/postgres_sink/Cargo.toml b/core/connectors/sinks/postgres_sink/Cargo.toml
index dd68a7ab3..b38de8a85 100644
--- a/core/connectors/sinks/postgres_sink/Cargo.toml
+++ b/core/connectors/sinks/postgres_sink/Cargo.toml
@@ -39,9 +39,11 @@ async-trait = { workspace = true }
chrono = { workspace = true }
dashmap = { workspace = true }
futures = { workspace = true }
+humantime = { workspace = true }
iggy_connector_sdk = { workspace = true }
once_cell = { workspace = true }
serde = { workspace = true }
+serde_json = { workspace = true }
simd-json = { workspace = true }
sqlx = { version = "0.8", features = [
"runtime-tokio-rustls",
diff --git a/core/connectors/sinks/postgres_sink/README.md b/core/connectors/sinks/postgres_sink/README.md
index eb0573ad4..5a4884acb 100644
--- a/core/connectors/sinks/postgres_sink/README.md
+++ b/core/connectors/sinks/postgres_sink/README.md
@@ -1,188 +1,228 @@
# PostgreSQL Sink Connector
-The PostgreSQL sink connector allows you to consume messages from Iggy topics and store them in PostgreSQL databases.
+The PostgreSQL sink connector consumes messages from Iggy topics and stores them in PostgreSQL databases. Supports multiple payload storage formats including BYTEA, JSONB, and TEXT.
## Features
-- **Automatic Table Creation**: Optionally create tables automatically
-- **Batch Processing**: Insert messages in configurable batches for performance
+- **Flexible Payload Storage**: Store payloads as BYTEA (raw bytes), JSONB, or TEXT
+- **Automatic Table Creation**: Optionally create the target table on startup
- **Metadata Storage**: Store Iggy message metadata (offset, timestamp, topic, etc.)
-- **Raw Payload Storage**: Store original message payload as raw bytes
-- **Flexible Data Handling**: Works with any payload type (JSON, text, binary, protobuf, etc.)
+- **Batch Processing**: Insert messages in configurable batches
- **Connection Pooling**: Efficient database connection management
+- **Any Payload Type**: Works with JSON, text, binary, protobuf, or any byte format
## Configuration
-```json
-{
- "connection_string": "postgresql://username:password@localhost:5432/database",
- "target_table": "iggy_messages",
- "batch_size": 100,
- "max_connections": 10,
- "auto_create_table": true,
- "include_metadata": true,
- "include_checksum": true,
- "include_origin_timestamp": true
-}
+```toml
+[[streams]]
+stream = "user_events"
+topics = ["users", "orders"]
+schema = "json"
+batch_length = 100
+poll_interval = "5ms"
+consumer_group = "postgres_sink"
+
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/database"
+target_table = "iggy_messages"
+batch_size = 100
+max_connections = 10
+auto_create_table = true
+include_metadata = true
+include_checksum = true
+include_origin_timestamp = true
+payload_format = "bytea"
```
-### Configuration Options
+## Configuration Options
-- `connection_string`: PostgreSQL connection string
-- `target_table`: Name of the table to insert messages into
-- `batch_size`: Number of messages to insert in each batch (default: 100)
-- `max_connections`: Maximum database connections (default: 10)
-- `auto_create_table`: Automatically create the target table if it doesn't exist (default: false)
-- `include_metadata`: Include Iggy metadata columns (default: true)
-- `include_checksum`: Include message checksum (default: true)
-- `include_origin_timestamp`: Include original message timestamp (default: true)
+| Option | Type | Default | Description |
+| ------ | ---- | ------- | ----------- |
+| `connection_string` | string | required | PostgreSQL connection string |
+| `target_table` | string | required | Target table name |
+| `batch_size` | u32 | `100` | Messages per insert batch |
+| `max_connections` | u32 | `10` | Max database connections |
+| `auto_create_table` | bool | `false` | Create table if not exists |
+| `include_metadata` | bool | `true` | Include Iggy metadata columns |
+| `include_checksum` | bool | `true` | Include message checksum |
+| `include_origin_timestamp` | bool | `true` | Include original timestamp |
+| `payload_format` | string | `bytea` | Payload column type: `bytea`, `json`, or `text` |
+| `verbose_logging` | bool | `false` | Log at info level instead of debug |
+| `max_retries` | u32 | `3` | Max retry attempts for transient errors |
+| `retry_delay` | string | `1s` | Base delay between retries (e.g., `500ms`, `2s`) |
-## Table Schema
+## Payload Format
-When `auto_create_table` is enabled, the following table structure is created:
+The `payload_format` option determines how the payload is stored in PostgreSQL:
-```sql
-CREATE TABLE iggy_messages (
- id DECIMAL(39, 0) PRIMARY KEY,
- iggy_offset BIGINT,
- iggy_timestamp TIMESTAMP WITH TIME ZONE,
- iggy_stream TEXT,
- iggy_topic TEXT,
- iggy_partition_id INTEGER,
- iggy_checksum BIGINT,
- iggy_origin_timestamp TIMESTAMP WITH TIME ZONE,
- payload BYTEA,
- created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
-);
-```
+| Format | Column Type | Description |
+| ------ | ----------- | ----------- |
+| `bytea` | `BYTEA` | Raw bytes (default). Preserves exact binary content. |
+| `json` / `jsonb` | `JSONB` | Native JSON. Enables JSON queries and indexing. Payload must be valid JSON. |
+| `text` | `TEXT` | UTF-8 text. Payload must be valid UTF-8. |
-## Data Storage
+### BYTEA (Default)
-The payload is stored as raw bytes in the `payload` column, regardless of the original format:
+Stores the exact bytes from the Iggy message. Use for binary data, protobuf, or when you want to preserve the original format.
-### JSON Messages
+```toml
+[plugin_config]
+payload_format = "bytea"
+```
-JSON payloads are stored as UTF-8 encoded bytes. You can query them using PostgreSQL's JSON functions:
+### JSONB
-```sql
-SELECT id, payload::text::jsonb->>'user_id' as user_id
-FROM iggy_messages
-WHERE payload::text::jsonb->>'user_id' IS NOT NULL;
-```
+Stores payload as native PostgreSQL JSONB. Enables efficient JSON queries and GIN indexing. The incoming message payload must be valid JSON.
-### Text Messages
+```toml
+[plugin_config]
+payload_format = "json"
+```
-Text payloads are stored as UTF-8 encoded bytes:
+Query example:
```sql
-SELECT id, convert_from(payload, 'UTF8') as message_text
-FROM iggy_messages;
+SELECT id, payload->>'user_id' as user_id
+FROM iggy_messages
+WHERE payload->>'status' = 'active';
+
+CREATE INDEX idx_payload_gin ON iggy_messages USING GIN (payload);
```
-### Binary Messages
+### TEXT
-Binary data is stored directly as bytes:
+Stores payload as UTF-8 text. Use for plain text messages or logs.
-```sql
-SELECT id, encode(payload, 'base64') as payload_base64
-FROM iggy_messages;
+```toml
+[plugin_config]
+payload_format = "text"
```
-### Protocol Buffer Messages
-
-Protobuf messages are stored as raw bytes and can be processed by your application:
+Query example:
```sql
-SELECT id, payload, length(payload) as payload_size
-FROM iggy_messages;
+SELECT id, payload FROM iggy_messages WHERE payload LIKE '%error%';
```
-## Usage Example
+## Table Schema
-1. Configure the sink connector in your Iggy connectors runtime
-2. Messages consumed from the specified topics will be inserted into PostgreSQL
-3. Query the data using standard SQL:
+When `auto_create_table` is enabled, the table is created with the appropriate payload column type:
```sql
--- Get all messages from a specific stream
-SELECT * FROM iggy_messages WHERE iggy_stream = 'user_events';
-
--- Get messages with their payload as text (for text/JSON payloads)
-SELECT id, iggy_offset, convert_from(payload, 'UTF8') as payload_text
-FROM iggy_messages
-WHERE iggy_stream = 'user_events';
-
--- Get messages from a specific time range
-SELECT * FROM iggy_messages
-WHERE created_at >= '2024-01-01'
-AND created_at < '2024-02-01';
-
--- Get payload size statistics
-SELECT
- iggy_stream,
- iggy_topic,
- COUNT(*) as message_count,
- AVG(length(payload)) as avg_payload_size,
- MAX(length(payload)) as max_payload_size
-FROM iggy_messages
-GROUP BY iggy_stream, iggy_topic;
+CREATE TABLE iggy_messages (
+ id DECIMAL(39, 0) PRIMARY KEY,
+ iggy_offset BIGINT,
+ iggy_timestamp TIMESTAMP WITH TIME ZONE,
+ iggy_stream TEXT,
+ iggy_topic TEXT,
+ iggy_partition_id INTEGER,
+ iggy_checksum BIGINT,
+ iggy_origin_timestamp TIMESTAMP WITH TIME ZONE,
+ payload BYTEA, -- or JSONB or TEXT based on payload_format
+ created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
```
-## Performance Considerations
+## Performance
-- Use appropriate `batch_size` for your workload (larger batches = better throughput)
-- Consider creating indexes on frequently queried columns
-- Monitor connection pool usage with `max_connections`
-- Create indexes for efficient queries:
+### Recommended Indexes
```sql
CREATE INDEX idx_iggy_messages_stream ON iggy_messages (iggy_stream);
CREATE INDEX idx_iggy_messages_topic ON iggy_messages (iggy_topic);
-CREATE INDEX idx_iggy_messages_created_at ON iggy_messages (created_at);
CREATE INDEX idx_iggy_messages_offset ON iggy_messages (iggy_offset);
+CREATE INDEX idx_iggy_messages_created_at ON iggy_messages (created_at);
+
+-- For JSONB payload_format
+CREATE INDEX idx_payload_gin ON iggy_messages USING GIN (payload);
```
-## Working with Different Payload Types
+### Tuning Tips
-### JSON Payloads
+- Increase `batch_size` for higher throughput (larger batches = fewer round trips)
+- Adjust `max_connections` based on PostgreSQL's `max_connections` setting
+- Use `poll_interval` to control how often the sink checks for new messages
+- Use `payload_format = "json"` for JSON data to enable native querying
-For JSON data, you can use PostgreSQL's JSON operators:
+## Example Configs
-```sql
--- Extract JSON fields (assuming payload is JSON)
-SELECT
- id,
- payload::text::jsonb->>'name' as name,
- payload::text::jsonb->>'email' as email
-FROM iggy_messages
-WHERE payload::text::jsonb->>'name' IS NOT NULL;
-
--- Create a GIN index for faster JSON queries
-CREATE INDEX idx_iggy_messages_payload_gin ON iggy_messages USING GIN ((payload::text::jsonb));
-```
+### JSON Messages with JSONB Storage
-### Text Payloads
+```toml
+[[streams]]
+stream = "events"
+topics = ["user_events"]
+schema = "json"
+batch_length = 100
+poll_interval = "10ms"
+consumer_group = "pg_sink"
-For text data, convert bytes to text:
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/analytics"
+target_table = "events"
+auto_create_table = true
+batch_size = 500
+payload_format = "json"
+```
-```sql
--- Full-text search on text payloads
-SELECT id, convert_from(payload, 'UTF8') as message
-FROM iggy_messages
-WHERE convert_from(payload, 'UTF8') LIKE '%error%';
+### Binary/Raw Messages
+
+```toml
+[[streams]]
+stream = "binary_data"
+topics = ["images", "files"]
+schema = "raw"
+batch_length = 50
+poll_interval = "100ms"
+consumer_group = "pg_sink"
+
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/storage"
+target_table = "binary_messages"
+auto_create_table = true
+batch_size = 100
+payload_format = "bytea"
```
-### Binary Payloads
+### Text Logs
+
+```toml
+[[streams]]
+stream = "logs"
+topics = ["app_logs"]
+schema = "text"
+batch_length = 1000
+poll_interval = "5ms"
+consumer_group = "pg_sink"
+
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/logs"
+target_table = "log_messages"
+auto_create_table = true
+include_checksum = false
+include_origin_timestamp = false
+batch_size = 1000
+payload_format = "text"
+```
-For binary data, work with raw bytes or encode as needed:
+## Reliability Features
-```sql
--- Get binary payload as hex
-SELECT id, encode(payload, 'hex') as payload_hex
-FROM iggy_messages;
+### Automatic Retries
--- Get payload size
-SELECT id, length(payload) as payload_size
-FROM iggy_messages
-ORDER BY payload_size DESC;
-```
+The connector automatically retries transient database errors (connection issues, deadlocks, serialization failures) with exponential backoff. Configure with `max_retries` (default: 3) and `retry_delay` (default: `1s`). The actual delay is `retry_delay * attempt_number`. Non-transient errors fail immediately.
+
+### Connection Pool Management
+
+The connection pool is properly closed when the connector shuts down, ensuring clean resource cleanup.
+
+## Usage with Source Connector
+
+The sink can work with the source connector for pass-through scenarios:
+
+1. **Sink** with `payload_format = "bytea"` stores messages as raw bytes
+2. **Source** with `payload_column = "payload"` and `payload_format = "bytea"` reads them back
+
+For JSON data:
+
+1. **Sink** with `payload_format = "json"` stores as JSONB
+2. **Source** with `payload_column = "data"` and `payload_format = "json_direct"` reads JSONB directly
diff --git a/core/integration/tests/connectors/postgres/connectors_config/postgres.toml b/core/connectors/sinks/postgres_sink/config.toml
similarity index 81%
copy from core/integration/tests/connectors/postgres/connectors_config/postgres.toml
copy to core/connectors/sinks/postgres_sink/config.toml
index 7476a2126..84a79def5 100644
--- a/core/integration/tests/connectors/postgres/connectors_config/postgres.toml
+++ b/core/connectors/sinks/postgres_sink/config.toml
@@ -20,18 +20,19 @@ key = "postgres"
enabled = true
version = 0
name = "Postgres sink"
-path = "../../target/debug/libiggy_connector_postgres_sink"
+path = "../../../target/debug/libiggy_connector_postgres_sink"
+verbose = false
[[streams]]
-stream = "test_stream"
-topics = ["test_topic"]
+stream = "user_events"
+topics = ["users", "orders"]
schema = "json"
batch_length = 100
poll_interval = "5ms"
-consumer_group = "test"
+consumer_group = "postgres_sink"
[plugin_config]
-connection_string = ""
+connection_string = "postgresql://user:pass@localhost:5432/database"
target_table = "iggy_messages"
batch_size = 100
max_connections = 10
@@ -39,3 +40,4 @@ auto_create_table = true
include_metadata = true
include_checksum = true
include_origin_timestamp = true
+payload_format = "bytea"
diff --git a/core/connectors/sinks/postgres_sink/src/lib.rs b/core/connectors/sinks/postgres_sink/src/lib.rs
index 4d7c2490e..03757b729 100644
--- a/core/connectors/sinks/postgres_sink/src/lib.rs
+++ b/core/connectors/sinks/postgres_sink/src/lib.rs
@@ -18,22 +18,30 @@
use async_trait::async_trait;
use chrono::{DateTime, Utc};
+use humantime::Duration as HumanDuration;
use iggy_connector_sdk::{
ConsumedMessage, Error, MessagesMetadata, Sink, TopicMetadata, sink_connector,
};
use serde::{Deserialize, Serialize};
-use sqlx::{Pool, Postgres, postgres::PgPoolOptions};
+use sqlx::postgres::PgPoolOptions;
+use sqlx::{Pool, Postgres};
+use std::str::FromStr;
+use std::time::Duration;
use tokio::sync::Mutex;
-use tracing::{error, info};
+use tracing::{debug, error, info, warn};
sink_connector!(PostgresSink);
+const DEFAULT_MAX_RETRIES: u32 = 3;
+const DEFAULT_RETRY_DELAY: &str = "1s";
+
#[derive(Debug)]
pub struct PostgresSink {
pub id: u32,
pool: Option<Pool<Postgres>>,
config: PostgresSinkConfig,
state: Mutex<State>,
+ verbose: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -46,6 +54,36 @@ pub struct PostgresSinkConfig {
pub include_metadata: Option<bool>,
pub include_checksum: Option<bool>,
pub include_origin_timestamp: Option<bool>,
+ pub payload_format: Option<String>,
+ pub verbose_logging: Option<bool>,
+ pub max_retries: Option<u32>,
+ pub retry_delay: Option<String>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum PayloadFormat {
+ #[default]
+ Bytea,
+ Json,
+ Text,
+}
+
+impl PayloadFormat {
+ fn from_config(s: Option<&str>) -> Self {
+ match s.map(|s| s.to_lowercase()).as_deref() {
+ Some("json") | Some("jsonb") => PayloadFormat::Json,
+ Some("text") => PayloadFormat::Text,
+ _ => PayloadFormat::Bytea,
+ }
+ }
+
+ fn sql_type(&self) -> &'static str {
+ match self {
+ PayloadFormat::Bytea => "BYTEA",
+ PayloadFormat::Json => "JSONB",
+ PayloadFormat::Text => "TEXT",
+ }
+ }
}
#[derive(Debug)]
@@ -56,6 +94,7 @@ struct State {
impl PostgresSink {
pub fn new(id: u32, config: PostgresSinkConfig) -> Self {
+ let verbose = config.verbose_logging.unwrap_or(false);
PostgresSink {
id,
pool: None,
@@ -64,27 +103,65 @@ impl PostgresSink {
messages_processed: 0,
insertion_errors: 0,
}),
+ verbose,
}
}
+}
- async fn connect(&mut self) -> Result<(), Error> {
- let max_connections = self.config.max_connections.unwrap_or(10);
- let redacted_connection_string = self
- .config
- .connection_string
- .chars()
- .take(15)
- .collect::<String>();
+#[async_trait]
+impl Sink for PostgresSink {
+ async fn open(&mut self) -> Result<(), Error> {
+ info!(
+ "Opening PostgreSQL sink connector with ID: {}. Target table: {}",
+ self.id, self.config.target_table
+ );
+ self.connect().await?;
+ self.ensure_table_exists().await?;
+ Ok(())
+ }
+
+ async fn consume(
+ &self,
+ topic_metadata: &TopicMetadata,
+ messages_metadata: MessagesMetadata,
+ messages: Vec<ConsumedMessage>,
+ ) -> Result<(), Error> {
+ self.process_messages(topic_metadata, &messages_metadata, &messages)
+ .await
+ }
+ async fn close(&mut self) -> Result<(), Error> {
+ info!("Closing PostgreSQL sink connector with ID: {}", self.id);
+
+ if let Some(pool) = self.pool.take() {
+ pool.close().await;
+ info!(
+ "PostgreSQL connection pool closed for sink connector ID: {}",
+ self.id
+ );
+ }
+
+ let state = self.state.lock().await;
info!(
- "Connecting to PostgreSQL database with max {max_connections} connections, connection string: {redacted_connection_string}",
+ "PostgreSQL sink ID: {} processed {} messages with {} errors",
+ self.id, state.messages_processed, state.insertion_errors
);
+ Ok(())
+ }
+}
+
+impl PostgresSink {
+ async fn connect(&mut self) -> Result<(), Error> {
+ let max_connections = self.config.max_connections.unwrap_or(10);
+ let redacted = redact_connection_string(&self.config.connection_string);
+
+ info!("Connecting to PostgreSQL with max {max_connections} connections: {redacted}");
+
let pool = PgPoolOptions::new()
.max_connections(max_connections)
.connect(&self.config.connection_string)
.await
.map_err(|e| Error::InitError(format!("Failed to connect to PostgreSQL: {e}")))?;
- info!("Connected to PostgreSQL database");
sqlx::query("SELECT 1")
.execute(&pool)
@@ -92,10 +169,7 @@ impl PostgresSink {
.map_err(|e| Error::InitError(format!("Database connectivity test failed: {e}")))?;
self.pool = Some(pool);
- info!(
- "Connected to PostgreSQL database with {} max connections",
- max_connections
- );
+ info!("Connected to PostgreSQL database with {max_connections} max connections");
Ok(())
}
@@ -104,43 +178,42 @@ impl PostgresSink {
return Ok(());
}
- let pool = self
- .pool
- .as_ref()
- .ok_or_else(|| Error::InitError("Database not connected".to_string()))?;
+ let pool = self.get_pool()?;
let table_name = &self.config.target_table;
+ let quoted_table = quote_identifier(table_name)?;
let include_metadata = self.config.include_metadata.unwrap_or(true);
let include_checksum = self.config.include_checksum.unwrap_or(true);
let include_origin_timestamp = self.config.include_origin_timestamp.unwrap_or(true);
+ let payload_type = self.payload_format().sql_type();
- let mut create_table_sql = format!("CREATE TABLE IF NOT EXISTS {table_name} (");
- create_table_sql.push_str("id DECIMAL(39, 0) PRIMARY KEY");
+ let mut sql = format!("CREATE TABLE IF NOT EXISTS {quoted_table} (");
+ sql.push_str("id DECIMAL(39, 0) PRIMARY KEY");
if include_metadata {
- create_table_sql.push_str(", iggy_offset BIGINT");
- create_table_sql.push_str(", iggy_timestamp TIMESTAMP WITH TIME ZONE");
- create_table_sql.push_str(", iggy_stream TEXT");
- create_table_sql.push_str(", iggy_topic TEXT");
- create_table_sql.push_str(", iggy_partition_id INTEGER");
+ sql.push_str(", iggy_offset BIGINT");
+ sql.push_str(", iggy_timestamp TIMESTAMP WITH TIME ZONE");
+ sql.push_str(", iggy_stream TEXT");
+ sql.push_str(", iggy_topic TEXT");
+ sql.push_str(", iggy_partition_id INTEGER");
}
if include_checksum {
- create_table_sql.push_str(", iggy_checksum BIGINT");
+ sql.push_str(", iggy_checksum BIGINT");
}
if include_origin_timestamp {
- create_table_sql.push_str(", iggy_origin_timestamp TIMESTAMP WITH TIME ZONE");
+ sql.push_str(", iggy_origin_timestamp TIMESTAMP WITH TIME ZONE");
}
- create_table_sql.push_str(", payload BYTEA");
- create_table_sql.push_str(", created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW())");
+ sql.push_str(&format!(", payload {payload_type}"));
+ sql.push_str(", created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW())");
- sqlx::query(&create_table_sql)
+ sqlx::query(&sql)
.execute(pool)
.await
.map_err(|e| Error::InitError(format!("Failed to create table '{table_name}': {e}")))?;
- info!("Ensured table '{}' exists", table_name);
+ info!("Ensured table '{table_name}' exists with payload type {payload_type}");
Ok(())
}
@@ -150,10 +223,7 @@ impl PostgresSink {
messages_metadata: &MessagesMetadata,
messages: &[ConsumedMessage],
) -> Result<(), Error> {
- let pool = self
- .pool
- .as_ref()
- .ok_or_else(|| Error::InitError("Database not connected".to_string()))?;
+ let pool = self.get_pool()?;
let batch_size = self.config.batch_size.unwrap_or(100) as usize;
for batch in messages.chunks(batch_size) {
@@ -163,19 +233,26 @@ impl PostgresSink {
{
let mut state = self.state.lock().await;
state.insertion_errors += batch.len() as u64;
- error!("Failed to insert batch: {}", e);
+ error!("Failed to insert batch: {e}");
}
}
let mut state = self.state.lock().await;
state.messages_processed += messages.len() as u64;
- info!(
- "PostgreSQL sink ID: {} processed {} messages to table '{}'",
- self.id,
- messages.len(),
- self.config.target_table
- );
+ let msg_count = messages.len();
+ let table = &self.config.target_table;
+ if self.verbose {
+ info!(
+ "PostgreSQL sink ID: {} processed {msg_count} messages to table '{table}'",
+ self.id
+ );
+ } else {
+ debug!(
+ "PostgreSQL sink ID: {} processed {msg_count} messages to table '{table}'",
+ self.id
+ );
+ }
Ok(())
}
@@ -187,122 +264,540 @@ impl PostgresSink {
messages_metadata: &MessagesMetadata,
pool: &Pool<Postgres>,
) -> Result<(), Error> {
+ if messages.is_empty() {
+ return Ok(());
+ }
+
let table_name = &self.config.target_table;
let include_metadata = self.config.include_metadata.unwrap_or(true);
let include_checksum = self.config.include_checksum.unwrap_or(true);
let include_origin_timestamp = self.config.include_origin_timestamp.unwrap_or(true);
+ let payload_format = self.payload_format();
+
+ let (query, _params_per_row) = self.build_batch_insert_query(
+ table_name,
+ include_metadata,
+ include_checksum,
+ include_origin_timestamp,
+ messages.len(),
+ )?;
+
+ self.execute_batch_insert_with_retry(
+ pool,
+ &query,
+ messages,
+ topic_metadata,
+ messages_metadata,
+ include_metadata,
+ include_checksum,
+ include_origin_timestamp,
+ payload_format,
+ )
+ .await
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn execute_batch_insert_with_retry(
+ &self,
+ pool: &Pool<Postgres>,
+ query: &str,
+ messages: &[ConsumedMessage],
+ topic_metadata: &TopicMetadata,
+ messages_metadata: &MessagesMetadata,
+ include_metadata: bool,
+ include_checksum: bool,
+ include_origin_timestamp: bool,
+ payload_format: PayloadFormat,
+ ) -> Result<(), Error> {
+ let max_retries = self.get_max_retries();
+ let retry_delay = self.get_retry_delay();
+ let mut attempts = 0u32;
+
+ loop {
+ let result = self
+ .bind_and_execute_batch(
+ pool,
+ query,
+ messages,
+ topic_metadata,
+ messages_metadata,
+ include_metadata,
+ include_checksum,
+ include_origin_timestamp,
+ payload_format,
+ )
+ .await;
+
+ match result {
+ Ok(_) => return Ok(()),
+ Err((e, is_transient)) => {
+ attempts += 1;
+ if !is_transient || attempts >= max_retries {
+ error!("Batch insert failed after {attempts} attempts: {e}");
+ return Err(Error::CannotStoreData(format!(
+ "Batch insert failed after {attempts} attempts: {e}"
+ )));
+ }
+ warn!(
+ "Transient database error (attempt {attempts}/{max_retries}): {e}. Retrying..."
+ );
+ tokio::time::sleep(retry_delay * attempts).await;
+ }
+ }
+ }
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn bind_and_execute_batch(
+ &self,
+ pool: &Pool<Postgres>,
+ query: &str,
+ messages: &[ConsumedMessage],
+ topic_metadata: &TopicMetadata,
+ messages_metadata: &MessagesMetadata,
+ include_metadata: bool,
+ include_checksum: bool,
+ include_origin_timestamp: bool,
+ payload_format: PayloadFormat,
+ ) -> Result<(), (sqlx::Error, bool)> {
+ let mut query_builder = sqlx::query(query);
for message in messages {
let payload_bytes = message.payload.clone().try_into_vec().map_err(|e| {
- error!("Failed to convert payload to bytes: {}", e);
- Error::InvalidRecord
+ let err_msg = format!("Failed to convert payload to bytes: {e}");
+ (sqlx::Error::Protocol(err_msg), false)
})?;
- let mut query = format!("INSERT INTO {table_name} (id");
- let mut values = "($1".to_string();
- let mut param_count = 1;
+ let timestamp = self.parse_timestamp(message.timestamp);
+ let origin_timestamp_val = self.parse_timestamp(message.origin_timestamp);
+
+ query_builder = query_builder.bind(message.id.to_string());
if include_metadata {
- query.push_str(
- ", iggy_offset, iggy_timestamp, iggy_stream, iggy_topic, iggy_partition_id",
- );
- for i in 2..=6 {
- values.push_str(&format!(", ${i}"));
- }
- param_count = 6;
+ query_builder = query_builder
+ .bind(message.offset as i64)
+ .bind(timestamp)
+ .bind(topic_metadata.stream.clone())
+ .bind(topic_metadata.topic.clone())
+ .bind(messages_metadata.partition_id as i32);
}
if include_checksum {
- query.push_str(", iggy_checksum");
- param_count += 1;
- values.push_str(&format!(", ${param_count}"));
+ query_builder = query_builder.bind(message.checksum as i64);
}
if include_origin_timestamp {
- query.push_str(", iggy_origin_timestamp");
- param_count += 1;
- values.push_str(&format!(", ${param_count}"));
+ query_builder = query_builder.bind(origin_timestamp_val);
}
- query.push_str(", payload");
- param_count += 1;
- values.push_str(&format!(", ${param_count}"));
+ query_builder = match payload_format {
+ PayloadFormat::Bytea => query_builder.bind(payload_bytes),
+ PayloadFormat::Json => {
+ let json_value: serde_json::Value = serde_json::from_slice(&payload_bytes)
+ .map_err(|e| {
+ let err_msg = format!("Failed to parse payload as JSON: {e}");
+ error!("{err_msg}");
+ (sqlx::Error::Protocol(err_msg), false)
+ })?;
+ query_builder.bind(json_value)
+ }
+ PayloadFormat::Text => {
+ let text_value = String::from_utf8(payload_bytes).map_err(|e| {
+ let err_msg = format!("Failed to parse payload as UTF-8 text: {e}");
+ error!("{err_msg}");
+ (sqlx::Error::Protocol(err_msg), false)
+ })?;
+ query_builder.bind(text_value)
+ }
+ };
+ }
- query.push_str(&format!(") VALUES {values}"));
- query.push(')');
+ query_builder.execute(pool).await.map_err(|e| {
+ let is_transient = is_transient_error(&e);
+ (e, is_transient)
+ })?;
- let message_id_str = message.id.to_string();
- let mut query_obj = sqlx::query(&query).bind(message_id_str);
+ Ok(())
+ }
- if include_metadata {
- let timestamp = DateTime::from_timestamp(
- (message.timestamp / 1_000_000) as i64,
- ((message.timestamp % 1_000_000) * 1_000) as u32,
- )
- .unwrap_or_else(Utc::now);
+ fn get_pool(&self) -> Result<&Pool<Postgres>, Error> {
+ self.pool
+ .as_ref()
+ .ok_or_else(|| Error::InitError("Database not connected".to_string()))
+ }
- query_obj = query_obj
- .bind(message.offset as i64)
- .bind(timestamp)
- .bind(&topic_metadata.stream)
- .bind(&topic_metadata.topic)
- .bind(messages_metadata.partition_id as i32);
- }
+ fn payload_format(&self) -> PayloadFormat {
+ PayloadFormat::from_config(self.config.payload_format.as_deref())
+ }
- if include_checksum {
- query_obj = query_obj.bind(message.checksum as i64);
- }
+ fn get_max_retries(&self) -> u32 {
+ self.config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES)
+ }
- if include_origin_timestamp {
- let origin_timestamp = DateTime::from_timestamp(
- (message.origin_timestamp / 1_000_000) as i64,
- ((message.origin_timestamp % 1_000_000) * 1_000) as u32,
- )
- .unwrap_or_else(Utc::now);
- query_obj = query_obj.bind(origin_timestamp);
- }
+ fn get_retry_delay(&self) -> Duration {
+ let delay_str = self
+ .config
+ .retry_delay
+ .as_deref()
+ .unwrap_or(DEFAULT_RETRY_DELAY);
+ HumanDuration::from_str(delay_str)
+ .unwrap_or_else(|e| panic!("Invalid retry_delay '{delay_str}': {e}"))
+ .into()
+ }
- query_obj = query_obj.bind(payload_bytes);
+ fn parse_timestamp(&self, micros: u64) -> DateTime<Utc> {
+ DateTime::from_timestamp(
+ (micros / 1_000_000) as i64,
+ ((micros % 1_000_000) * 1_000) as u32,
+ )
+ .unwrap_or_else(Utc::now)
+ }
- query_obj.execute(pool).await.map_err(|e| {
- error!("Failed to insert message: {}", e);
- Error::InvalidRecord
- })?;
+ fn build_batch_insert_query(
+ &self,
+ table_name: &str,
+ include_metadata: bool,
+ include_checksum: bool,
+ include_origin_timestamp: bool,
+ row_count: usize,
+ ) -> Result<(String, u32), Error> {
+ let quoted_table = quote_identifier(table_name)?;
+ let mut query = format!("INSERT INTO {quoted_table} (id");
+
+ let mut params_per_row: u32 = 1; // id
+
+ if include_metadata {
+ query.push_str(
+ ", iggy_offset, iggy_timestamp, iggy_stream, iggy_topic, iggy_partition_id",
+ );
+ params_per_row += 5;
}
- Ok(())
+ if include_checksum {
+ query.push_str(", iggy_checksum");
+ params_per_row += 1;
+ }
+
+ if include_origin_timestamp {
+ query.push_str(", iggy_origin_timestamp");
+ params_per_row += 1;
+ }
+
+ query.push_str(", payload");
+ params_per_row += 1;
+
+ query.push_str(") VALUES ");
+
+ let mut value_groups = Vec::with_capacity(row_count);
+ for row_idx in 0..row_count {
+ let base_param = (row_idx as u32) * params_per_row;
+ let mut values = format!("(${}::numeric", base_param + 1);
+ for i in 2..=params_per_row {
+ values.push_str(&format!(", ${}", base_param + i));
+ }
+ values.push(')');
+ value_groups.push(values);
+ }
+
+ query.push_str(&value_groups.join(", "));
+
+ Ok((query, params_per_row))
}
}
-#[async_trait]
-impl Sink for PostgresSink {
- async fn open(&mut self) -> Result<(), Error> {
- info!(
- "Opening PostgreSQL sink connector with ID: {}. Target table: {}",
- self.id, self.config.target_table
+fn is_transient_error(e: &sqlx::Error) -> bool {
+ match e {
+ sqlx::Error::Io(_) => true,
+ sqlx::Error::PoolTimedOut => true,
+ sqlx::Error::PoolClosed => false,
+ sqlx::Error::Protocol(_) => false,
+ sqlx::Error::Database(db_err) => db_err.code().is_some_and(|code| {
+ matches!(
+ code.as_ref(),
+ "40001" | "40P01" | "57P01" | "57P02" | "57P03" | "08000" | "08003" | "08006"
+ )
+ }),
+ _ => false,
+ }
+}
+
+fn quote_identifier(name: &str) -> Result<String, Error> {
+ if name.is_empty() {
+ return Err(Error::InitError("Table name cannot be empty".to_string()));
+ }
+ if name.contains('\0') {
+ return Err(Error::InitError(
+ "Table name cannot contain null characters".to_string(),
+ ));
+ }
+ let escaped = name.replace('"', "\"\"");
+ Ok(format!("\"{escaped}\""))
+}
+
+fn redact_connection_string(conn_str: &str) -> String {
+ if let Some(scheme_end) = conn_str.find("://") {
+ let scheme = &conn_str[..scheme_end + 3];
+ let rest = &conn_str[scheme_end + 3..];
+ let preview: String = rest.chars().take(3).collect();
+ return format!("{scheme}{preview}***");
+ }
+ let preview: String = conn_str.chars().take(3).collect();
+ format!("{preview}***")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn test_config() -> PostgresSinkConfig {
+ PostgresSinkConfig {
+ connection_string: "postgres://localhost/db".to_string(),
+ target_table: "messages".to_string(),
+ batch_size: Some(100),
+ max_connections: None,
+ auto_create_table: None,
+ include_metadata: None,
+ include_checksum: None,
+ include_origin_timestamp: None,
+ payload_format: None,
+ verbose_logging: None,
+ max_retries: None,
+ retry_delay: None,
+ }
+ }
+
+ #[test]
+ fn given_json_format_should_return_json() {
+ assert_eq!(
+ PayloadFormat::from_config(Some("json")),
+ PayloadFormat::Json
+ );
+ assert_eq!(
+ PayloadFormat::from_config(Some("jsonb")),
+ PayloadFormat::Json
+ );
+ assert_eq!(
+ PayloadFormat::from_config(Some("JSON")),
+ PayloadFormat::Json
);
- self.connect().await?;
- self.ensure_table_exists().await?;
- Ok(())
}
- async fn consume(
- &self,
- topic_metadata: &TopicMetadata,
- messages_metadata: MessagesMetadata,
- messages: Vec<ConsumedMessage>,
- ) -> Result<(), Error> {
- self.process_messages(topic_metadata, &messages_metadata, &messages)
- .await
+ #[test]
+ fn given_text_format_should_return_text() {
+ assert_eq!(
+ PayloadFormat::from_config(Some("text")),
+ PayloadFormat::Text
+ );
+ assert_eq!(
+ PayloadFormat::from_config(Some("TEXT")),
+ PayloadFormat::Text
+ );
}
- async fn close(&mut self) -> Result<(), Error> {
- info!("Closing PostgreSQL sink connector with ID: {}", self.id);
- let state = self.state.lock().await;
- info!(
- "PostgreSQL sink ID: {} processed {} messages with {} errors",
- self.id, state.messages_processed, state.insertion_errors
+ #[test]
+ fn given_bytea_or_unknown_format_should_return_bytea() {
+ assert_eq!(
+ PayloadFormat::from_config(Some("bytea")),
+ PayloadFormat::Bytea
);
- Ok(())
+ assert_eq!(
+ PayloadFormat::from_config(Some("unknown")),
+ PayloadFormat::Bytea
+ );
+ assert_eq!(PayloadFormat::from_config(None), PayloadFormat::Bytea);
+ }
+
+ #[test]
+ fn given_payload_format_should_return_correct_sql_type() {
+ assert_eq!(PayloadFormat::Bytea.sql_type(), "BYTEA");
+ assert_eq!(PayloadFormat::Json.sql_type(), "JSONB");
+ assert_eq!(PayloadFormat::Text.sql_type(), "TEXT");
+ }
+
+ #[test]
+ fn given_all_options_enabled_should_build_full_insert_query() {
+ let sink = PostgresSink::new(1, test_config());
+ let (query, param_count) = sink
+ .build_batch_insert_query("messages", true, true, true, 1)
+ .expect("Failed to build query");
+
+ assert!(query.contains("INSERT INTO \"messages\""));
+ assert!(query.contains("iggy_offset"));
+ assert!(query.contains("iggy_timestamp"));
+ assert!(query.contains("iggy_stream"));
+ assert!(query.contains("iggy_topic"));
+ assert!(query.contains("iggy_partition_id"));
+ assert!(query.contains("iggy_checksum"));
+ assert!(query.contains("iggy_origin_timestamp"));
+ assert!(query.contains("payload"));
+ assert_eq!(param_count, 9);
+ }
+
+ #[test]
+ fn given_metadata_disabled_should_build_minimal_insert_query() {
+ let sink = PostgresSink::new(1, test_config());
+ let (query, param_count) = sink
+ .build_batch_insert_query("messages", false, false, false, 1)
+ .expect("Failed to build query");
+
+ assert!(query.contains("INSERT INTO \"messages\""));
+ assert!(!query.contains("iggy_offset"));
+ assert!(!query.contains("iggy_checksum"));
+ assert!(!query.contains("iggy_origin_timestamp"));
+ assert!(query.contains("payload"));
+ assert_eq!(param_count, 2);
+ }
+
+ #[test]
+ fn given_only_checksum_enabled_should_include_checksum() {
+ let sink = PostgresSink::new(1, test_config());
+ let (query, param_count) = sink
+ .build_batch_insert_query("messages", false, true, false, 1)
+ .expect("Failed to build query");
+
+ assert!(!query.contains("iggy_offset"));
+ assert!(query.contains("iggy_checksum"));
+ assert!(!query.contains("iggy_origin_timestamp"));
+ assert_eq!(param_count, 3);
+ }
+
+ #[test]
+ fn given_microseconds_should_parse_timestamp_correctly() {
+ let sink = PostgresSink::new(1, test_config());
+ let micros: u64 = 1_767_225_600_000_000; // 2026-01-01 00:00:00 UTC
+ let dt = sink.parse_timestamp(micros);
+
+ assert_eq!(dt.timestamp(), 1_767_225_600);
+ }
+
+ #[test]
+ fn given_default_config_should_use_default_retries() {
+ let sink = PostgresSink::new(1, test_config());
+ assert_eq!(sink.get_max_retries(), DEFAULT_MAX_RETRIES);
+ }
+
+ #[test]
+ fn given_custom_retries_should_use_custom_value() {
+ let mut config = test_config();
+ config.max_retries = Some(5);
+ let sink = PostgresSink::new(1, config);
+ assert_eq!(sink.get_max_retries(), 5);
+ }
+
+ #[test]
+ fn given_default_config_should_use_default_retry_delay() {
+ let sink = PostgresSink::new(1, test_config());
+ assert_eq!(sink.get_retry_delay(), Duration::from_secs(1));
+ }
+
+ #[test]
+ fn given_custom_retry_delay_should_parse_humantime() {
+ let mut config = test_config();
+ config.retry_delay = Some("500ms".to_string());
+ let sink = PostgresSink::new(1, config);
+ assert_eq!(sink.get_retry_delay(), Duration::from_millis(500));
+ }
+
+ #[test]
+ fn given_verbose_logging_enabled_should_set_verbose_flag() {
+ let mut config = test_config();
+ config.verbose_logging = Some(true);
+ let sink = PostgresSink::new(1, config);
+ assert!(sink.verbose);
+ }
+
+ #[test]
+ fn given_verbose_logging_disabled_should_not_set_verbose_flag() {
+ let sink = PostgresSink::new(1, test_config());
+ assert!(!sink.verbose);
+ }
+
+ #[test]
+ fn given_connection_string_with_credentials_should_redact() {
+ let conn = "postgres://user:password@localhost:5432/db";
+ let redacted = redact_connection_string(conn);
+ assert_eq!(redacted, "postgres://use***");
+ }
+
+ #[test]
+ fn given_connection_string_without_scheme_should_redact() {
+ let conn = "localhost:5432/db";
+ let redacted = redact_connection_string(conn);
+ assert_eq!(redacted, "loc***");
+ }
+
+ #[test]
+ fn given_postgresql_scheme_should_redact() {
+ let conn = "postgresql://admin:secret123@db.example.com:5432/mydb";
+ let redacted = redact_connection_string(conn);
+ assert_eq!(redacted, "postgresql://adm***");
+ }
+
+ #[test]
+ fn given_special_chars_in_identifier_should_escape() {
+ let result = quote_identifier("table\"name").expect("Failed to quote");
+ assert_eq!(result, "\"table\"\"name\"");
+ }
+
+ #[test]
+ fn given_empty_identifier_should_fail() {
+ let result = quote_identifier("");
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn given_null_char_in_identifier_should_fail() {
+ let result = quote_identifier("table\0name");
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn given_normal_identifier_should_quote() {
+ let result = quote_identifier("my_table").expect("Failed to quote");
+ assert_eq!(result, "\"my_table\"");
+ }
+
+ #[test]
+ fn given_identifier_with_spaces_should_quote() {
+ let result = quote_identifier("my table").expect("Failed to quote");
+ assert_eq!(result, "\"my table\"");
+ }
+
+ #[test]
+ fn given_identifier_with_sql_injection_should_escape() {
+ let result = quote_identifier("messages\"; DROP TABLE users; --").expect("Failed to quote");
+ assert_eq!(result, "\"messages\"\"; DROP TABLE users; --\"");
+ }
+
+ #[test]
+ fn given_batch_of_3_rows_should_build_multi_row_insert_query() {
+ let sink = PostgresSink::new(1, test_config());
+ let (query, params_per_row) = sink
+ .build_batch_insert_query("messages", true, true, true, 3)
+ .expect("Failed to build batch query");
+
+ // With all options: id + 5 metadata + checksum + origin_timestamp + payload = 9 params per row
+ assert_eq!(params_per_row, 9);
+
+ // Should have 3 value groups
+ assert!(query.contains("($1::numeric, $2, $3, $4, $5, $6, $7, $8, $9)"));
+ assert!(query.contains("($10::numeric, $11, $12, $13, $14, $15, $16, $17, $18)"));
+ assert!(query.contains("($19::numeric, $20, $21, $22, $23, $24, $25, $26, $27)"));
+ }
+
+ #[test]
+ fn given_batch_of_2_rows_minimal_should_build_correct_query() {
+ let sink = PostgresSink::new(1, test_config());
+ let (query, params_per_row) = sink
+ .build_batch_insert_query("messages", false, false, false, 2)
+ .expect("Failed to build batch query");
+
+ // With minimal options: id + payload = 2 params per row
+ assert_eq!(params_per_row, 2);
+
+ // Should have 2 value groups
+ assert!(query.contains("($1::numeric, $2)"));
+ assert!(query.contains("($3::numeric, $4)"));
+ assert!(!query.contains("$5"));
}
}
diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md
index 4bdb8e985..90c453545 100644
--- a/core/connectors/sources/README.md
+++ b/core/connectors/sources/README.md
@@ -35,6 +35,7 @@ pub struct SourceConfig {
pub streams: Vec<StreamProducerConfig>,
pub plugin_config_format: Option<ConfigFormat>,
pub plugin_config: Option<serde_json::Value>,
+ pub verbose: bool, // Log message processing at info level instead of debug (default: false)
}
```
@@ -59,6 +60,7 @@ version = 0
name = "Random source" # Name of the source
path = "libiggy_connector_random_source" # Path to the source connector
config_format = "toml"
+verbose = false # Log message processing at info level instead of debug
# Collection of the streams to which the produced messages are sent
[[streams]]
diff --git a/core/connectors/sources/postgres_source/Cargo.toml b/core/connectors/sources/postgres_source/Cargo.toml
index b6303f192..dc6b1dfc1 100644
--- a/core/connectors/sources/postgres_source/Cargo.toml
+++ b/core/connectors/sources/postgres_source/Cargo.toml
@@ -40,6 +40,7 @@ cdc_pg_replicate = []
[dependencies]
async-trait = { workspace = true }
+base64 = { workspace = true }
chrono = { workspace = true }
dashmap = { workspace = true }
futures = { workspace = true }
diff --git a/core/connectors/sources/postgres_source/README.md b/core/connectors/sources/postgres_source/README.md
index 89ddd0661..000f3fb1f 100644
--- a/core/connectors/sources/postgres_source/README.md
+++ b/core/connectors/sources/postgres_source/README.md
@@ -1,76 +1,88 @@
# PostgreSQL Source Connector
-The PostgreSQL source connector allows you to fetch data from PostgreSQL databases and stream it to Iggy topics. It supports both table polling and Change Data Capture (CDC) modes.
+The PostgreSQL source connector fetches data from PostgreSQL databases and streams it to Iggy topics. It supports table polling and Change Data Capture (CDC) modes with flexible payload extraction.
## Features
- **Table Polling**: Incrementally fetch data from PostgreSQL tables
- **Change Data Capture**: Monitor database changes using PostgreSQL logical replication
- - Built-in logical decoding using `pg_logical_slot_get_changes`
- - Optional backend using Supabase's ETL (`pg_replicate`) framework via feature flag
-- **Configurable Polling Intervals**: Control how often to check for new data
+- **Flexible Payload Extraction**: Extract BYTEA, TEXT, or JSONB columns directly as payload
+- **Custom Queries**: Use custom SQL queries with parameter substitution
+- **Delete After Read**: Automatically delete rows after processing
+- **Mark as Processed**: Mark rows as processed using a boolean column
+- **Multiple Tables**: Monitor multiple tables simultaneously
- **Batch Processing**: Fetch data in configurable batch sizes
- **Offset Tracking**: Keep track of processed records to avoid duplicates
-- **Multiple Tables**: Monitor multiple tables simultaneously
-- **Column Mapping**: Transform column names (e.g., to snake_case)
-- **Custom Queries**: Use custom SQL queries instead of simple table polling
## Configuration
-```json
-{
- "connection_string": "postgresql://username:password@localhost:5432/database",
- "mode": "polling",
- "tables": ["users", "orders", "products"],
- "poll_interval": "30s",
- "batch_size": 1000,
- "tracking_column": "updated_at",
- "initial_offset": "2024-01-01T00:00:00Z",
- "max_connections": 10,
- "enable_wal_cdc": false,
- "custom_query": "SELECT * FROM users WHERE updated_at > $1 ORDER BY updated_at LIMIT $2",
- "snake_case_columns": true,
- "include_metadata": true,
- "publication_name": "iggy_publication",
- "replication_slot": "iggy_slot",
- "capture_operations": ["INSERT", "UPDATE", "DELETE"],
- "cdc_backend": "builtin"
-}
-```
+```toml
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/database"
+mode = "polling"
+tables = ["users", "orders"]
+poll_interval = "1s"
+batch_size = 1000
+tracking_column = "id"
+initial_offset = "0"
+max_connections = 10
+snake_case_columns = false
+include_metadata = true
-### Configuration Options
+# Payload extraction (optional)
+payload_column = "payload"
+payload_format = "bytea"
-- `connection_string`: PostgreSQL connection string
-- `mode`: Operation mode - `polling` or `cdc`
-- `tables`: List of tables to monitor (used for polling and to create publications in CDC)
-- `poll_interval`: How often to poll for new data (e.g., "30s", "5m")
-- `batch_size`: Maximum number of rows to fetch per poll (default: 1000)
-- `tracking_column`: Column to track for incremental updates (default: "id")
-- `initial_offset`: Starting value for the tracking column
-- `max_connections`: Maximum database connections (default: 10)
-- `enable_wal_cdc`: Enable WAL-based CDC (requires logical replication setup)
-- `custom_query`: Custom SQL query (overrides table + tracking_column)
-- `snake_case_columns`: Convert column names to snake_case (default: false)
-- `include_metadata`: Wrap results with metadata (default: true)
-- `publication_name`: Logical replication publication name (default: `iggy_publication`)
-- `replication_slot`: Replication slot name (default: `iggy_slot`)
-- `capture_operations`: Operations to capture in CDC mode (default: ["INSERT","UPDATE","DELETE"])
-- `cdc_backend`: CDC backend implementation: `builtin` (default) or `pg_replicate`
+# Delete/mark processed (optional)
+delete_after_read = false
+processed_column = "is_processed"
+primary_key_column = "id"
-### pg_replicate backend
+# Custom query (optional)
+custom_query = "SELECT * FROM $table WHERE id > $offset ORDER BY id LIMIT $limit"
-- Build-time feature flag: enable with `--features cdc_pg_replicate` on the `iggy_connector_postgres_source` crate.
-- When `cdc_backend` is set to `pg_replicate` but the feature is not enabled, the connector will return a clear initialization error.
-- The backend integrates Supabase's ETL for robust WAL decoding and resume from LSN.
+# CDC options (optional)
+enable_wal_cdc = false
+publication_name = "iggy_publication"
+replication_slot = "iggy_slot"
+capture_operations = ["INSERT", "UPDATE", "DELETE"]
+cdc_backend = "builtin"
+```
+
+## Configuration Options
-### Notes
+| Option | Type | Default | Description |
+| ------ | ---- | ------- | ----------- |
+| `connection_string` | string | required | PostgreSQL connection string |
+| `mode` | string | required | `polling` or `cdc` |
+| `tables` | array | required | List of tables to monitor |
+| `poll_interval` | string | `1s` | How often to poll (e.g., `1s`, `5m`) |
+| `batch_size` | u32 | `1000` | Max rows per poll |
+| `tracking_column` | string | `id` | Column for incremental updates |
+| `initial_offset` | string | none | Starting value for tracking column |
+| `max_connections` | u32 | `10` | Max database connections |
+| `snake_case_columns` | bool | `false` | Convert column names to snake_case |
+| `include_metadata` | bool | `true` | Wrap results with metadata |
+| `payload_column` | string | none | Column to extract as payload |
+| `payload_format` | string | `bytea` | Format of payload_column: `bytea`, `text`, or `json_direct` |
+| `delete_after_read` | bool | `false` | Delete rows after reading |
+| `processed_column` | string | none | Boolean column to mark as processed |
+| `primary_key_column` | string | tracking_column | PK for delete/mark operations |
+| `custom_query` | string | none | Custom SQL with parameter substitution |
+| `enable_wal_cdc` | bool | `false` | Enable WAL-based CDC |
+| `publication_name` | string | `iggy_publication` | Logical replication publication |
+| `replication_slot` | string | `iggy_slot` | Replication slot name |
+| `capture_operations` | array | `["INSERT","UPDATE","DELETE"]` | CDC operations to capture |
+| `cdc_backend` | string | `builtin` | `builtin` or `pg_replicate` |
+| `verbose_logging` | bool | `false` | Log at info level instead of debug |
+| `max_retries` | u32 | `3` | Max retry attempts for transient errors |
+| `retry_delay` | string | `1s` | Base delay between retries (e.g., `500ms`, `2s`) |
-- CDC requires `wal_level = logical` on the PostgreSQL server.
-- Ensure a direct connection (no pooler) for replication where required.
+## Output Modes
-## Output Format
+### JSON Mode (Default)
-Each message contains:
+When `payload_column` is not set, each row is wrapped in a `DatabaseRecord` JSON structure:
```json
{
@@ -80,16 +92,283 @@ Each message contains:
"data": {
"id": 123,
"name": "John Doe",
- "email": "john@example.com",
- "updated_at": "2024-01-15T10:29:50Z"
+ "email": "john@example.com"
},
"old_data": null
}
```
-## Usage Example
+The stream config should use `schema = "json"`.
+
+### Payload Column Extraction
+
+When `payload_column` is set, the connector extracts that column directly as the Iggy message payload. The `payload_format` option determines how the column is read:
+
+| Format | Column Type | Schema | Description |
+| ------ | ----------- | ------ | ----------- |
+| `bytea` / `raw` | `BYTEA` | `raw` | Raw bytes passthrough |
+| `text` | `TEXT` | `text` | UTF-8 text |
+| `json_direct` / `jsonb` | `JSONB` | `json` | JSON object serialized to bytes |
+
+## Payload Format Examples
+
+### BYTEA (Raw Bytes)
+
+Extract raw bytes from a BYTEA column:
+
+```sql
+CREATE TABLE message_queue (
+ id SERIAL PRIMARY KEY,
+ payload BYTEA NOT NULL
+);
+```
+
+```toml
+[[streams]]
+stream = "messages"
+topic = "queue"
+schema = "raw"
+batch_length = 100
+
+[plugin_config]
+tables = ["message_queue"]
+tracking_column = "id"
+payload_column = "payload"
+payload_format = "bytea"
+```
+
+### TEXT
+
+Extract text from a TEXT column:
+
+```sql
+CREATE TABLE logs (
+ id SERIAL PRIMARY KEY,
+ message TEXT NOT NULL
+);
+```
+
+```toml
+[[streams]]
+stream = "logs"
+topic = "app_logs"
+schema = "text"
+batch_length = 100
+
+[plugin_config]
+tables = ["logs"]
+tracking_column = "id"
+payload_column = "message"
+payload_format = "text"
+```
+
+### JSONB (Direct)
+
+Extract JSONB directly as JSON payload (without `DatabaseRecord` wrapper):
+
+```sql
+CREATE TABLE events (
+ id SERIAL PRIMARY KEY,
+ data JSONB NOT NULL
+);
+```
+
+```toml
+[[streams]]
+stream = "events"
+topic = "user_events"
+schema = "json"
+batch_length = 100
+
+[plugin_config]
+tables = ["events"]
+tracking_column = "id"
+payload_column = "data"
+payload_format = "json_direct"
+```
+
+## Custom Query Parameters
+
+When using `custom_query`, these placeholders are available:
+
+| Placeholder | Replaced With |
+| ----------- | ------------- |
+| `$table` | Current table name |
+| `$offset` | Last processed offset (or `initial_offset`) |
+| `$limit` | `batch_size` value |
+| `$now` | Current UTC timestamp (RFC3339) |
+| `$now_unix` | Current Unix timestamp (seconds) |
+
+Example:
+
+```sql
+SELECT * FROM $table
+WHERE created_at > '$offset'
+ AND (scheduled_at IS NULL OR scheduled_at <= '$now')
+ORDER BY created_at
+LIMIT $limit
+```
+
+## Delete After Read / Mark as Processed
+
+### Delete After Read
+
+Deletes rows from the source table after successful processing:
+
+```toml
+[plugin_config]
+delete_after_read = true
+primary_key_column = "id"
+```
+
+### Mark as Processed
+
+Updates a boolean column instead of deleting:
+
+```toml
+[plugin_config]
+processed_column = "is_processed"
+primary_key_column = "id"
+```
+
+Your table needs the boolean column:
+
+```sql
+ALTER TABLE users ADD COLUMN is_processed BOOLEAN DEFAULT false;
+```
+
+When `processed_column` is set, the connector automatically adds a `WHERE is_processed = FALSE` filter to the polling query, so only unprocessed rows are fetched. This improves polling efficiency as the table grows.
+
+## Supported Column Types
+
+The connector handles these PostgreSQL types in JSON mode:
+
+| PostgreSQL Type | JSON Output |
+| --------------- | ----------- |
+| `BOOL` | boolean |
+| `INT2`, `INT4`, `INT8` | number |
+| `FLOAT4`, `FLOAT8` | number |
+| `NUMERIC` | number (parsed as f64) |
+| `VARCHAR`, `TEXT`, `CHAR` | string |
+| `TIMESTAMP`, `TIMESTAMPTZ` | string (RFC3339) |
+| `UUID` | string |
+| `JSON`, `JSONB` | object |
+| `BYTEA` | base64 string |
+| Other | string (fallback) |
+
+## CDC Mode
+
+CDC requires PostgreSQL logical replication setup:
+
+1. Set `wal_level = logical` in `postgresql.conf`
+2. Restart PostgreSQL
+3. Use a direct connection (no pooler)
+
+```toml
+[plugin_config]
+mode = "cdc"
+enable_wal_cdc = true
+tables = ["users", "orders"]
+capture_operations = ["INSERT", "UPDATE", "DELETE"]
+```
+
+The `pg_replicate` backend requires the `cdc_pg_replicate` feature flag at build time.
+
+## Example Configs
+
+### Basic Polling (JSON Mode)
+
+```toml
+[[streams]]
+stream = "user_events"
+topic = "users"
+schema = "json"
+batch_length = 100
+
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/mydb"
+mode = "polling"
+tables = ["users"]
+poll_interval = "1s"
+tracking_column = "updated_at"
+```
+
+### Raw Payload Passthrough
+
+```toml
+[[streams]]
+stream = "messages"
+topic = "queue"
+schema = "raw"
+batch_length = 100
+
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/mydb"
+mode = "polling"
+tables = ["message_queue"]
+poll_interval = "100ms"
+tracking_column = "id"
+payload_column = "payload"
+payload_format = "bytea"
+delete_after_read = true
+```
+
+### JSONB Direct Extraction
+
+```toml
+[[streams]]
+stream = "events"
+topic = "user_events"
+schema = "json"
+batch_length = 100
+
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/mydb"
+mode = "polling"
+tables = ["events"]
+poll_interval = "1s"
+tracking_column = "id"
+payload_column = "data"
+payload_format = "json_direct"
+```
+
+### CDC with Custom Operations
+
+```toml
+[[streams]]
+stream = "audit"
+topic = "changes"
+schema = "json"
+batch_length = 100
+
+[plugin_config]
+connection_string = "postgresql://user:pass@localhost:5432/mydb"
+mode = "cdc"
+enable_wal_cdc = true
+tables = ["users", "orders"]
+capture_operations = ["INSERT", "UPDATE"]
+```
+
+## Reliability Features
+
+### Automatic Retries
+
+The connector automatically retries transient database errors (connection issues, deadlocks, serialization failures) with exponential backoff. Configure with `max_retries` (default: 3) and `retry_delay` (default: `1s`). The actual delay is `retry_delay * attempt_number`. Non-transient errors fail immediately.
+
+### SQL Injection Protection
+
+All table names, column names, and identifiers are properly quoted to prevent SQL injection attacks. User-provided values in tracking offsets are also safely escaped.
+
+## Usage with Sink Connector
+
+The source and sink connectors can work together for pass-through scenarios:
+
+### Raw Bytes Pass-through
+
+1. **Sink** with `payload_format = "bytea"` stores messages as BYTEA
+2. **Source** with `payload_column = "payload"` and `payload_format = "bytea"` reads them back
+
+### JSON Pass-through
-1. Configure the connector in your Iggy connectors runtime
-2. The connector will start polling the specified tables
-3. Data changes will be streamed to the configured Iggy topic
-4. Each row becomes a separate message in JSON format
+1. **Sink** with `payload_format = "json"` stores messages as JSONB
+2. **Source** with `payload_column = "payload"` and `payload_format = "json_direct"` reads JSONB directly
diff --git a/core/integration/tests/connectors/postgres/connectors_config/postgres.toml b/core/connectors/sources/postgres_source/config.toml
similarity index 70%
rename from core/integration/tests/connectors/postgres/connectors_config/postgres.toml
rename to core/connectors/sources/postgres_source/config.toml
index 7476a2126..be53647af 100644
--- a/core/integration/tests/connectors/postgres/connectors_config/postgres.toml
+++ b/core/connectors/sources/postgres_source/config.toml
@@ -15,27 +15,28 @@
# specific language governing permissions and limitations
# under the License.
-type = "sink"
+type = "source"
key = "postgres"
enabled = true
version = 0
-name = "Postgres sink"
-path = "../../target/debug/libiggy_connector_postgres_sink"
+name = "Postgres source"
+path = "../../../target/debug/libiggy_connector_postgres_source"
+verbose = false
[[streams]]
-stream = "test_stream"
-topics = ["test_topic"]
+stream = "user_events"
+topic = "users"
schema = "json"
batch_length = 100
-poll_interval = "5ms"
-consumer_group = "test"
[plugin_config]
-connection_string = ""
-target_table = "iggy_messages"
-batch_size = 100
+connection_string = "postgresql://user:pass@localhost:5432/database"
+mode = "polling"
+tables = ["users", "orders"]
+poll_interval = "1s"
+batch_size = 1000
+tracking_column = "id"
+initial_offset = "0"
max_connections = 10
-auto_create_table = true
+snake_case_columns = false
include_metadata = true
-include_checksum = true
-include_origin_timestamp = true
diff --git a/core/connectors/sources/postgres_source/src/lib.rs b/core/connectors/sources/postgres_source/src/lib.rs
index 27b8e3efa..111101679 100644
--- a/core/connectors/sources/postgres_source/src/lib.rs
+++ b/core/connectors/sources/postgres_source/src/lib.rs
@@ -17,27 +17,34 @@
* under the License.
*/
-use std::{collections::HashMap, str::FromStr, time::Duration};
-
use async_trait::async_trait;
use chrono::{DateTime, Utc};
+use humantime::Duration as HumanDuration;
use iggy_connector_sdk::{
ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source_connector,
};
use serde::{Deserialize, Serialize};
-use sqlx::{Column, Pool, Postgres, Row, TypeInfo, postgres::PgPoolOptions};
+use sqlx::postgres::PgPoolOptions;
+use sqlx::{Column, Pool, Postgres, Row, TypeInfo};
+use std::collections::HashMap;
+use std::str::FromStr;
+use std::time::Duration;
use tokio::sync::Mutex;
-use tracing::{error, info};
+use tracing::{debug, error, info, warn};
use uuid::Uuid;
source_connector!(PostgresSource);
+const DEFAULT_MAX_RETRIES: u32 = 3;
+const DEFAULT_RETRY_DELAY: &str = "1s";
+
#[derive(Debug)]
pub struct PostgresSource {
pub id: u32,
pool: Option<Pool<Postgres>>,
config: PostgresSourceConfig,
state: Mutex<State>,
+ verbose: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -56,8 +63,36 @@ pub struct PostgresSourceConfig {
pub include_metadata: Option<bool>,
pub replication_slot: Option<String>,
pub publication_name: Option<String>,
- pub capture_operations: Option<Vec<String>>, // INSERT, UPDATE, DELETE
- pub cdc_backend: Option<String>, // "builtin" | "pg_replicate"
+ pub capture_operations: Option<Vec<String>>,
+ pub cdc_backend: Option<String>,
+ pub delete_after_read: Option<bool>,
+ pub processed_column: Option<String>,
+ pub primary_key_column: Option<String>,
+ pub payload_column: Option<String>,
+ pub payload_format: Option<String>,
+ pub verbose_logging: Option<bool>,
+ pub max_retries: Option<u32>,
+ pub retry_delay: Option<String>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum PayloadFormat {
+ #[default]
+ Json,
+ Bytea,
+ Text,
+ JsonDirect,
+}
+
+impl PayloadFormat {
+ fn from_config(s: Option<&str>) -> Self {
+ match s.map(|s| s.to_lowercase()).as_deref() {
+ Some("bytea") | Some("raw") => PayloadFormat::Bytea,
+ Some("text") => PayloadFormat::Text,
+ Some("json_direct") | Some("jsonb") | Some("jsonb_direct") => PayloadFormat::JsonDirect,
+ _ => PayloadFormat::Json,
+ }
+ }
}
#[derive(Debug)]
@@ -70,7 +105,7 @@ struct State {
#[derive(Debug, Serialize, Deserialize)]
pub struct DatabaseRecord {
pub table_name: String,
- pub operation_type: String, // INSERT, UPDATE, DELETE
+ pub operation_type: String,
pub timestamp: DateTime<Utc>,
pub data: serde_json::Value,
pub old_data: Option<serde_json::Value>,
@@ -78,6 +113,7 @@ pub struct DatabaseRecord {
impl PostgresSource {
pub fn new(id: u32, config: PostgresSourceConfig, _state: Option<ConnectorState>) -> Self {
+ let verbose = config.verbose_logging.unwrap_or(false);
PostgresSource {
id,
pool: None,
@@ -87,11 +123,119 @@ impl PostgresSource {
tracking_offsets: HashMap::new(),
processed_rows: 0,
}),
+ verbose,
+ }
+ }
+}
+
+#[async_trait]
+impl Source for PostgresSource {
+ async fn open(&mut self) -> Result<(), Error> {
+ info!(
+ "Opening PostgreSQL source connector with ID: {}. Mode: {}, Tables: {:?}",
+ self.id, self.config.mode, self.config.tables
+ );
+
+ self.connect().await?;
+
+ match self.config.mode.as_str() {
+ "cdc" => {
+ self.setup_cdc().await?;
+ let backend = self.config.cdc_backend.as_deref().unwrap_or("builtin");
+ info!(
+ "PostgreSQL CDC mode enabled (backend: {backend}) for connector ID: {}",
+ self.id
+ );
+ }
+ "polling" => {
+ info!(
+ "PostgreSQL polling mode enabled for connector ID: {}",
+ self.id
+ );
+ info!("Poll interval: {:?}", self.get_poll_interval());
+ }
+ _ => {
+ return Err(Error::InitError(format!(
+ "Invalid mode '{}'. Supported modes: 'polling', 'cdc'",
+ self.config.mode
+ )));
+ }
+ }
+
+ info!(
+ "PostgreSQL source connector with ID: {} opened successfully",
+ self.id
+ );
+ Ok(())
+ }
+
+ async fn poll(&self) -> Result<ProducedMessages, Error> {
+ let poll_interval = self.get_poll_interval();
+ tokio::time::sleep(poll_interval).await;
+
+ let messages = match self.config.mode.as_str() {
+ "polling" => self.poll_tables().await?,
+ "cdc" => self.poll_cdc().await?,
+ _ => {
+ error!("Invalid mode: {}", self.config.mode);
+ return Err(Error::InvalidConfig);
+ }
+ };
+
+ let state = self.state.lock().await;
+ if self.verbose {
+ info!(
+ "PostgreSQL source connector ID: {} produced {} messages. Total processed: {}",
+ self.id,
+ messages.len(),
+ state.processed_rows
+ );
+ } else {
+ debug!(
+ "PostgreSQL source connector ID: {} produced {} messages. Total processed: {}",
+ self.id,
+ messages.len(),
+ state.processed_rows
+ );
+ }
+
+ let schema = match self.payload_format() {
+ PayloadFormat::Bytea => Schema::Raw,
+ PayloadFormat::Text => Schema::Text,
+ PayloadFormat::JsonDirect | PayloadFormat::Json => Schema::Json,
+ };
+
+ Ok(ProducedMessages {
+ schema,
+ messages,
+ state: None,
+ })
+ }
+
+ async fn close(&mut self) -> Result<(), Error> {
+ if let Some(pool) = self.pool.take() {
+ pool.close().await;
+ info!(
+ "PostgreSQL connection pool closed for connector ID: {}",
+ self.id
+ );
}
+
+ let state = self.state.lock().await;
+ info!(
+ "PostgreSQL source connector ID: {} closed. Total rows processed: {}",
+ self.id, state.processed_rows
+ );
+ Ok(())
}
+}
+impl PostgresSource {
async fn connect(&mut self) -> Result<(), Error> {
let max_connections = self.config.max_connections.unwrap_or(10);
+ let redacted = redact_connection_string(&self.config.connection_string);
+
+ info!("Connecting to PostgreSQL with max {max_connections} connections: {redacted}");
let pool = PgPoolOptions::new()
.max_connections(max_connections)
@@ -105,10 +249,7 @@ impl PostgresSource {
.map_err(|e| Error::InitError(format!("Database connectivity test failed: {e}")))?;
self.pool = Some(pool);
- info!(
- "Connected to PostgreSQL database with {} max connections",
- max_connections
- );
+ info!("Connected to PostgreSQL database with {max_connections} max connections");
Ok(())
}
@@ -117,10 +258,7 @@ impl PostgresSource {
return Ok(());
}
- let pool = self
- .pool
- .as_ref()
- .ok_or_else(|| Error::InitError("Database not connected".to_string()))?;
+ let pool = self.get_pool()?;
let wal_level: String = sqlx::query_scalar("SHOW wal_level")
.fetch_one(pool)
@@ -166,10 +304,7 @@ impl PostgresSource {
.await
.map_err(|e| Error::InitError(format!("Failed to create replication slot: {e}")))?;
- info!(
- "PostgreSQL CDC setup completed. Publication: {}, Slot: {}",
- publication_name, slot_name
- );
+ info!("PostgreSQL CDC setup completed. Publication: {publication_name}, Slot: {slot_name}");
Ok(())
}
@@ -192,17 +327,13 @@ impl PostgresSource {
}
}
other => Err(Error::InitError(format!(
- "Unsupported cdc_backend '{}'. Use 'builtin' or 'pg_replicate'",
- other
+ "Unsupported cdc_backend '{other}'. Use 'builtin' or 'pg_replicate'"
))),
}
}
async fn poll_cdc_builtin(&self) -> Result<Vec<ProducedMessage>, Error> {
- let pool = self
- .pool
- .as_ref()
- .ok_or_else(|| Error::InitError("Database not connected".to_string()))?;
+ let pool = self.get_pool()?;
let slot_name = self
.config
@@ -229,7 +360,7 @@ impl PostgresSource {
.fetch_all(pool)
.await
.map_err(|e| {
- error!("Failed to fetch CDC changes: {}", e);
+ error!("Failed to fetch CDC changes: {e}");
Error::InvalidRecord
})?;
@@ -248,8 +379,8 @@ impl PostgresSource {
id: Some(Uuid::new_v4().as_u128()),
headers: None,
checksum: None,
- timestamp: Some(chrono::Utc::now().timestamp_millis() as u64),
- origin_timestamp: Some(chrono::Utc::now().timestamp_millis() as u64),
+ timestamp: Some(Utc::now().timestamp_millis() as u64),
+ origin_timestamp: Some(Utc::now().timestamp_millis() as u64),
payload,
};
@@ -258,231 +389,122 @@ impl PostgresSource {
}
}
- info!("CDC: Fetched {} change records", messages.len());
- Ok(messages)
- }
-
- fn parse_logical_replication_message(
- &self,
- data: &str,
- capture_ops: &[&str],
- ) -> Option<DatabaseRecord> {
- if data.starts_with("BEGIN") || data.starts_with("COMMIT") {
- return None;
- }
-
- if data.starts_with("INSERT:") && capture_ops.contains(&"INSERT") {
- return self.parse_insert_message(data);
- }
-
- if data.starts_with("UPDATE:") && capture_ops.contains(&"UPDATE") {
- return self.parse_update_message(data);
- }
-
- if data.starts_with("DELETE:") && capture_ops.contains(&"DELETE") {
- return self.parse_delete_message(data);
- }
-
- None
- }
-
- fn parse_insert_message(&self, data: &str) -> Option<DatabaseRecord> {
- if let Some(table_start) = data.find("table ")
- && let Some(colon_pos) = data[table_start..].find(':')
- {
- let table_part = &data[table_start + 6..table_start + colon_pos];
- let table_name = table_part
- .split('.')
- .next_back()
- .unwrap_or(table_part)
- .to_string();
-
- let data_part = &data[table_start + colon_pos + 1..];
- let parsed_data = self.parse_record_data(data_part);
-
- return Some(DatabaseRecord {
- table_name,
- operation_type: "INSERT".to_string(),
- timestamp: Utc::now(),
- data: serde_json::Value::Object(parsed_data),
- old_data: None,
- });
- }
- None
- }
-
- fn parse_update_message(&self, data: &str) -> Option<DatabaseRecord> {
- if let Some(table_start) = data.find("table ")
- && let Some(colon_pos) = data[table_start..].find(':')
- {
- let table_part = &data[table_start + 6..table_start + colon_pos];
- let table_name = table_part
- .split('.')
- .next_back()
- .unwrap_or(table_part)
- .to_string();
-
- let data_part = &data[table_start + colon_pos + 1..];
- let parsed_data = self.parse_record_data(data_part);
-
- return Some(DatabaseRecord {
- table_name,
- operation_type: "UPDATE".to_string(),
- timestamp: Utc::now(),
- data: serde_json::Value::Object(parsed_data),
- old_data: None,
- });
- }
- None
- }
-
- fn parse_delete_message(&self, data: &str) -> Option<DatabaseRecord> {
- if let Some(table_start) = data.find("table ")
- && let Some(colon_pos) = data[table_start..].find(':')
- {
- let table_part = &data[table_start + 6..table_start + colon_pos];
- let table_name = table_part
- .split('.')
- .next_back()
- .unwrap_or(table_part)
- .to_string();
-
- let data_part = &data[table_start + colon_pos + 1..];
- let parsed_data = self.parse_record_data(data_part);
-
- return Some(DatabaseRecord {
- table_name,
- operation_type: "DELETE".to_string(),
- timestamp: Utc::now(),
- data: serde_json::Value::Object(parsed_data),
- old_data: None,
- });
- }
- None
- }
-
- fn parse_record_data(&self, data: &str) -> serde_json::Map<String, serde_json::Value> {
- let mut result = serde_json::Map::new();
-
- let parts = data.split_whitespace();
-
- for part in parts {
- if let Some(bracket_pos) = part.find('[')
- && let Some(_close_bracket) = part.find(']')
- && let Some(colon_pos) = part.find(':')
- {
- let column_name = &part[..bracket_pos];
- let value_str = &part[colon_pos + 1..];
-
- let cleaned_value = if value_str.starts_with('\'') && value_str.ends_with('\'') {
- &value_str[1..value_str.len() - 1]
- } else {
- value_str
- };
-
- let value = if let Ok(num) = cleaned_value.parse::<i64>() {
- serde_json::Value::Number(serde_json::Number::from(num))
- } else if let Ok(float) = cleaned_value.parse::<f64>() {
- serde_json::Value::Number(
- serde_json::Number::from_f64(float).unwrap_or(serde_json::Number::from(0)),
- )
- } else if cleaned_value.eq_ignore_ascii_case("true") {
- serde_json::Value::Bool(true)
- } else if cleaned_value.eq_ignore_ascii_case("false") {
- serde_json::Value::Bool(false)
- } else {
- serde_json::Value::String(cleaned_value.to_string())
- };
-
- result.insert(column_name.to_string(), value);
- }
+ if self.verbose {
+ info!("CDC: Fetched {} change records", messages.len());
+ } else {
+ debug!("CDC: Fetched {} change records", messages.len());
}
-
- result
+ Ok(messages)
}
async fn poll_tables(&self) -> Result<Vec<ProducedMessage>, Error> {
- let pool = self
- .pool
- .as_ref()
- .ok_or_else(|| Error::InitError("Database not connected".to_string()))?;
+ let pool = self.get_pool()?;
let mut state = self.state.lock().await;
let mut messages = Vec::new();
let batch_size = self.config.batch_size.unwrap_or(1000);
let tracking_column = self.config.tracking_column.as_deref().unwrap_or("id");
+ let pk_column = self
+ .config
+ .primary_key_column
+ .as_deref()
+ .unwrap_or(tracking_column);
+ let payload_format = self.payload_format();
+ let payload_col = self.config.payload_column.as_deref().unwrap_or("");
for table in &self.config.tables {
let last_offset = state.tracking_offsets.get(table).cloned();
let query = if let Some(custom_query) = &self.config.custom_query {
- custom_query.clone()
+ self.validate_custom_query(custom_query)?;
+ self.substitute_query_params(custom_query, table, &last_offset, batch_size)
} else {
- self.build_polling_query(table, tracking_column, &last_offset, batch_size)
+ self.build_polling_query(table, tracking_column, &last_offset, batch_size)?
};
- info!("Executing query for table '{}': {}", table, query);
-
- let rows = sqlx::query(&query)
- .fetch_all(pool)
- .await
- .map_err(|_| Error::InvalidRecord)?;
+ let rows = with_retry(
+ || sqlx::query(&query).fetch_all(pool),
+ self.get_max_retries(),
+ self.get_retry_delay().as_millis() as u64,
+ )
+ .await?;
let mut max_offset: Option<String> = None;
+ let mut processed_ids: Vec<String> = Vec::new();
for row in rows {
+ let mut row_pk: Option<String> = None;
+ let mut extracted_payload: Option<Vec<u8>> = None;
let mut data = serde_json::Map::new();
for (i, column) in row.columns().iter().enumerate() {
let column_name = if self.config.snake_case_columns.unwrap_or(false) {
- self.to_snake_case(column.name())
+ to_snake_case(column.name())
} else {
column.name().to_string()
};
+ if !payload_col.is_empty() && column.name() == payload_col {
+ extracted_payload =
+ Some(self.extract_payload_column(&row, i, payload_format)?);
+ continue;
+ }
+
let value = self.extract_column_value(&row, i)?;
- data.insert(column_name.clone(), value);
+ data.insert(column_name.clone(), value.clone());
if column.name() == tracking_column {
- if let Some(serde_json::Value::String(offset_str)) = data.get(&column_name)
- {
- max_offset = Some(offset_str.clone());
- } else if let Some(serde_json::Value::Number(offset_num)) =
- data.get(&column_name)
- {
- max_offset = Some(offset_num.to_string());
+ if let serde_json::Value::String(ref s) = value {
+ max_offset = Some(s.clone());
+ } else if let serde_json::Value::Number(ref n) = value {
+ max_offset = Some(n.to_string());
}
}
- }
- let record = if self.config.include_metadata.unwrap_or(true) {
- DatabaseRecord {
- table_name: table.clone(),
- operation_type: "SELECT".to_string(),
- timestamp: Utc::now(),
- data: serde_json::Value::Object(data),
- old_data: None,
+ if column.name() == pk_column {
+ if let serde_json::Value::String(ref s) = value {
+ row_pk = Some(s.clone());
+ } else if let serde_json::Value::Number(ref n) = value {
+ row_pk = Some(n.to_string());
+ }
}
+ }
+
+ if let Some(pk) = row_pk {
+ processed_ids.push(pk);
+ }
+
+ let payload = if let Some(bytes) = extracted_payload {
+ bytes
} else {
- let mut simple_record = serde_json::Map::new();
- simple_record.insert("data".to_string(), serde_json::Value::Object(data));
- DatabaseRecord {
- table_name: table.clone(),
- operation_type: "SELECT".to_string(),
- timestamp: Utc::now(),
- data: serde_json::Value::Object(simple_record),
- old_data: None,
- }
+ let record = if self.config.include_metadata.unwrap_or(true) {
+ DatabaseRecord {
+ table_name: table.clone(),
+ operation_type: "SELECT".to_string(),
+ timestamp: Utc::now(),
+ data: serde_json::Value::Object(data),
+ old_data: None,
+ }
+ } else {
+ let mut simple_record = serde_json::Map::new();
+ simple_record.insert("data".to_string(), serde_json::Value::Object(data));
+ DatabaseRecord {
+ table_name: table.clone(),
+ operation_type: "SELECT".to_string(),
+ timestamp: Utc::now(),
+ data: serde_json::Value::Object(simple_record),
+ old_data: None,
+ }
+ };
+ simd_json::to_vec(&record).map_err(|_| Error::InvalidRecord)?
};
- let payload = simd_json::to_vec(&record).map_err(|_| Error::InvalidRecord)?;
-
let message = ProducedMessage {
id: Some(Uuid::new_v4().as_u128()),
headers: None,
checksum: None,
- timestamp: Some(chrono::Utc::now().timestamp_millis() as u64),
- origin_timestamp: Some(chrono::Utc::now().timestamp_millis() as u64),
+ timestamp: Some(Utc::now().timestamp_millis() as u64),
+ origin_timestamp: Some(Utc::now().timestamp_millis() as u64),
payload,
};
@@ -490,42 +512,344 @@ impl PostgresSource {
state.processed_rows += 1;
}
+ if !processed_ids.is_empty() {
+ self.mark_or_delete_processed_rows(pool, table, pk_column, &processed_ids)
+ .await?;
+ }
+
if let Some(offset) = max_offset {
state.tracking_offsets.insert(table.clone(), offset);
}
- info!("Fetched {} rows from table '{}'", messages.len(), table);
+ if self.verbose {
+ info!("Fetched {} rows from table '{table}'", messages.len());
+ } else {
+ debug!("Fetched {} rows from table '{table}'", messages.len());
+ }
}
state.last_poll_time = Utc::now();
Ok(messages)
}
- fn build_polling_query(
+ async fn mark_or_delete_processed_rows(
&self,
+ pool: &Pool<Postgres>,
table: &str,
- tracking_column: &str,
- last_offset: &Option<String>,
- batch_size: u32,
- ) -> String {
- let base_query = format!("SELECT * FROM {table}");
+ pk_column: &str,
+ ids: &[String],
+ ) -> Result<(), Error> {
+ if ids.is_empty() {
+ return Ok(());
+ }
- let where_clause = if let Some(offset) = last_offset {
- format!(" WHERE {tracking_column} > '{offset}'")
- } else if let Some(initial) = &self.config.initial_offset {
- format!(" WHERE {tracking_column} > '{initial}'")
- } else {
- String::new()
- };
+ let quoted_table = quote_identifier(table)?;
+ let quoted_pk = quote_identifier(pk_column)?;
- let order_clause = format!(" ORDER BY {tracking_column} ASC");
- let limit_clause = format!(" LIMIT {batch_size}");
+ let ids_list = ids
+ .iter()
+ .map(|id| {
+ if id.parse::<i64>().is_ok() {
+ id.clone()
+ } else {
+ format!("'{}'", id.replace('\'', "''"))
+ }
+ })
+ .collect::<Vec<_>>()
+ .join(", ");
- format!("{base_query}{where_clause}{order_clause}{limit_clause}")
- }
+ if self.config.delete_after_read.unwrap_or(false) {
+ let delete_query =
+ format!("DELETE FROM {quoted_table} WHERE {quoted_pk} IN ({ids_list})");
- fn extract_column_value(
- &self,
+ if self.verbose {
+ info!("Deleting {} processed rows from '{table}'", ids.len());
+ } else {
+ debug!("Deleting {} processed rows from '{table}'", ids.len());
+ }
+
+ sqlx::query(&delete_query)
+ .execute(pool)
+ .await
+ .map_err(|e| {
+ error!("Failed to delete processed rows: {e}");
+ Error::InvalidRecord
+ })?;
+ } else if let Some(processed_col) = &self.config.processed_column {
+ let quoted_processed = quote_identifier(processed_col)?;
+ let update_query = format!(
+ "UPDATE {quoted_table} SET {quoted_processed} = TRUE WHERE {quoted_pk} IN ({ids_list})"
+ );
+
+ if self.verbose {
+ info!("Marking {} rows as processed in '{table}'", ids.len());
+ } else {
+ debug!("Marking {} rows as processed in '{table}'", ids.len());
+ }
+
+ sqlx::query(&update_query)
+ .execute(pool)
+ .await
+ .map_err(|e| {
+ error!("Failed to mark rows as processed: {e}");
+ Error::InvalidRecord
+ })?;
+ }
+
+ Ok(())
+ }
+
+ fn get_pool(&self) -> Result<&Pool<Postgres>, Error> {
+ self.pool
+ .as_ref()
+ .ok_or_else(|| Error::InitError("Database not connected".to_string()))
+ }
+
+ fn payload_format(&self) -> PayloadFormat {
+ if let Some(ref payload_col) = self.config.payload_column
+ && !payload_col.is_empty()
+ {
+ return PayloadFormat::from_config(self.config.payload_format.as_deref());
+ }
+ PayloadFormat::Json
+ }
+
+ fn get_poll_interval(&self) -> Duration {
+ let interval_str = self.config.poll_interval.as_deref().unwrap_or("10s");
+ HumanDuration::from_str(interval_str)
+ .unwrap_or_else(|_| std::time::Duration::from_secs(10).into())
+ .into()
+ }
+
+ fn get_max_retries(&self) -> u32 {
+ self.config.max_retries.unwrap_or(DEFAULT_MAX_RETRIES)
+ }
+
+ fn get_retry_delay(&self) -> Duration {
+ let delay_str = self
+ .config
+ .retry_delay
+ .as_deref()
+ .unwrap_or(DEFAULT_RETRY_DELAY);
+ HumanDuration::from_str(delay_str)
+ .unwrap_or_else(|e| panic!("Invalid retry_delay '{delay_str}': {e}"))
+ .into()
+ }
+
+ fn build_polling_query(
+ &self,
+ table: &str,
+ tracking_column: &str,
+ last_offset: &Option<String>,
+ batch_size: u32,
+ ) -> Result<String, Error> {
+ let quoted_table = quote_identifier(table)?;
+ let quoted_tracking = quote_identifier(tracking_column)?;
+
+ let base_query = format!("SELECT * FROM {quoted_table}");
+
+ let mut conditions = Vec::new();
+
+ if let Some(offset) = last_offset {
+ conditions.push(format!(
+ "{quoted_tracking} > {}",
+ format_offset_value(offset)
+ ));
+ } else if let Some(initial) = &self.config.initial_offset {
+ conditions.push(format!(
+ "{quoted_tracking} > {}",
+ format_offset_value(initial)
+ ));
+ }
+
+ if let Some(processed_col) = &self.config.processed_column {
+ let quoted_processed = quote_identifier(processed_col)?;
+ conditions.push(format!("{quoted_processed} = FALSE"));
+ }
+
+ let where_clause = if conditions.is_empty() {
+ String::new()
+ } else {
+ format!(" WHERE {}", conditions.join(" AND "))
+ };
+
+ let order_clause = format!(" ORDER BY {quoted_tracking} ASC");
+ let limit_clause = format!(" LIMIT {batch_size}");
+
+ Ok(format!(
+ "{base_query}{where_clause}{order_clause}{limit_clause}"
+ ))
+ }
+
+ fn validate_custom_query(&self, query: &str) -> Result<(), Error> {
+ let query_upper = query.to_uppercase();
+ if !query_upper.contains("SELECT") {
+ warn!("Custom query should contain SELECT statement");
+ }
+ if query.contains("$table") && self.config.tables.is_empty() {
+ return Err(Error::InvalidConfig);
+ }
+ Ok(())
+ }
+
+ fn substitute_query_params(
+ &self,
+ query: &str,
+ table: &str,
+ last_offset: &Option<String>,
+ batch_size: u32,
+ ) -> String {
+ let offset_value = last_offset
+ .clone()
+ .or_else(|| self.config.initial_offset.clone())
+ .unwrap_or_default();
+
+ let now = Utc::now();
+
+ query
+ .replace("$table", table)
+ .replace("$offset", &offset_value)
+ .replace("$limit", &batch_size.to_string())
+ .replace("$now", &now.to_rfc3339())
+ .replace("$now_unix", &now.timestamp().to_string())
+ }
+
+ fn parse_logical_replication_message(
+ &self,
+ data: &str,
+ capture_ops: &[&str],
+ ) -> Option<DatabaseRecord> {
+ if data.starts_with("BEGIN") || data.starts_with("COMMIT") {
+ return None;
+ }
+
+ if data.starts_with("INSERT:") && capture_ops.contains(&"INSERT") {
+ return self.parse_insert_message(data);
+ }
+
+ if data.starts_with("UPDATE:") && capture_ops.contains(&"UPDATE") {
+ return self.parse_update_message(data);
+ }
+
+ if data.starts_with("DELETE:") && capture_ops.contains(&"DELETE") {
+ return self.parse_delete_message(data);
+ }
+
+ None
+ }
+
+ fn parse_insert_message(&self, data: &str) -> Option<DatabaseRecord> {
+ if let Some(table_start) = data.find("table ")
+ && let Some(colon_pos) = data[table_start..].find(':')
+ {
+ let table_part = &data[table_start + 6..table_start + colon_pos];
+ let table_name = table_part
+ .split('.')
+ .next_back()
+ .unwrap_or(table_part)
+ .to_string();
+
+ let data_part = &data[table_start + colon_pos + 1..];
+ let parsed_data = parse_record_data(data_part);
+
+ return Some(DatabaseRecord {
+ table_name,
+ operation_type: "INSERT".to_string(),
+ timestamp: Utc::now(),
+ data: serde_json::Value::Object(parsed_data),
+ old_data: None,
+ });
+ }
+ None
+ }
+
+ fn parse_update_message(&self, data: &str) -> Option<DatabaseRecord> {
+ if let Some(table_start) = data.find("table ")
+ && let Some(colon_pos) = data[table_start..].find(':')
+ {
+ let table_part = &data[table_start + 6..table_start + colon_pos];
+ let table_name = table_part
+ .split('.')
+ .next_back()
+ .unwrap_or(table_part)
+ .to_string();
+
+ let data_part = &data[table_start + colon_pos + 1..];
+ let parsed_data = parse_record_data(data_part);
+
+ return Some(DatabaseRecord {
+ table_name,
+ operation_type: "UPDATE".to_string(),
+ timestamp: Utc::now(),
+ data: serde_json::Value::Object(parsed_data),
+ old_data: None,
+ });
+ }
+ None
+ }
+
+ fn parse_delete_message(&self, data: &str) -> Option<DatabaseRecord> {
+ if let Some(table_start) = data.find("table ")
+ && let Some(colon_pos) = data[table_start..].find(':')
+ {
+ let table_part = &data[table_start + 6..table_start + colon_pos];
+ let table_name = table_part
+ .split('.')
+ .next_back()
+ .unwrap_or(table_part)
+ .to_string();
+
+ let data_part = &data[table_start + colon_pos + 1..];
+ let parsed_data = parse_record_data(data_part);
+
+ return Some(DatabaseRecord {
+ table_name,
+ operation_type: "DELETE".to_string(),
+ timestamp: Utc::now(),
+ data: serde_json::Value::Object(parsed_data),
+ old_data: None,
+ });
+ }
+ None
+ }
+
+ fn extract_payload_column(
+ &self,
+ row: &sqlx::postgres::PgRow,
+ column_index: usize,
+ format: PayloadFormat,
+ ) -> Result<Vec<u8>, Error> {
+ match format {
+ PayloadFormat::Bytea => {
+ let bytes: Option<Vec<u8>> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(bytes.unwrap_or_default())
+ }
+ PayloadFormat::Text => {
+ let text: Option<String> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(text.unwrap_or_default().into_bytes())
+ }
+ PayloadFormat::JsonDirect => {
+ let json_value: Option<serde_json::Value> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ simd_json::to_vec(&json_value.unwrap_or(serde_json::Value::Null))
+ .map_err(|_| Error::InvalidRecord)
+ }
+ PayloadFormat::Json => {
+ let bytes: Option<Vec<u8>> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(bytes.unwrap_or_default())
+ }
+ }
+ }
+
+ fn extract_column_value(
+ &self,
row: &sqlx::postgres::PgRow,
column_index: usize,
) -> Result<serde_json::Value, Error> {
@@ -541,7 +865,23 @@ impl PostgresSource {
.map(serde_json::Value::Bool)
.unwrap_or(serde_json::Value::Null))
}
- "INT2" | "INT4" | "INT8" => {
+ "INT2" => {
+ let value: Option<i16> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(value
+ .map(|v| serde_json::Value::from(v as i64))
+ .unwrap_or(serde_json::Value::Null))
+ }
+ "INT4" => {
+ let value: Option<i32> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(value
+ .map(|v| serde_json::Value::from(v as i64))
+ .unwrap_or(serde_json::Value::Null))
+ }
+ "INT8" => {
let value: Option<i64> = row
.try_get(column_index)
.map_err(|_| Error::InvalidRecord)?;
@@ -549,7 +889,15 @@ impl PostgresSource {
.map(serde_json::Value::from)
.unwrap_or(serde_json::Value::Null))
}
- "FLOAT4" | "FLOAT8" | "NUMERIC" => {
+ "FLOAT4" => {
+ let value: Option<f32> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(value
+ .map(|v| serde_json::Value::from(v as f64))
+ .unwrap_or(serde_json::Value::Null))
+ }
+ "FLOAT8" => {
let value: Option<f64> = row
.try_get(column_index)
.map_err(|_| Error::InvalidRecord)?;
@@ -557,6 +905,15 @@ impl PostgresSource {
.map(serde_json::Value::from)
.unwrap_or(serde_json::Value::Null))
}
+ "NUMERIC" => {
+ let value: Option<String> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(value
+ .and_then(|s| s.parse::<f64>().ok())
+ .map(serde_json::Value::from)
+ .unwrap_or(serde_json::Value::Null))
+ }
"VARCHAR" | "TEXT" | "CHAR" => {
let value: Option<String> = row
.try_get(column_index)
@@ -587,6 +944,19 @@ impl PostgresSource {
.map_err(|_| Error::InvalidRecord)?;
Ok(value.unwrap_or(serde_json::Value::Null))
}
+ "BYTEA" => {
+ let value: Option<Vec<u8>> = row
+ .try_get(column_index)
+ .map_err(|_| Error::InvalidRecord)?;
+ Ok(value
+ .map(|bytes| {
+ use base64::Engine;
+ serde_json::Value::String(
+ base64::engine::general_purpose::STANDARD.encode(&bytes),
+ )
+ })
+ .unwrap_or(serde_json::Value::Null))
+ }
_ => {
let value: Option<String> = row
.try_get(column_index)
@@ -597,217 +967,238 @@ impl PostgresSource {
}
}
}
+}
- fn to_snake_case(&self, input: &str) -> String {
- let mut result = String::new();
- let mut prev_was_uppercase = false;
+fn quote_identifier(name: &str) -> Result<String, Error> {
+ if name.is_empty() {
+ return Err(Error::InvalidConfig);
+ }
+ if name.contains('\0') {
+ return Err(Error::InvalidConfig);
+ }
+ let escaped = name.replace('"', "\"\"");
+ Ok(format!("\"{escaped}\""))
+}
- for (i, ch) in input.chars().enumerate() {
- if ch.is_uppercase() {
- if i > 0 && !prev_was_uppercase {
- result.push('_');
- }
- if let Some(lowercase_ch) = ch.to_lowercase().next() {
- result.push(lowercase_ch);
- } else {
- result.push(ch);
- }
- prev_was_uppercase = true;
+fn format_offset_value(value: &str) -> String {
+ if value.parse::<i64>().is_ok() || value.parse::<f64>().is_ok() {
+ value.to_string()
+ } else {
+ format!("'{}'", value.replace('\'', "''"))
+ }
+}
+
+fn to_snake_case(input: &str) -> String {
+ let mut result = String::new();
+ let mut prev_was_uppercase = false;
+
+ for (i, ch) in input.chars().enumerate() {
+ if ch.is_uppercase() {
+ if i > 0 && !prev_was_uppercase {
+ result.push('_');
+ }
+ if let Some(lowercase_ch) = ch.to_lowercase().next() {
+ result.push(lowercase_ch);
} else {
result.push(ch);
- prev_was_uppercase = false;
}
+ prev_was_uppercase = true;
+ } else {
+ result.push(ch);
+ prev_was_uppercase = false;
}
-
- result
}
- fn get_poll_interval(&self) -> Duration {
- let interval_str = self.config.poll_interval.as_deref().unwrap_or("10s");
- humantime::Duration::from_str(interval_str)
- .unwrap_or_else(|_| std::time::Duration::from_secs(10).into())
- .into()
- }
+ result
}
-#[async_trait]
-impl Source for PostgresSource {
- async fn open(&mut self) -> Result<(), Error> {
- info!(
- "Opening PostgreSQL source connector with ID: {}. Mode: {}, Tables: {:?}",
- self.id, self.config.mode, self.config.tables
- );
+fn parse_record_data(data: &str) -> serde_json::Map<String, serde_json::Value> {
+ let mut result = serde_json::Map::new();
- self.connect().await?;
+ for part in data.split_whitespace() {
+ if let Some(bracket_pos) = part.find('[')
+ && let Some(_close_bracket) = part.find(']')
+ && let Some(colon_pos) = part.find(':')
+ {
+ let column_name = &part[..bracket_pos];
+ let value_str = &part[colon_pos + 1..];
- match self.config.mode.as_str() {
- "cdc" => {
- self.setup_cdc().await?;
- let backend = self.config.cdc_backend.as_deref().unwrap_or("builtin");
- info!(
- "PostgreSQL CDC mode enabled (backend: {}) for connector ID: {}",
- backend, self.id
- );
- }
- "polling" => {
- info!(
- "PostgreSQL polling mode enabled for connector ID: {}",
- self.id
- );
- info!("Poll interval: {:?}", self.get_poll_interval());
- }
- _ => {
- return Err(Error::InitError(format!(
- "Invalid mode '{}'. Supported modes: 'polling', 'cdc'",
- self.config.mode
- )));
- }
- }
+ let cleaned_value = if value_str.starts_with('\'') && value_str.ends_with('\'') {
+ &value_str[1..value_str.len() - 1]
+ } else {
+ value_str
+ };
- info!(
- "PostgreSQL source connector with ID: {} opened successfully",
- self.id
- );
- Ok(())
+ let value = if let Ok(num) = cleaned_value.parse::<i64>() {
+ serde_json::Value::Number(serde_json::Number::from(num))
+ } else if let Ok(float) = cleaned_value.parse::<f64>() {
+ serde_json::Value::Number(
+ serde_json::Number::from_f64(float).unwrap_or(serde_json::Number::from(0)),
+ )
+ } else if cleaned_value.eq_ignore_ascii_case("true") {
+ serde_json::Value::Bool(true)
+ } else if cleaned_value.eq_ignore_ascii_case("false") {
+ serde_json::Value::Bool(false)
+ } else {
+ serde_json::Value::String(cleaned_value.to_string())
+ };
+
+ result.insert(column_name.to_string(), value);
+ }
}
- async fn poll(&self) -> Result<ProducedMessages, Error> {
- let poll_interval = self.get_poll_interval();
- tokio::time::sleep(poll_interval).await;
+ result
+}
- let messages = match self.config.mode.as_str() {
- "polling" => self.poll_tables().await?,
- "cdc" => self.poll_cdc().await?,
- _ => {
- error!("Invalid mode: {}", self.config.mode);
- return Err(Error::InvalidConfig);
+async fn with_retry<T, F, Fut>(operation: F, max_retries: u32, delay_ms: u64) -> Result<T, Error>
+where
+ F: Fn() -> Fut,
+ Fut: std::future::Future<Output = Result<T, sqlx::Error>>,
+{
+ let mut attempts = 0;
+ loop {
+ match operation().await {
+ Ok(result) => return Ok(result),
+ Err(e) => {
+ attempts += 1;
+ if attempts >= max_retries || !is_transient_error(&e) {
+ error!("Database operation failed after {attempts} attempts: {e}");
+ return Err(Error::InvalidRecord);
+ }
+ warn!(
+ "Transient database error (attempt {attempts}/{max_retries}): {e}. Retrying in {delay_ms}ms..."
+ );
+ tokio::time::sleep(Duration::from_millis(delay_ms * attempts as u64)).await;
}
- };
-
- let state = self.state.lock().await;
- info!(
- "PostgreSQL source connector ID: {} produced {} messages. Total processed: {}",
- self.id,
- messages.len(),
- state.processed_rows
- );
-
- Ok(ProducedMessages {
- schema: Schema::Json,
- messages,
- state: None,
- })
+ }
}
+}
- async fn close(&mut self) -> Result<(), Error> {
- if let Some(pool) = self.pool.take() {
- pool.close().await;
- info!(
- "PostgreSQL connection pool closed for connector ID: {}",
- self.id
- );
- }
+fn is_transient_error(e: &sqlx::Error) -> bool {
+ match e {
+ sqlx::Error::Io(_) => true,
+ sqlx::Error::PoolTimedOut => true,
+ sqlx::Error::PoolClosed => false,
+ sqlx::Error::Protocol(_) => false,
+ sqlx::Error::Database(db_err) => db_err.code().is_some_and(|code| {
+ matches!(
+ code.as_ref(),
+ "40001" | "40P01" | "57P01" | "57P02" | "57P03" | "08000" | "08003" | "08006"
+ )
+ }),
+ _ => false,
+ }
+}
- let state = self.state.lock().await;
- info!(
- "PostgreSQL source connector ID: {} closed. Total rows processed: {}",
- self.id, state.processed_rows
- );
- Ok(())
+fn redact_connection_string(conn_str: &str) -> String {
+ if let Some(scheme_end) = conn_str.find("://") {
+ let scheme = &conn_str[..scheme_end + 3];
+ let rest = &conn_str[scheme_end + 3..];
+ let preview: String = rest.chars().take(3).collect();
+ return format!("{scheme}{preview}***");
}
+ let preview: String = conn_str.chars().take(3).collect();
+ format!("{preview}***")
}
#[cfg(test)]
mod tests {
use super::*;
+ fn test_config() -> PostgresSourceConfig {
+ PostgresSourceConfig {
+ connection_string: "postgres://localhost/db".to_string(),
+ mode: "polling".to_string(),
+ tables: vec!["users".to_string()],
+ poll_interval: Some("5s".to_string()),
+ batch_size: Some(500),
+ tracking_column: Some("updated_at".to_string()),
+ initial_offset: None,
+ max_connections: None,
+ enable_wal_cdc: None,
+ custom_query: None,
+ snake_case_columns: None,
+ include_metadata: None,
+ replication_slot: None,
+ publication_name: None,
+ capture_operations: None,
+ cdc_backend: None,
+ delete_after_read: None,
+ processed_column: None,
+ primary_key_column: None,
+ payload_column: None,
+ payload_format: None,
+ verbose_logging: None,
+ max_retries: None,
+ retry_delay: None,
+ }
+ }
+
#[test]
fn given_last_offset_polling_query_should_be_built() {
- let src = PostgresSource::new(
- 1,
- PostgresSourceConfig {
- connection_string: "postgres://localhost/db".to_string(),
- mode: "polling".to_string(),
- tables: vec!["users".to_string()],
- poll_interval: Some("5s".to_string()),
- batch_size: Some(500),
- tracking_column: Some("updated_at".to_string()),
- initial_offset: None,
- max_connections: None,
- enable_wal_cdc: None,
- custom_query: None,
- snake_case_columns: None,
- include_metadata: None,
- replication_slot: None,
- publication_name: None,
- capture_operations: None,
- cdc_backend: None,
- },
- None,
- );
- let query =
- src.build_polling_query("users", "updated_at", &Some("2024-01-01".to_string()), 500);
+ let src = PostgresSource::new(1, test_config(), None);
+ let query = src
+ .build_polling_query("users", "updated_at", &Some("2024-01-01".to_string()), 500)
+ .expect("Failed to build query");
assert_eq!(
query,
- "SELECT * FROM users WHERE updated_at > '2024-01-01' ORDER BY updated_at ASC LIMIT 500"
+ "SELECT * FROM \"users\" WHERE \"updated_at\" > '2024-01-01' ORDER BY \"updated_at\" ASC LIMIT 500"
);
}
#[test]
fn given_initial_offset_polling_query_should_be_built() {
- let src = PostgresSource::new(
- 1,
- PostgresSourceConfig {
- connection_string: "postgres://localhost/db".to_string(),
- mode: "polling".to_string(),
- tables: vec!["users".to_string()],
- poll_interval: Some("5s".to_string()),
- batch_size: Some(1000),
- tracking_column: Some("id".to_string()),
- initial_offset: Some("100".to_string()),
- max_connections: None,
- enable_wal_cdc: None,
- custom_query: None,
- snake_case_columns: None,
- include_metadata: None,
- replication_slot: None,
- publication_name: None,
- capture_operations: None,
- cdc_backend: None,
- },
- None,
- );
- let query = src.build_polling_query("users", "id", &None, 1000);
+ let mut config = test_config();
+ config.tracking_column = Some("id".to_string());
+ config.initial_offset = Some("100".to_string());
+ let src = PostgresSource::new(1, config, None);
+ let query = src
+ .build_polling_query("users", "id", &None, 1000)
+ .expect("Failed to build query");
assert_eq!(
query,
- "SELECT * FROM users WHERE id > '100' ORDER BY id ASC LIMIT 1000"
+ "SELECT * FROM \"users\" WHERE \"id\" > 100 ORDER BY \"id\" ASC LIMIT 1000"
);
}
+ #[test]
+ fn given_processed_column_polling_query_should_include_filter() {
+ let mut config = test_config();
+ config.processed_column = Some("is_processed".to_string());
+ let src = PostgresSource::new(1, config, None);
+ let query = src
+ .build_polling_query("events", "id", &None, 100)
+ .expect("Failed to build query");
+ assert!(query.contains("\"is_processed\" = FALSE"));
+ }
+
+ #[test]
+ fn given_numeric_offset_should_not_quote_value() {
+ let src = PostgresSource::new(1, test_config(), None);
+ let query = src
+ .build_polling_query("users", "id", &Some("42".to_string()), 100)
+ .expect("Failed to build query");
+ assert!(query.contains("\"id\" > 42"));
+ assert!(!query.contains("'42'"));
+ }
+
+ #[test]
+ fn given_special_chars_in_identifier_should_escape() {
+ let result = quote_identifier("table\"name").expect("Failed to quote");
+ assert_eq!(result, "\"table\"\"name\"");
+ }
+
+ #[test]
+ fn given_empty_identifier_should_fail() {
+ let result = quote_identifier("");
+ assert!(result.is_err());
+ }
+
#[test]
fn given_insert_message_should_parse_correctly() {
- let src = PostgresSource::new(
- 1,
- PostgresSourceConfig {
- connection_string: String::new(),
- mode: "cdc".to_string(),
- tables: vec![],
- poll_interval: None,
- batch_size: None,
- tracking_column: None,
- initial_offset: None,
- max_connections: None,
- enable_wal_cdc: None,
- custom_query: None,
- snake_case_columns: None,
- include_metadata: None,
- replication_slot: None,
- publication_name: None,
- capture_operations: None,
- cdc_backend: None,
- },
- None,
- );
+ let mut config = test_config();
+ config.mode = "cdc".to_string();
+ let src = PostgresSource::new(1, config, None);
let data = "INSERT: table public.users: id[1] name['Alice'] active[true]";
let rec = src
@@ -819,28 +1210,9 @@ mod tests {
#[test]
fn given_update_message_should_parse_correctly() {
- let src = PostgresSource::new(
- 1,
- PostgresSourceConfig {
- connection_string: String::new(),
- mode: "cdc".to_string(),
- tables: vec![],
- poll_interval: None,
- batch_size: None,
- tracking_column: None,
- initial_offset: None,
- max_connections: None,
- enable_wal_cdc: None,
- custom_query: None,
- snake_case_columns: None,
- include_metadata: None,
- replication_slot: None,
- publication_name: None,
- capture_operations: None,
- cdc_backend: None,
- },
- None,
- );
+ let mut config = test_config();
+ config.mode = "cdc".to_string();
+ let src = PostgresSource::new(1, config, None);
let data = "UPDATE: table public.orders: id[42] total[99.5]";
let rec = src
@@ -852,28 +1224,9 @@ mod tests {
#[test]
fn given_delete_message_should_parse_correctly() {
- let src = PostgresSource::new(
- 1,
- PostgresSourceConfig {
- connection_string: String::new(),
- mode: "cdc".to_string(),
- tables: vec![],
- poll_interval: None,
- batch_size: None,
- tracking_column: None,
- initial_offset: None,
- max_connections: None,
- enable_wal_cdc: None,
- custom_query: None,
- snake_case_columns: None,
- include_metadata: None,
- replication_slot: None,
- publication_name: None,
- capture_operations: None,
- cdc_backend: None,
- },
- None,
- );
+ let mut config = test_config();
+ config.mode = "cdc".to_string();
+ let src = PostgresSource::new(1, config, None);
let data = "DELETE: table public.products: id[7]";
let rec = src
@@ -882,4 +1235,62 @@ mod tests {
assert_eq!(rec.table_name, "products");
assert_eq!(rec.operation_type, "DELETE");
}
+
+ #[test]
+ fn given_custom_query_params_should_substitute_correctly() {
+ let mut config = test_config();
+ config.initial_offset = Some("0".to_string());
+ let src = PostgresSource::new(1, config, None);
+
+ let query = "SELECT * FROM $table WHERE id > $offset ORDER BY id LIMIT $limit";
+ let result = src.substitute_query_params(query, "events", &Some("100".to_string()), 50);
+
+ assert!(result.contains("FROM events"));
+ assert!(result.contains("id > 100"));
+ assert!(result.contains("LIMIT 50"));
+ }
+
+ #[test]
+ fn given_custom_query_with_time_params_should_substitute_correctly() {
+ let src = PostgresSource::new(1, test_config(), None);
+
+ let query = "SELECT * FROM $table WHERE created_at < '$now'";
+ let result = src.substitute_query_params(query, "logs", &None, 100);
+
+ assert!(result.contains("FROM logs"));
+ assert!(!result.contains("$now"));
+ }
+
+ #[test]
+ fn given_no_last_offset_should_use_initial_offset() {
+ let mut config = test_config();
+ config.initial_offset = Some("500".to_string());
+ let src = PostgresSource::new(1, config, None);
+
+ let query = "SELECT * FROM $table WHERE id > $offset";
+ let result = src.substitute_query_params(query, "data", &None, 100);
+
+ assert!(result.contains("id > 500"));
+ }
+
+ #[test]
+ fn given_connection_string_with_credentials_should_redact() {
+ let conn = "postgres://user:password@localhost:5432/db";
+ let redacted = redact_connection_string(conn);
+ assert_eq!(redacted, "postgres://use***");
+ }
+
+ #[test]
+ fn given_connection_string_without_scheme_should_redact() {
+ let conn = "localhost:5432/db";
+ let redacted = redact_connection_string(conn);
+ assert_eq!(redacted, "loc***");
+ }
+
+ #[test]
+ fn given_postgresql_scheme_should_redact() {
+ let conn = "postgresql://admin:secret123@db.example.com:5432/mydb";
+ let redacted = redact_connection_string(conn);
+ assert_eq!(redacted, "postgresql://adm***");
+ }
}
diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml
index e30ef758c..6dec491f5 100644
--- a/core/integration/Cargo.toml
+++ b/core/integration/Cargo.toml
@@ -28,7 +28,7 @@ ci-qemu = []
[dependencies]
ahash = { workspace = true }
-assert_cmd = "2.1.1"
+assert_cmd = "2.1.2"
async-trait = { workspace = true }
bytes = { workspace = true }
chrono = { workspace = true }
@@ -43,21 +43,23 @@ iggy_binary_protocol = { workspace = true }
iggy_common = { workspace = true }
keyring = { workspace = true }
lazy_static = { workspace = true }
-libc = "0.2.178"
+libc = "0.2.180"
log = { workspace = true }
predicates = { workspace = true }
rand = { workspace = true }
rcgen = "0.14.6"
reqwest = { workspace = true }
-rmcp = { version = "0.12.0", features = [
+rmcp = { version = "0.13.0", features = [
"client",
"reqwest",
"transport-streamable-http-client",
"transport-streamable-http-client-reqwest",
] }
+serde = { workspace = true }
serde_json = { workspace = true }
serial_test = { workspace = true }
server = { workspace = true }
+sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chrono"] }
strum = { workspace = true }
strum_macros = { workspace = true }
tempfile = { workspace = true }
diff --git a/core/integration/tests/connectors/mod.rs b/core/integration/tests/connectors/mod.rs
index 146de4d4b..ffbbf0dca 100644
--- a/core/integration/tests/connectors/mod.rs
+++ b/core/integration/tests/connectors/mod.rs
@@ -17,85 +17,94 @@
* under the License.
*/
-use iggy::prelude::{Client, DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, IggyClient};
+use iggy::prelude::{
+ Client, DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, IggyClient, IggyMessage, Partitioning,
+};
use iggy_binary_protocol::{MessageClient, StreamClient, TopicClient, UserClient};
-use iggy_common::{CompressionAlgorithm, IggyExpiry, MaxTopicSize, PolledMessages};
+use iggy_common::{
+ CompressionAlgorithm, Consumer, Identifier, IggyExpiry, IggyTimestamp, MaxTopicSize,
+ PolledMessages, PollingStrategy,
+};
use integration::{
tcp_client::TcpClientFactory,
test_connectors_runtime::TestConnectorsRuntime,
test_server::{ClientFactory, IpAddrKind, TestServer},
};
+use serde::{Deserialize, Serialize};
use std::collections::HashMap;
mod http_config_provider;
mod postgres;
mod random;
-const DEFAULT_TEST_STREAM: &str = "test_stream";
-const DEFAULT_TEST_TOPIC: &str = "test_topic";
+pub const DEFAULT_TEST_STREAM: &str = "test_stream";
+pub const DEFAULT_TEST_TOPIC: &str = "test_topic";
+const TEST_CONSUMER_NAME: &str = "test_consumer";
+const ONE_DAY_MICROS: u64 = 24 * 60 * 60 * 1_000_000;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct TestMessage {
+ pub id: u64,
+ pub name: String,
+ pub count: u32,
+ pub amount: f64,
+ pub active: bool,
+ pub timestamp: i64,
+}
-fn setup_runtime() -> ConnectorsRuntime {
- let mut iggy_envs = HashMap::new();
- iggy_envs.insert("IGGY_QUIC_ENABLED".to_owned(), "false".to_owned());
- iggy_envs.insert("IGGY_WEBSOCKET_ENABLED".to_owned(), "false".to_owned());
- let mut test_server = TestServer::new(Some(iggy_envs), true, None, IpAddrKind::V4);
- test_server.start();
- ConnectorsRuntime {
- iggy_server: test_server,
- connectors_runtime: None,
- stream: "".to_owned(),
- topic: "".to_owned(),
+pub struct IggySetup {
+ pub stream: String,
+ pub topic: String,
+}
+
+impl Default for IggySetup {
+ fn default() -> Self {
+ Self {
+ stream: DEFAULT_TEST_STREAM.to_owned(),
+ topic: DEFAULT_TEST_TOPIC.to_owned(),
+ }
}
}
-#[derive(Debug)]
struct ConnectorsRuntime {
- stream: String,
- topic: String,
iggy_server: TestServer,
connectors_runtime: Option<TestConnectorsRuntime>,
}
-#[derive(Debug)]
struct ConnectorsIggyClient {
- stream: String,
- topic: String,
+ stream_id: Identifier,
+ topic_id: Identifier,
client: IggyClient,
}
-impl ConnectorsIggyClient {
- async fn get_messages(&self) -> Result<PolledMessages, iggy_common::IggyError> {
- self.client
- .poll_messages(
- &self.stream.clone().try_into().unwrap(),
- &self.topic.clone().try_into().unwrap(),
- None,
- &iggy_common::Consumer::new("test_consumer".try_into().unwrap()),
- &iggy_common::PollingStrategy::next(),
- 10,
- true,
- )
- .await
- }
-}
-
-#[derive(Debug)]
-pub struct IggySetup {
- pub stream: String,
- pub topic: String,
+fn create_test_messages(count: usize) -> Vec<TestMessage> {
+ let base_timestamp = IggyTimestamp::now().as_micros();
+ (1..=count)
+ .map(|i| TestMessage {
+ id: i as u64,
+ name: format!("user_{}", i - 1),
+ count: ((i - 1) * 10) as u32,
+ amount: (i - 1) as f64 * 99.99,
+ active: (i - 1) % 2 == 0,
+ timestamp: (base_timestamp + (i - 1) as u64 * ONE_DAY_MICROS) as i64,
+ })
+ .collect()
}
-impl Default for IggySetup {
- fn default() -> Self {
- Self {
- stream: DEFAULT_TEST_STREAM.to_owned(),
- topic: DEFAULT_TEST_TOPIC.to_owned(),
- }
+fn setup_runtime() -> ConnectorsRuntime {
+ let mut iggy_envs = HashMap::new();
+ iggy_envs.insert("IGGY_QUIC_ENABLED".to_owned(), "false".to_owned());
+ iggy_envs.insert("IGGY_WEBSOCKET_ENABLED".to_owned(), "false".to_owned());
+ let mut test_server = TestServer::new(Some(iggy_envs), true, None, IpAddrKind::V4);
+ test_server.start();
+ ConnectorsRuntime {
+ iggy_server: test_server,
+ connectors_runtime: None,
}
}
impl ConnectorsRuntime {
- pub async fn init(
+ async fn init(
&mut self,
config_path: &str,
envs: Option<HashMap<String, String>>,
@@ -123,10 +132,10 @@ impl ConnectorsRuntime {
.create_stream(&iggy_setup.stream)
.await
.expect("Failed to create stream");
- let stream_id = iggy_setup
+ let stream_id: Identifier = iggy_setup
.stream
.try_into()
- .expect("Invalid stream name in Iggy setup");
+ .expect("Failed to parse stream identifier");
client
.create_topic(
&stream_id,
@@ -149,15 +158,19 @@ impl ConnectorsRuntime {
TestConnectorsRuntime::with_iggy_address(&iggy_server_address, Some(all_envs));
connectors_runtime.start();
connectors_runtime.ensure_started().await;
- self.stream = self.stream.clone();
- self.topic = self.topic.clone();
self.connectors_runtime = Some(connectors_runtime);
}
- pub async fn create_client(&self) -> ConnectorsIggyClient {
+ async fn create_client(&self) -> ConnectorsIggyClient {
+ let stream_id: Identifier = DEFAULT_TEST_STREAM
+ .try_into()
+ .expect("Failed to parse stream identifier");
+ let topic_id: Identifier = DEFAULT_TEST_TOPIC
+ .try_into()
+ .expect("Failed to parse topic identifier");
ConnectorsIggyClient {
- stream: DEFAULT_TEST_STREAM.to_owned(),
- topic: DEFAULT_TEST_TOPIC.to_owned(),
+ stream_id,
+ topic_id,
client: self.create_iggy_client().await,
}
}
@@ -180,9 +193,40 @@ impl ConnectorsRuntime {
IggyClient::create(client, None, None)
}
- pub fn connectors_api_address(&self) -> Option<String> {
+ fn connectors_api_address(&self) -> Option<String> {
self.connectors_runtime
.as_ref()
- .map(|connectors_runtime| connectors_runtime.get_http_api_address())
+ .map(|r| r.get_http_api_address())
+ }
+}
+
+impl ConnectorsIggyClient {
+ async fn send_messages(&self, messages: &mut [IggyMessage]) {
+ self.client
+ .send_messages(
+ &self.stream_id,
+ &self.topic_id,
+ &Partitioning::partition_id(0),
+ messages,
+ )
+ .await
+ .expect("Failed to send messages to Iggy");
+ }
+
+ async fn poll_messages(&self) -> Result<PolledMessages, iggy_common::IggyError> {
+ let consumer_id: Identifier = TEST_CONSUMER_NAME
+ .try_into()
+ .expect("Failed to parse consumer identifier");
+ self.client
+ .poll_messages(
+ &self.stream_id,
+ &self.topic_id,
+ None,
+ &Consumer::new(consumer_id),
+ &PollingStrategy::next(),
+ 10,
+ true,
+ )
+ .await
}
}
diff --git a/core/integration/tests/connectors/postgres/mod.rs b/core/integration/tests/connectors/postgres/mod.rs
index c18900102..e6e41bbb2 100644
--- a/core/integration/tests/connectors/postgres/mod.rs
+++ b/core/integration/tests/connectors/postgres/mod.rs
@@ -17,44 +17,519 @@
* under the License.
*/
-use crate::connectors::{ConnectorsRuntime, IggySetup, setup_runtime};
+mod postgres_sink;
+mod postgres_source;
+
+use crate::connectors::{
+ ConnectorsRuntime, DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, IggySetup, TestMessage,
+ setup_runtime,
+};
+use serde::Deserialize;
+use sqlx::postgres::PgPoolOptions;
+use sqlx::{Pool, Postgres};
use std::collections::HashMap;
-use testcontainers_modules::{postgres, testcontainers::runners::AsyncRunner};
+use std::time::Duration;
+use testcontainers_modules::{
+ postgres,
+ testcontainers::{ContainerAsync, runners::AsyncRunner},
+};
+use tokio::time::sleep;
-mod postgres_sink;
+const SOURCE_TABLE: &str = "test_messages";
+const SOURCE_TABLE_BYTEA: &str = "test_payloads";
+const SOURCE_TABLE_JSON: &str = "test_json_payloads";
+const SOURCE_TABLE_DELETE: &str = "test_delete_rows";
+const SOURCE_TABLE_MARK: &str = "test_mark_rows";
+const SINK_TABLE: &str = "iggy_messages";
+const TEST_STREAM: &str = DEFAULT_TEST_STREAM;
+const TEST_TOPIC: &str = DEFAULT_TEST_TOPIC;
+const TEST_MESSAGE_COUNT: usize = 3;
+const POSTGRES_PORT: u16 = 5432;
+const POLL_ATTEMPTS: usize = 100;
+const POLL_INTERVAL_MS: u64 = 50;
+const FETCH_INTERVAL_MS: u64 = 10;
+const TABLE_WAIT_INTERVAL_MS: u64 = 50;
+const ENV_SINK_CONNECTION_STRING: &str =
+ "IGGY_CONNECTORS_SINK_POSTGRES_PLUGIN_CONFIG_CONNECTION_STRING";
+const ENV_SINK_TARGET_TABLE: &str = "IGGY_CONNECTORS_SINK_POSTGRES_PLUGIN_CONFIG_TARGET_TABLE";
+const ENV_SINK_PAYLOAD_FORMAT: &str = "IGGY_CONNECTORS_SINK_POSTGRES_PLUGIN_CONFIG_PAYLOAD_FORMAT";
+const ENV_SINK_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SINK_POSTGRES_STREAMS_0_STREAM";
+const ENV_SINK_STREAMS_0_TOPICS: &str = "IGGY_CONNECTORS_SINK_POSTGRES_STREAMS_0_TOPICS";
+const ENV_SINK_STREAMS_0_SCHEMA: &str = "IGGY_CONNECTORS_SINK_POSTGRES_STREAMS_0_SCHEMA";
+const ENV_SINK_STREAMS_0_CONSUMER_GROUP: &str =
+ "IGGY_CONNECTORS_SINK_POSTGRES_STREAMS_0_CONSUMER_GROUP";
+const ENV_SOURCE_CONNECTION_STRING: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_CONNECTION_STRING";
+const ENV_SOURCE_TABLES: &str = "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_TABLES";
+const ENV_SOURCE_TRACKING_COLUMN: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_TRACKING_COLUMN";
+const ENV_SOURCE_PAYLOAD_COLUMN: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_PAYLOAD_COLUMN";
+const ENV_SOURCE_PAYLOAD_FORMAT: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_PAYLOAD_FORMAT";
+const ENV_SOURCE_DELETE_AFTER_READ: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_DELETE_AFTER_READ";
+const ENV_SOURCE_PRIMARY_KEY_COLUMN: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_PRIMARY_KEY_COLUMN";
+const ENV_SOURCE_PROCESSED_COLUMN: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_PROCESSED_COLUMN";
+const ENV_SOURCE_INCLUDE_METADATA: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_INCLUDE_METADATA";
+const ENV_SOURCE_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SOURCE_POSTGRES_STREAMS_0_STREAM";
+const ENV_SOURCE_STREAMS_0_TOPIC: &str = "IGGY_CONNECTORS_SOURCE_POSTGRES_STREAMS_0_TOPIC";
+const ENV_SOURCE_STREAMS_0_SCHEMA: &str = "IGGY_CONNECTORS_SOURCE_POSTGRES_STREAMS_0_SCHEMA";
+const ENV_SOURCE_POLL_INTERVAL: &str =
+ "IGGY_CONNECTORS_SOURCE_POSTGRES_PLUGIN_CONFIG_POLL_INTERVAL";
+const ENV_SINK_PATH: &str = "IGGY_CONNECTORS_SINK_POSTGRES_PATH";
+const ENV_SOURCE_PATH: &str = "IGGY_CONNECTORS_SOURCE_POSTGRES_PATH";
+
+type SinkRow = (i64, String, String, Vec<u8>);
+type SinkJsonRow = (i64, serde_json::Value);
+
+#[derive(Debug, Deserialize)]
+struct DatabaseRecord {
+ table_name: String,
+ operation_type: String,
+ data: TestMessage,
+}
-async fn setup() -> ConnectorsRuntime {
+struct PostgresTestSetup {
+ runtime: ConnectorsRuntime,
+ connection_string: String,
+ #[allow(dead_code)]
+ container: ContainerAsync<postgres::Postgres>,
+}
+
+async fn start_container() -> (ContainerAsync<postgres::Postgres>, String) {
let container = postgres::Postgres::default()
.start()
.await
- .expect("Failed to start Postgres");
+ .expect("Failed to start Postgres container");
let host_port = container
- .get_host_port_ipv4(5432)
+ .get_host_port_ipv4(POSTGRES_PORT)
.await
.expect("Failed to get Postgres port");
+ let connection_string = format!("postgres://postgres:postgres@localhost:{host_port}");
+ (container, connection_string)
+}
+async fn create_pool(connection_string: &str) -> Pool<Postgres> {
+ PgPoolOptions::new()
+ .max_connections(1)
+ .connect(connection_string)
+ .await
+ .expect("Failed to connect to PostgreSQL")
+}
+
+async fn setup_sink() -> PostgresTestSetup {
let mut envs = HashMap::new();
- let iggy_setup = IggySetup::default();
- let connection_string = format!("postgres://postgres:postgres@localhost:{host_port}");
- envs.insert(
- "IGGY_CONNECTORS_SINK_POSTGRES_CONFIG_CONNECTION_STRING".to_owned(),
+ envs.insert(ENV_SINK_STREAMS_0_SCHEMA.to_owned(), "json".to_owned());
+ setup_sink_with_envs(envs).await
+}
+
+async fn setup_sink_bytea() -> PostgresTestSetup {
+ let mut envs = HashMap::new();
+ envs.insert(ENV_SINK_STREAMS_0_SCHEMA.to_owned(), "raw".to_owned());
+ envs.insert(ENV_SINK_PAYLOAD_FORMAT.to_owned(), "bytea".to_owned());
+ setup_sink_with_envs(envs).await
+}
+
+async fn setup_sink_json() -> PostgresTestSetup {
+ let mut envs = HashMap::new();
+ envs.insert(ENV_SINK_STREAMS_0_SCHEMA.to_owned(), "json".to_owned());
+ envs.insert(ENV_SINK_PAYLOAD_FORMAT.to_owned(), "json".to_owned());
+ setup_sink_with_envs(envs).await
+}
+
+async fn setup_sink_with_envs(mut extra_envs: HashMap<String, String>) -> PostgresTestSetup {
+ let (container, connection_string) = start_container().await;
+
+ extra_envs.insert(
+ ENV_SINK_CONNECTION_STRING.to_owned(),
+ connection_string.clone(),
+ );
+ extra_envs.insert(ENV_SINK_TARGET_TABLE.to_owned(), SINK_TABLE.to_owned());
+ extra_envs.insert(ENV_SINK_STREAMS_0_STREAM.to_owned(), TEST_STREAM.to_owned());
+ extra_envs.insert(
+ ENV_SINK_STREAMS_0_TOPICS.to_owned(),
+ format!("[{TEST_TOPIC}]"),
+ );
+ extra_envs.insert(
+ ENV_SINK_STREAMS_0_CONSUMER_GROUP.to_owned(),
+ "test".to_owned(),
+ );
+ extra_envs.insert(
+ ENV_SINK_PATH.to_owned(),
+ "../../target/debug/libiggy_connector_postgres_sink".to_owned(),
+ );
+
+ let mut runtime = setup_runtime();
+ runtime
+ .init("postgres/sink.toml", Some(extra_envs), IggySetup::default())
+ .await;
+
+ PostgresTestSetup {
+ runtime,
connection_string,
+ container,
+ }
+}
+
+async fn setup_source() -> PostgresTestSetup {
+ let (container, connection_string) = start_container().await;
+ let pool = create_pool(&connection_string).await;
+
+ sqlx::query(&format!(
+ "CREATE TABLE IF NOT EXISTS {SOURCE_TABLE} (
+ id SERIAL PRIMARY KEY,
+ name VARCHAR(255) NOT NULL,
+ count INTEGER NOT NULL,
+ amount DOUBLE PRECISION NOT NULL,
+ active BOOLEAN NOT NULL,
+ timestamp BIGINT NOT NULL
+ )"
+ ))
+ .execute(&pool)
+ .await
+ .expect("Failed to create source table");
+ pool.close().await;
+
+ let mut envs = HashMap::new();
+ envs.insert(ENV_SOURCE_TABLES.to_owned(), format!("[{SOURCE_TABLE}]"));
+ envs.insert(ENV_SOURCE_INCLUDE_METADATA.to_owned(), "true".to_owned());
+ envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_owned(), "json".to_owned());
+
+ setup_source_with_envs(container, connection_string, envs).await
+}
+
+async fn setup_source_bytea() -> PostgresTestSetup {
+ let (container, connection_string) = start_container().await;
+ let pool = create_pool(&connection_string).await;
+
+ sqlx::query(&format!(
+ "CREATE TABLE IF NOT EXISTS {SOURCE_TABLE_BYTEA} (
+ id SERIAL PRIMARY KEY,
+ payload BYTEA NOT NULL
+ )"
+ ))
+ .execute(&pool)
+ .await
+ .expect("Failed to create bytea source table");
+ pool.close().await;
+
+ let mut envs = HashMap::new();
+ envs.insert(
+ ENV_SOURCE_TABLES.to_owned(),
+ format!("[{SOURCE_TABLE_BYTEA}]"),
);
+ envs.insert(ENV_SOURCE_PAYLOAD_COLUMN.to_owned(), "payload".to_owned());
+ envs.insert(ENV_SOURCE_PAYLOAD_FORMAT.to_owned(), "bytea".to_owned());
+ envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_owned(), "raw".to_owned());
+
+ setup_source_with_envs(container, connection_string, envs).await
+}
+
+async fn setup_source_json() -> PostgresTestSetup {
+ let (container, connection_string) = start_container().await;
+ let pool = create_pool(&connection_string).await;
+
+ sqlx::query(&format!(
+ "CREATE TABLE IF NOT EXISTS {SOURCE_TABLE_JSON} (
+ id SERIAL PRIMARY KEY,
+ data JSONB NOT NULL
+ )"
+ ))
+ .execute(&pool)
+ .await
+ .expect("Failed to create json source table");
+ pool.close().await;
+
+ let mut envs = HashMap::new();
envs.insert(
- "IGGY_CONNECTORS_SINK_POSTGRES_STREAMS_0_STREAM".to_owned(),
- iggy_setup.stream.to_owned(),
+ ENV_SOURCE_TABLES.to_owned(),
+ format!("[{SOURCE_TABLE_JSON}]"),
);
+ envs.insert(ENV_SOURCE_PAYLOAD_COLUMN.to_owned(), "data".to_owned());
envs.insert(
- "IGGY_CONNECTORS_SINK_POSTGRES_STREAMS_0_TOPICS_0".to_owned(),
- iggy_setup.topic.to_owned(),
+ ENV_SOURCE_PAYLOAD_FORMAT.to_owned(),
+ "json_direct".to_owned(),
);
+ envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_owned(), "json".to_owned());
+
+ setup_source_with_envs(container, connection_string, envs).await
+}
+
+async fn setup_source_delete() -> PostgresTestSetup {
+ let (container, connection_string) = start_container().await;
+ let pool = create_pool(&connection_string).await;
+
+ sqlx::query(&format!(
+ "CREATE TABLE IF NOT EXISTS {SOURCE_TABLE_DELETE} (
+ id SERIAL PRIMARY KEY,
+ name VARCHAR(255) NOT NULL,
+ value INTEGER NOT NULL
+ )"
+ ))
+ .execute(&pool)
+ .await
+ .expect("Failed to create delete source table");
+ pool.close().await;
+
+ let mut envs = HashMap::new();
envs.insert(
- "IGGY_CONNECTORS_SINK_POSTGRES_STREAMS_0_SCHEMA".to_owned(),
- "json".to_owned(),
+ ENV_SOURCE_TABLES.to_owned(),
+ format!("[{SOURCE_TABLE_DELETE}]"),
);
+ envs.insert(ENV_SOURCE_PRIMARY_KEY_COLUMN.to_owned(), "id".to_owned());
+ envs.insert(ENV_SOURCE_DELETE_AFTER_READ.to_owned(), "true".to_owned());
+ envs.insert(ENV_SOURCE_INCLUDE_METADATA.to_owned(), "true".to_owned());
+ envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_owned(), "json".to_owned());
+
+ setup_source_with_envs(container, connection_string, envs).await
+}
+
+async fn setup_source_mark() -> PostgresTestSetup {
+ let (container, connection_string) = start_container().await;
+ let pool = create_pool(&connection_string).await;
+
+ sqlx::query(&format!(
+ "CREATE TABLE IF NOT EXISTS {SOURCE_TABLE_MARK} (
+ id SERIAL PRIMARY KEY,
+ name VARCHAR(255) NOT NULL,
+ value INTEGER NOT NULL,
+ is_processed BOOLEAN NOT NULL DEFAULT FALSE
+ )"
+ ))
+ .execute(&pool)
+ .await
+ .expect("Failed to create mark source table");
+ pool.close().await;
+
+ let mut envs = HashMap::new();
+ envs.insert(
+ ENV_SOURCE_TABLES.to_owned(),
+ format!("[{SOURCE_TABLE_MARK}]"),
+ );
+ envs.insert(ENV_SOURCE_PRIMARY_KEY_COLUMN.to_owned(), "id".to_owned());
+ envs.insert(
+ ENV_SOURCE_PROCESSED_COLUMN.to_owned(),
+ "is_processed".to_owned(),
+ );
+ envs.insert(ENV_SOURCE_INCLUDE_METADATA.to_owned(), "true".to_owned());
+ envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_owned(), "json".to_owned());
+
+ setup_source_with_envs(container, connection_string, envs).await
+}
+
+async fn setup_source_with_envs(
+ container: ContainerAsync<postgres::Postgres>,
+ connection_string: String,
+ mut extra_envs: HashMap<String, String>,
+) -> PostgresTestSetup {
+ extra_envs.insert(
+ ENV_SOURCE_CONNECTION_STRING.to_owned(),
+ connection_string.clone(),
+ );
+ extra_envs.insert(ENV_SOURCE_TRACKING_COLUMN.to_owned(), "id".to_owned());
+ extra_envs.insert(
+ ENV_SOURCE_STREAMS_0_STREAM.to_owned(),
+ TEST_STREAM.to_owned(),
+ );
+ extra_envs.insert(ENV_SOURCE_STREAMS_0_TOPIC.to_owned(), TEST_TOPIC.to_owned());
+ extra_envs.insert(ENV_SOURCE_POLL_INTERVAL.to_owned(), "10ms".to_owned());
+ extra_envs.insert(
+ ENV_SOURCE_PATH.to_owned(),
+ "../../target/debug/libiggy_connector_postgres_source".to_owned(),
+ );
+
let mut runtime = setup_runtime();
runtime
- .init("postgres/config.toml", Some(envs), iggy_setup)
+ .init(
+ "postgres/source.toml",
+ Some(extra_envs),
+ IggySetup::default(),
+ )
.await;
- runtime
+
+ PostgresTestSetup {
+ runtime,
+ connection_string,
+ container,
+ }
+}
+
+impl PostgresTestSetup {
+ async fn create_pool(&self) -> Pool<Postgres> {
+ create_pool(&self.connection_string).await
+ }
+
+ async fn wait_for_table(&self, pool: &Pool<Postgres>, table: &str) {
+ let query = format!("SELECT 1 FROM {table} LIMIT 1");
+ for _ in 0..POLL_ATTEMPTS {
+ if sqlx::query(&query).fetch_optional(pool).await.is_ok() {
+ return;
+ }
+ sleep(Duration::from_millis(TABLE_WAIT_INTERVAL_MS)).await;
+ }
+ panic!("Table {table} was not created in time");
+ }
+
+ async fn insert_test_messages(&self, pool: &Pool<Postgres>, messages: &[TestMessage]) {
+ let query = format!(
+ "INSERT INTO {SOURCE_TABLE} (id, name, count, amount, active, timestamp) \
+ VALUES ($1, $2, $3, $4, $5, $6)"
+ );
+ for msg in messages {
+ sqlx::query(&query)
+ .bind(msg.id as i32)
+ .bind(&msg.name)
+ .bind(msg.count as i32)
+ .bind(msg.amount)
+ .bind(msg.active)
+ .bind(msg.timestamp)
+ .execute(pool)
+ .await
+ .expect("Failed to insert test message");
+ }
+ }
+
+ async fn insert_bytea_payloads(&self, pool: &Pool<Postgres>, payloads: &[Vec<u8>]) {
+ let query = format!("INSERT INTO {SOURCE_TABLE_BYTEA} (id, payload) VALUES ($1, $2)");
+ for (i, payload) in payloads.iter().enumerate() {
+ sqlx::query(&query)
+ .bind((i + 1) as i32)
+ .bind(payload)
+ .execute(pool)
+ .await
+ .expect("Failed to insert bytea payload");
+ }
+ }
+
+ async fn insert_json_payloads(&self, pool: &Pool<Postgres>, payloads: &[serde_json::Value]) {
+ let query = format!("INSERT INTO {SOURCE_TABLE_JSON} (id, data) VALUES ($1, $2)");
+ for (i, payload) in payloads.iter().enumerate() {
+ sqlx::query(&query)
+ .bind((i + 1) as i32)
+ .bind(payload)
+ .execute(pool)
+ .await
+ .expect("Failed to insert json payload");
+ }
+ }
+
+ async fn insert_delete_rows(&self, pool: &Pool<Postgres>, count: usize) {
+ let query = format!("INSERT INTO {SOURCE_TABLE_DELETE} (name, value) VALUES ($1, $2)");
+ for i in 0..count {
+ sqlx::query(&query)
+ .bind(format!("row_{i}"))
+ .bind(i as i32 * 10)
+ .execute(pool)
+ .await
+ .expect("Failed to insert delete row");
+ }
+ }
+
+ async fn insert_mark_rows(&self, pool: &Pool<Postgres>, count: usize) {
+ let query = format!(
+ "INSERT INTO {SOURCE_TABLE_MARK} (name, value, is_processed) VALUES ($1, $2, $3)"
+ );
+ for i in 0..count {
+ sqlx::query(&query)
+ .bind(format!("row_{i}"))
+ .bind(i as i32 * 10)
+ .bind(false)
+ .execute(pool)
+ .await
+ .expect("Failed to insert mark row");
+ }
+ }
+
+ async fn count_rows(&self, pool: &Pool<Postgres>, table: &str) -> i64 {
+ let query = format!("SELECT COUNT(*) FROM {table}");
+ let count: (i64,) = sqlx::query_as(&query)
+ .fetch_one(pool)
+ .await
+ .expect("Failed to count rows");
+ count.0
+ }
+
+ async fn count_unprocessed_rows(&self, pool: &Pool<Postgres>, table: &str) -> i64 {
+ let query = format!("SELECT COUNT(*) FROM {table} WHERE is_processed = FALSE");
+ let count: (i64,) = sqlx::query_as(&query)
+ .fetch_one(pool)
+ .await
+ .expect("Failed to count unprocessed rows");
+ count.0
+ }
+
+ async fn count_processed_rows(&self, pool: &Pool<Postgres>, table: &str) -> i64 {
+ let query = format!("SELECT COUNT(*) FROM {table} WHERE is_processed = TRUE");
+ let count: (i64,) = sqlx::query_as(&query)
+ .fetch_one(pool)
+ .await
+ .expect("Failed to count processed rows");
+ count.0
+ }
+
+ async fn fetch_sink_rows(&self, pool: &Pool<Postgres>, expected_count: usize) -> Vec<SinkRow> {
+ let query = format!(
+ "SELECT iggy_offset, iggy_stream, iggy_topic, payload \
+ FROM {SINK_TABLE} ORDER BY iggy_offset"
+ );
+ let mut rows = Vec::new();
+ for _ in 0..POLL_ATTEMPTS {
+ if let Ok(fetched) = sqlx::query_as::<_, SinkRow>(&query).fetch_all(pool).await {
+ rows = fetched;
+ if rows.len() >= expected_count {
+ break;
+ }
+ }
+ sleep(Duration::from_millis(FETCH_INTERVAL_MS)).await;
+ }
+ rows
+ }
+
+ async fn fetch_sink_json_rows(
+ &self,
+ pool: &Pool<Postgres>,
+ expected_count: usize,
+ ) -> Vec<SinkJsonRow> {
+ let query = format!("SELECT iggy_offset, payload FROM {SINK_TABLE} ORDER BY iggy_offset");
+ let mut rows = Vec::new();
+ for _ in 0..POLL_ATTEMPTS {
+ if let Ok(fetched) = sqlx::query_as::<_, SinkJsonRow>(&query)
+ .fetch_all(pool)
+ .await
+ {
+ rows = fetched;
+ if rows.len() >= expected_count {
+ break;
+ }
+ }
+ sleep(Duration::from_millis(FETCH_INTERVAL_MS)).await;
+ }
+ rows
+ }
+
+ async fn poll_messages<T, F>(&self, expected_count: usize, transform: F) -> Vec<T>
+ where
+ F: Fn(&[u8]) -> Option<T>,
+ {
+ let client = self.runtime.create_client().await;
+ let mut received = Vec::new();
+ for _ in 0..POLL_ATTEMPTS {
+ if let Ok(polled) = client.poll_messages().await {
+ for msg in polled.messages {
+ if let Some(item) = transform(&msg.payload) {
+ received.push(item);
+ }
+ }
+ if received.len() >= expected_count {
+ break;
+ }
+ }
+ sleep(Duration::from_millis(POLL_INTERVAL_MS)).await;
+ }
+ received
+ }
}
diff --git a/core/integration/tests/connectors/postgres/postgres_sink.rs b/core/integration/tests/connectors/postgres/postgres_sink.rs
index ee05048b2..5927b7c54 100644
--- a/core/integration/tests/connectors/postgres/postgres_sink.rs
+++ b/core/integration/tests/connectors/postgres/postgres_sink.rs
@@ -1,4 +1,5 @@
-/* Licensed to the Apache Software Foundation (ASF) under one
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
@@ -16,9 +17,141 @@
* under the License.
*/
-use crate::connectors::postgres::setup;
+use super::{
+ SINK_TABLE, TEST_MESSAGE_COUNT, TEST_STREAM, TEST_TOPIC, setup_sink, setup_sink_bytea,
+ setup_sink_json,
+};
+use crate::connectors::{TestMessage, create_test_messages};
+use bytes::Bytes;
+use iggy::prelude::IggyMessage;
#[tokio::test]
-async fn given_valid_configuration_postgres_sink_connector_should_start() {
- let _runtime = setup().await;
+async fn given_json_messages_sink_connector_should_store_as_bytea() {
+ let setup = setup_sink().await;
+ let client = setup.runtime.create_client().await;
+ let pool = setup.create_pool().await;
+
+ setup.wait_for_table(&pool, SINK_TABLE).await;
+
+ let messages_data = create_test_messages(TEST_MESSAGE_COUNT);
+ let mut messages: Vec<IggyMessage> = messages_data
+ .iter()
+ .enumerate()
+ .map(|(i, msg)| {
+ let payload = serde_json::to_vec(msg).expect("Failed to serialize message");
+ IggyMessage::builder()
+ .id((i + 1) as u128)
+ .payload(Bytes::from(payload))
+ .build()
+ .expect("Failed to build message")
+ })
+ .collect();
+
+ client.send_messages(&mut messages).await;
+
+ let rows = setup.fetch_sink_rows(&pool, TEST_MESSAGE_COUNT).await;
+
+ assert_eq!(
+ rows.len(),
+ TEST_MESSAGE_COUNT,
+ "Expected {TEST_MESSAGE_COUNT} rows in PostgreSQL table"
+ );
+
+ for (i, (offset, stream, topic, payload)) in rows.iter().enumerate() {
+ assert_eq!(*offset, i as i64, "Offset mismatch at row {i}");
+ assert_eq!(stream, TEST_STREAM, "Stream mismatch at row {i}");
+ assert_eq!(topic, TEST_TOPIC, "Topic mismatch at row {i}");
+
+ let stored: TestMessage =
+ serde_json::from_slice(payload).expect("Failed to deserialize stored payload");
+ assert_eq!(stored, messages_data[i], "Message data mismatch at row {i}");
+ }
+}
+
+#[tokio::test]
+async fn given_binary_messages_sink_connector_should_store_as_bytea() {
+ let setup = setup_sink_bytea().await;
+ let client = setup.runtime.create_client().await;
+ let pool = setup.create_pool().await;
+
+ setup.wait_for_table(&pool, SINK_TABLE).await;
+
+ let raw_payloads: Vec<Vec<u8>> = vec![
+ b"plain text message".to_vec(),
+ vec![0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD],
+ vec![0xDE, 0xAD, 0xBE, 0xEF],
+ ];
+
+ let mut messages: Vec<IggyMessage> = raw_payloads
+ .iter()
+ .enumerate()
+ .map(|(i, payload)| {
+ IggyMessage::builder()
+ .id((i + 1) as u128)
+ .payload(Bytes::from(payload.clone()))
+ .build()
+ .expect("Failed to build message")
+ })
+ .collect();
+
+ client.send_messages(&mut messages).await;
+
+ let rows = setup.fetch_sink_rows(&pool, TEST_MESSAGE_COUNT).await;
+
+ assert_eq!(
+ rows.len(),
+ TEST_MESSAGE_COUNT,
+ "Expected {TEST_MESSAGE_COUNT} rows in PostgreSQL table"
+ );
+
+ for (i, (offset, _, _, payload)) in rows.iter().enumerate() {
+ assert_eq!(*offset, i as i64, "Offset mismatch at row {i}");
+ assert_eq!(payload, &raw_payloads[i], "Payload mismatch at row {i}");
+ }
+}
+
+#[tokio::test]
+async fn given_json_messages_sink_connector_should_store_as_jsonb() {
+ let setup = setup_sink_json().await;
+ let client = setup.runtime.create_client().await;
+ let pool = setup.create_pool().await;
+
+ setup.wait_for_table(&pool, SINK_TABLE).await;
+
+ let json_payloads: Vec<serde_json::Value> = vec![
+ serde_json::json!({"name": "Alice", "age": 30}),
+ serde_json::json!({"items": [1, 2, 3], "active": true}),
+ serde_json::json!({"nested": {"key": "value"}, "count": 42}),
+ ];
+
+ let mut messages: Vec<IggyMessage> = json_payloads
+ .iter()
+ .enumerate()
+ .map(|(i, payload)| {
+ let bytes = serde_json::to_vec(payload).expect("Failed to serialize json");
+ IggyMessage::builder()
+ .id((i + 1) as u128)
+ .payload(Bytes::from(bytes))
+ .build()
+ .expect("Failed to build message")
+ })
+ .collect();
+
+ client.send_messages(&mut messages).await;
+
+ let rows = setup.fetch_sink_json_rows(&pool, TEST_MESSAGE_COUNT).await;
+
+ assert_eq!(
+ rows.len(),
+ TEST_MESSAGE_COUNT,
+ "Expected {TEST_MESSAGE_COUNT} rows in PostgreSQL table"
+ );
+
+ for (i, (offset, payload)) in rows.iter().enumerate() {
+ assert_eq!(*offset, i as i64, "Offset mismatch at row {i}");
+ assert_eq!(
+ payload, &json_payloads[i],
+ "JSON payload mismatch at row {i}"
+ );
+ }
}
diff --git a/core/integration/tests/connectors/postgres/postgres_source.rs b/core/integration/tests/connectors/postgres/postgres_source.rs
new file mode 100644
index 000000000..195e2f258
--- /dev/null
+++ b/core/integration/tests/connectors/postgres/postgres_source.rs
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use super::{
+ DatabaseRecord, SOURCE_TABLE, SOURCE_TABLE_DELETE, SOURCE_TABLE_MARK, TEST_MESSAGE_COUNT,
+ setup_source, setup_source_bytea, setup_source_delete, setup_source_json, setup_source_mark,
+};
+use crate::connectors::create_test_messages;
+
+#[tokio::test]
+async fn given_json_rows_source_connector_should_produce_messages_to_iggy() {
+ let setup = setup_source().await;
+ let pool = setup.create_pool().await;
+ let test_messages = create_test_messages(TEST_MESSAGE_COUNT);
+
+ setup.insert_test_messages(&pool, &test_messages).await;
+ pool.close().await;
+
+ let received: Vec<DatabaseRecord> = setup
+ .poll_messages(TEST_MESSAGE_COUNT, |payload| {
+ serde_json::from_slice(payload).ok()
+ })
+ .await;
+
+ assert!(
+ received.len() >= TEST_MESSAGE_COUNT,
+ "Expected at least {TEST_MESSAGE_COUNT} messages, got {}",
+ received.len()
+ );
+
+ for (i, record) in received.iter().enumerate() {
+ assert_eq!(
+ record.table_name, SOURCE_TABLE,
+ "Table name mismatch at record {i}"
+ );
+ assert_eq!(
+ record.operation_type, "SELECT",
+ "Operation type mismatch at record {i}"
+ );
+ assert_eq!(
+ record.data, test_messages[i],
+ "Message data mismatch at record {i}"
+ );
+ }
+}
+
+#[tokio::test]
+async fn given_bytea_rows_source_connector_should_produce_raw_messages_to_iggy() {
+ let setup = setup_source_bytea().await;
+ let pool = setup.create_pool().await;
+
+ let payloads: Vec<Vec<u8>> = vec![
+ b"hello world".to_vec(),
+ vec![0x00, 0x01, 0x02, 0xFF, 0xFE],
+ serde_json::to_vec(&serde_json::json!({"key": "value", "number": 42}))
+ .expect("Failed to serialize json"),
+ ];
+
+ setup.insert_bytea_payloads(&pool, &payloads).await;
+ pool.close().await;
+
+ let received: Vec<Vec<u8>> = setup
+ .poll_messages(TEST_MESSAGE_COUNT, |payload| Some(payload.to_vec()))
+ .await;
+
+ assert!(
+ received.len() >= TEST_MESSAGE_COUNT,
+ "Expected at least {TEST_MESSAGE_COUNT} messages, got {}",
+ received.len()
+ );
+
+ for (i, payload) in received.iter().enumerate() {
+ assert_eq!(payload, &payloads[i], "Payload mismatch at index {i}");
+ }
+}
+
+#[tokio::test]
+async fn given_jsonb_rows_source_connector_should_produce_json_messages_to_iggy() {
+ let setup = setup_source_json().await;
+ let pool = setup.create_pool().await;
+
+ let json_payloads: Vec<serde_json::Value> = vec![
+ serde_json::json!({"name": "Alice", "score": 100}),
+ serde_json::json!({"items": ["a", "b", "c"]}),
+ serde_json::json!({"nested": {"deep": {"value": 42}}}),
+ ];
+
+ setup.insert_json_payloads(&pool, &json_payloads).await;
+ pool.close().await;
+
+ let received: Vec<serde_json::Value> = setup
+ .poll_messages(TEST_MESSAGE_COUNT, |payload| {
+ serde_json::from_slice(payload).ok()
+ })
+ .await;
+
+ assert!(
+ received.len() >= TEST_MESSAGE_COUNT,
+ "Expected at least {TEST_MESSAGE_COUNT} messages, got {}",
+ received.len()
+ );
+
+ for (i, payload) in received.iter().enumerate() {
+ assert_eq!(
+ payload, &json_payloads[i],
+ "JSON payload mismatch at index {i}"
+ );
+ }
+}
+
+#[tokio::test]
+async fn given_delete_after_read_source_connector_should_remove_rows_after_producing() {
+ let setup = setup_source_delete().await;
+ let pool = setup.create_pool().await;
+
+ setup.insert_delete_rows(&pool, TEST_MESSAGE_COUNT).await;
+
+ let initial_count = setup.count_rows(&pool, SOURCE_TABLE_DELETE).await;
+ assert_eq!(
+ initial_count, TEST_MESSAGE_COUNT as i64,
+ "Expected {TEST_MESSAGE_COUNT} rows before processing"
+ );
+
+ let received: Vec<serde_json::Value> = setup
+ .poll_messages(TEST_MESSAGE_COUNT, |payload| {
+ serde_json::from_slice(payload).ok()
+ })
+ .await;
+
+ assert!(
+ received.len() >= TEST_MESSAGE_COUNT,
+ "Expected at least {TEST_MESSAGE_COUNT} messages, got {}",
+ received.len()
+ );
+
+ let final_count = setup.count_rows(&pool, SOURCE_TABLE_DELETE).await;
+ assert_eq!(
+ final_count, 0,
+ "Expected 0 rows after delete_after_read, got {final_count}"
+ );
+
+ pool.close().await;
+}
+
+#[tokio::test]
+async fn given_processed_column_source_connector_should_mark_rows_after_producing() {
+ let setup = setup_source_mark().await;
+ let pool = setup.create_pool().await;
+
+ setup.insert_mark_rows(&pool, TEST_MESSAGE_COUNT).await;
+
+ let initial_unprocessed = setup.count_unprocessed_rows(&pool, SOURCE_TABLE_MARK).await;
+ let initial_processed = setup.count_processed_rows(&pool, SOURCE_TABLE_MARK).await;
+ assert_eq!(
+ initial_unprocessed, TEST_MESSAGE_COUNT as i64,
+ "Expected {TEST_MESSAGE_COUNT} unprocessed rows before processing"
+ );
+ assert_eq!(
+ initial_processed, 0,
+ "Expected 0 processed rows before processing"
+ );
+
+ let received: Vec<serde_json::Value> = setup
+ .poll_messages(TEST_MESSAGE_COUNT, |payload| {
+ serde_json::from_slice(payload).ok()
+ })
+ .await;
+
+ assert!(
+ received.len() >= TEST_MESSAGE_COUNT,
+ "Expected at least {TEST_MESSAGE_COUNT} messages, got {}",
+ received.len()
+ );
+
+ let final_unprocessed = setup.count_unprocessed_rows(&pool, SOURCE_TABLE_MARK).await;
+ let final_processed = setup.count_processed_rows(&pool, SOURCE_TABLE_MARK).await;
+ assert_eq!(
+ final_unprocessed, 0,
+ "Expected 0 unprocessed rows after processing, got {final_unprocessed}"
+ );
+ assert_eq!(
+ final_processed, TEST_MESSAGE_COUNT as i64,
+ "Expected {TEST_MESSAGE_COUNT} processed rows after processing, got {final_processed}"
+ );
+
+ let total_count = setup.count_rows(&pool, SOURCE_TABLE_MARK).await;
+ assert_eq!(
+ total_count, TEST_MESSAGE_COUNT as i64,
+ "Rows should not be deleted, expected {TEST_MESSAGE_COUNT}, got {total_count}"
+ );
+
+ pool.close().await;
+}
diff --git a/core/integration/tests/connectors/postgres/config.toml b/core/integration/tests/connectors/postgres/sink.toml
similarity index 93%
copy from core/integration/tests/connectors/postgres/config.toml
copy to core/integration/tests/connectors/postgres/sink.toml
index 7336b47b1..61b3ddada 100644
--- a/core/integration/tests/connectors/postgres/config.toml
+++ b/core/integration/tests/connectors/postgres/sink.toml
@@ -17,4 +17,4 @@
[connectors]
config_type = "local"
-config_dir = "tests/connectors/postgres/connectors_config"
+config_dir = "../connectors/sinks/postgres_sink"
diff --git a/core/integration/tests/connectors/postgres/config.toml b/core/integration/tests/connectors/postgres/source.toml
similarity index 93%
rename from core/integration/tests/connectors/postgres/config.toml
rename to core/integration/tests/connectors/postgres/source.toml
index 7336b47b1..049329c08 100644
--- a/core/integration/tests/connectors/postgres/config.toml
+++ b/core/integration/tests/connectors/postgres/source.toml
@@ -17,4 +17,4 @@
[connectors]
config_type = "local"
-config_dir = "tests/connectors/postgres/connectors_config"
+config_dir = "../connectors/sources/postgres_source"
diff --git a/core/integration/tests/connectors/random/random_source.rs b/core/integration/tests/connectors/random/random_source.rs
index b7d6eeccf..1bfe2de9f 100644
--- a/core/integration/tests/connectors/random/random_source.rs
+++ b/core/integration/tests/connectors/random/random_source.rs
@@ -27,7 +27,10 @@ async fn given_valid_configuration_random_source_connector_should_produce_messag
let client = runtime.create_client().await;
// Wait for some messages to be produced
sleep(Duration::from_secs(1)).await;
- let messages = client.get_messages().await.expect("Failed to get messages");
+ let messages = client
+ .poll_messages()
+ .await
+ .expect("Failed to poll messages");
assert!(
!messages.messages.is_empty(),
"No messages received from random source"
diff --git a/core/integration/tests/mcp/mod.rs b/core/integration/tests/mcp/mod.rs
index 2793e00ec..3fab31930 100644
--- a/core/integration/tests/mcp/mod.rs
+++ b/core/integration/tests/mcp/mod.rs
@@ -701,6 +701,7 @@ impl TestMcpClient {
.call_tool(CallToolRequestParam {
name: method.to_owned().into(),
arguments: data.and_then(|value| value.as_object().cloned()),
+ task: None,
})
.await
}
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml
index f5c0deb5f..e80c3446d 100644
--- a/core/server/Cargo.toml
+++ b/core/server/Cargo.toml
@@ -102,7 +102,7 @@ rand = { workspace = true }
reqwest = { workspace = true, features = ["rustls-tls-no-provider"] }
ringbuffer = "0.16.0"
rmp-serde = { workspace = true }
-rust-embed = { version = "8.9.0", optional = true }
+rust-embed = { version = "8.11.0", optional = true }
rustls = { workspace = true }
rustls-pemfile = "2.2.0"
send_wrapper = "0.6.0"
@@ -120,7 +120,7 @@ toml = { workspace = true }
tower-http = { workspace = true }
tracing = { workspace = true }
tracing-appender = { workspace = true }
-tracing-opentelemetry = "0.32.0"
+tracing-opentelemetry = "0.32.1"
tracing-subscriber = { workspace = true }
tungstenite = { workspace = true }
ulid = "1.2.1"
@@ -134,7 +134,7 @@ hwlocality = { workspace = true, features = ["vendored"] }
[build-dependencies]
figment = { version = "0.10.19", features = ["json", "toml", "env"] }
-vergen-git2 = { version = "1.0.7", features = [
+vergen-git2 = { version = "9.1.0", features = [
"build",
"cargo",
"rustc",