You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by lz...@apache.org on 2026/07/09 20:20:37 UTC
(iggy) branch dotnet-encryption updated (7e6d7d0ae -> d7c540c83)
This is an automated email from the ASF dual-hosted git repository.
lukaszzborek pushed a change to branch dotnet-encryption
in repository https://gitbox.apache.org/repos/asf/iggy.git
discard 7e6d7d0ae refactor(csharp): update message status handling and remove encryption references
new d7c540c83 refactor(csharp): update message status handling and remove encryption references
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (7e6d7d0ae)
\
N -- N -- N refs/heads/dotnet-encryption (d7c540c83)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../HeaderEncryptionIntegrationTests.cs | 62 ++-------
.../MessageEncryptionIntegrationTests.cs | 2 -
.../Iggy_SDK/Encryption/AesMessageEncryptor.cs | 10 --
.../Exceptions/MessageDecryptionException.cs | 27 +---
.../Implementations/HttpMessageStream.cs | 1 -
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 1 -
.../Iggy_SDK/Utils/TcpMessageStreamHelpers.cs | 2 -
.../ClientTests/RawClientEncryptionTests.cs | 4 -
.../ConsumerTests/IggyConsumerBuilderTests.cs | 4 -
.../ConsumerTests/RentedConsumerTests.cs | 3 -
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 11 --
.../MapperTests/HeaderEncryptionTests.cs | 146 +++------------------
.../PublisherTests/MessageEncryptionTests.cs | 6 -
13 files changed, 37 insertions(+), 242 deletions(-)
(iggy) 01/01: refactor(csharp): update message status handling and remove encryption references
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lukaszzborek pushed a commit to branch dotnet-encryption
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit d7c540c83a63285e5f5f5ee4bbbe082059be7717
Author: Ćukasz Zborek <lu...@gmail.com>
AuthorDate: Mon Jun 29 23:01:12 2026 +0200
refactor(csharp): update message status handling and remove encryption references
---
.../Fixtures/IggyServerFixture.cs | 15 +-
.../HeaderEncryptionIntegrationTests.cs | 50 ++---
.../IggyTypedPublisherTests.cs | 25 +--
.../Iggy_SDK.Tests.Integration.csproj | 15 +-
.../MessageEncryptionIntegrationTests.cs | 238 +++++++++++++++++++++
.../Configuration/IggyClientConfigurator.cs | 16 ++
.../Iggy_SDK/Consumers/IggyConsumer.Logging.cs | 4 +-
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 54 ++---
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 79 ++++---
.../Iggy_SDK/Consumers/IggyConsumerBuilder.cs | 38 +++-
.../Iggy_SDK/Consumers/IggyConsumerConfig.cs | 6 -
foreign/csharp/Iggy_SDK/Consumers/MessageStatus.cs | 19 +-
.../csharp/Iggy_SDK/Consumers/ReceivedMessage.cs | 4 +-
.../Iggy_SDK/Consumers/ReceivedRentedMessage.cs | 3 +-
.../csharp/Iggy_SDK/Contracts/MessageResponse.cs | 11 -
.../Iggy_SDK/Contracts/PolledMessagesRental.cs | 15 +-
.../csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs | 135 ++++++++----
.../Iggy_SDK/Encryption/AesMessageEncryptor.cs | 109 +++++++---
.../Iggy_SDK/Encryption/IMessageEncryptor.cs | 44 ++--
.../Exceptions/InvalidResponseException.cs | 6 +-
...eException.cs => MalformedResponseException.cs} | 12 +-
.../Exceptions/MessageDecryptionException.cs | 53 +++++
.../csharp/Iggy_SDK/Factory/IggyClientFactory.cs | 22 +-
foreign/csharp/Iggy_SDK/IggyClient/IIggyClient.cs | 9 +
.../csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs | 5 +
.../Implementations/HttpMessageStream.cs | 131 +++++++++++-
.../IggyClient/Implementations/TcpMessageStream.cs | 142 +++++++-----
foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 2 +-
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 210 +++++++++++++-----
foreign/csharp/Iggy_SDK/Messages/Message.cs | 11 +-
foreign/csharp/Iggy_SDK/Messages/MessageHeader.cs | 2 +
.../Publishers/BackgroundMessageProcessor.cs | 23 +-
.../csharp/Iggy_SDK/Publishers/IggyPublisher.cs | 41 +---
.../Iggy_SDK/Publishers/IggyPublisherBuilder.cs | 33 ++-
.../Iggy_SDK/Publishers/IggyPublisherConfig.cs | 7 -
.../csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs | 19 +-
.../Publishers/MessageBatchFailedEventArgs.cs | 5 +-
.../Iggy_SDK/Publishers/PublisherEncryption.cs | 81 -------
foreign/csharp/Iggy_SDK/Publishers/SendUnit.cs | 32 +--
.../Iggy_SDK/Utils/TcpMessageStreamHelpers.cs | 20 +-
.../ClientTests/RawClientEncryptionTests.cs | 128 +++++++++++
.../ConsumerTests/IggyConsumerBuilderTests.cs | 72 +++++++
.../ConsumerTests/RentedConsumerTests.cs | 73 +++----
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 111 ++++++++++
.../MapperTests/HeaderEncryptionTests.cs | 10 +-
.../PublisherTests/IggyTypedPublisherTests.cs | 70 +-----
.../PublisherTests/MessageEncryptionTests.cs | 171 +++++++++++++++
.../PublisherTests/PublisherEncryptionTests.cs | 76 -------
.../Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs | 94 +++-----
foreign/csharp/Shared/EncryptorTestExtensions.cs | 40 ++++
foreign/csharp/Shared/Shared.csproj | 4 +-
51 files changed, 1778 insertions(+), 817 deletions(-)
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
index d8d7fe3b4..8c6718b60 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
@@ -16,6 +16,7 @@
// under the License.
using Apache.Iggy.Configuration;
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Apache.Iggy.Factory;
using Apache.Iggy.IggyClient;
@@ -172,9 +173,9 @@ public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable
}
public async Task<IIggyClient> CreateTcpClient(string userName = "iggy", string password = "iggy",
- bool connect = true)
+ bool connect = true, IMessageEncryptor? encryptor = null)
{
- var client = await CreateClient(Protocol.Tcp, connect: connect);
+ var client = await CreateClient(Protocol.Tcp, connect: connect, encryptor: encryptor);
if (connect)
{
@@ -184,9 +185,10 @@ public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable
return client;
}
- public async Task<IIggyClient> CreateHttpClient(string userName = "iggy", string password = "iggy")
+ public async Task<IIggyClient> CreateHttpClient(string userName = "iggy", string password = "iggy",
+ IMessageEncryptor? encryptor = null)
{
- var client = await CreateClient(Protocol.Http);
+ var client = await CreateClient(Protocol.Http, encryptor: encryptor);
await client.LoginUserAsync(userName, password);
@@ -194,7 +196,7 @@ public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable
}
public async Task<IIggyClient> CreateClient(Protocol protocol, Protocol? targetContainer = null,
- bool connect = true)
+ bool connect = true, IMessageEncryptor? encryptor = null)
{
var address = GetIggyAddress(protocol);
@@ -208,7 +210,8 @@ public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable
Enabled = true,
Username = "iggy",
Password = "iggy"
- }
+ },
+ MessageEncryptor = encryptor
});
if (connect)
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs
index 0b8e98b81..4004a5929 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs
@@ -25,6 +25,7 @@ using Apache.Iggy.Kinds;
using Apache.Iggy.Mappers;
using Apache.Iggy.Messages;
using Apache.Iggy.Publishers;
+using Apache.Iggy.Shared;
using Apache.Iggy.Tests.Integrations.Fixtures;
using Shouldly;
using Partitioning = Apache.Iggy.Kinds.Partitioning;
@@ -40,36 +41,35 @@ public class HeaderEncryptionIntegrationTests
[MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
public async Task SendMessages_WithEncryptedHeaders_Should_NotBeReadableWithoutDecryptor(Protocol protocol)
{
- var client = protocol == Protocol.Tcp
+ var encryptor = CreateEncryptor();
+
+ // Publisher on an encrypting client; a plain client raw-polls the same topic to prove the wire bytes
+ // stay encrypted.
+ var encryptingClient = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient(encryptor: encryptor)
+ : await Fixture.CreateHttpClient(encryptor: encryptor);
+ var plainClient = protocol == Protocol.Tcp
? await Fixture.CreateTcpClient()
: await Fixture.CreateHttpClient();
- var encryptor = CreateEncryptor();
- var testStream = await CreateTestStream(client, protocol);
+ var testStream = await CreateTestStream(plainClient, protocol);
var streamId = Identifier.String(testStream.StreamId);
var topicId = Identifier.String(testStream.TopicId);
- // Send message with encrypted headers via publisher
var publisher = IggyPublisherBuilder
- .Create(client, streamId, topicId)
+ .Create(encryptingClient, streamId, topicId)
.WithPartitioning(Partitioning.PartitionId(0))
- .WithEncryptor(encryptor)
.Build();
await publisher.InitAsync();
- var headers = CreateTestHeaders();
- var messages = new List<Message>
- {
- new(Guid.NewGuid(), Encoding.UTF8.GetBytes("encrypted payload"), headers)
- };
+ Dictionary<HeaderKey, HeaderValue> headers = CreateTestHeaders();
+ var messages = new List<Message> { new(Guid.NewGuid(), Encoding.UTF8.GetBytes("encrypted payload"), headers) };
await publisher.SendMessagesAsync(messages);
await publisher.DisposeAsync();
- // Poll with a normal client (no decryptor)
- var polled = await client.PollMessagesAsync(
- streamId, topicId, 0,
+ var polled = await plainClient.PollMessagesAsync(streamId, topicId, 0,
Consumer.New(0),
PollingStrategy.Next(),
1, false);
@@ -85,12 +85,12 @@ public class HeaderEncryptionIntegrationTests
msg.RawUserHeaders!.Length.ShouldBeGreaterThan(0);
// Manually decrypt and verify payload
- var decryptedPayload = encryptor.Decrypt(msg.Payload);
+ var decryptedPayload = encryptor.DecryptToArray(msg.Payload);
Encoding.UTF8.GetString(decryptedPayload).ShouldBe("encrypted payload");
// Manually decrypt and verify headers
- var decryptedHeaderBytesResult = encryptor.Decrypt(msg.RawUserHeaders);
- var decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytesResult);
+ var decryptedHeaderBytesResult = encryptor.DecryptToArray(msg.RawUserHeaders);
+ Dictionary<HeaderKey, HeaderValue> decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytesResult);
decryptedHeaders.Count.ShouldBe(3);
var typeHeader = decryptedHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "type"u8.ToArray() }];
@@ -99,13 +99,15 @@ public class HeaderEncryptionIntegrationTests
[Test]
[MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task ReceiveAsync_WithDecryptor_Should_DecryptHeadersCorrectly(Protocol protocol)
+ public async Task ReceiveAsync_WithEncryptingClient_Should_DecryptHeadersCorrectly(Protocol protocol)
{
+ var encryptor = CreateEncryptor();
+
+ // One encrypting client serves both publisher and consumer: it encrypts on send and decrypts on poll.
var client = protocol == Protocol.Tcp
- ? await Fixture.CreateTcpClient()
- : await Fixture.CreateHttpClient();
+ ? await Fixture.CreateTcpClient(encryptor: encryptor)
+ : await Fixture.CreateHttpClient(encryptor: encryptor);
- var encryptor = CreateEncryptor();
var testStream = await CreateTestStream(client, protocol);
var streamId = Identifier.String(testStream.StreamId);
var topicId = Identifier.String(testStream.TopicId);
@@ -114,12 +116,11 @@ public class HeaderEncryptionIntegrationTests
var publisher = IggyPublisherBuilder
.Create(client, streamId, topicId)
.WithPartitioning(Partitioning.PartitionId(0))
- .WithEncryptor(encryptor)
.Build();
await publisher.InitAsync();
- var headers = CreateTestHeaders();
+ Dictionary<HeaderKey, HeaderValue> headers = CreateTestHeaders();
var messages = new List<Message>
{
new(Guid.NewGuid(), Encoding.UTF8.GetBytes("consumer test payload"), headers)
@@ -128,14 +129,13 @@ public class HeaderEncryptionIntegrationTests
await publisher.SendMessagesAsync(messages);
await publisher.DisposeAsync();
- // Poll with IggyConsumer that has decryptor configured
+ // Consumer on the same encrypting client decrypts transparently.
var consumer = IggyConsumerBuilder
.Create(client, streamId, topicId, Consumer.New(1))
.WithPollingStrategy(PollingStrategy.Next())
.WithBatchSize(1)
.WithPartitionId(0)
.WithAutoCommitMode(AutoCommitMode.Disabled)
- .WithDecryptor(encryptor)
.Build();
await consumer.InitAsync();
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs
index d91fd0243..64d32e40b 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs
@@ -27,6 +27,7 @@ using Apache.Iggy.Headers;
using Apache.Iggy.IggyClient;
using Apache.Iggy.Kinds;
using Apache.Iggy.Publishers;
+using Apache.Iggy.Shared;
using Apache.Iggy.Tests.Integrations.Fixtures;
using Shouldly;
using Partitioning = Apache.Iggy.Kinds.Partitioning;
@@ -244,19 +245,24 @@ public class IggyTypedPublisherTests
[MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
public async Task SendAsync_WithEncryptor_Should_RoundTrip_Decrypted(Protocol protocol)
{
- var client = await Client(protocol);
- var stream = await CreateTestStream(client, protocol);
var encryptor = new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
- IggyPublisher<string> publisher
- = BuildTypedPublisher(client, stream, new Utf8StringSerializer(), encryptor: encryptor);
+ // Encryption is configured on the client. The publisher uses an encrypting client; a plain client polls
+ // to prove the wire bytes are ciphertext, then decrypts manually.
+ var encryptingClient = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient(encryptor: encryptor)
+ : await Fixture.CreateHttpClient(encryptor: encryptor);
+ var plainClient = await Client(protocol);
+ var stream = await CreateTestStream(plainClient, protocol);
+
+ IggyPublisher<string> publisher = BuildTypedPublisher(encryptingClient, stream, new Utf8StringSerializer());
await publisher.InitAsync();
await publisher.SendAsync("secret-payload");
- var message = (await Poll(client, stream, 1)).ShouldHaveSingleItem();
+ var message = (await Poll(plainClient, stream, 1)).ShouldHaveSingleItem();
Encoding.UTF8.GetString(message.Payload).ShouldNotBe("secret-payload");
- Encoding.UTF8.GetString(encryptor.Decrypt(message.Payload)).ShouldBe("secret-payload");
+ Encoding.UTF8.GetString(encryptor.DecryptToArray(message.Payload)).ShouldBe("secret-payload");
await publisher.DisposeAsync();
}
@@ -293,7 +299,7 @@ public class IggyTypedPublisherTests
// Base fluent methods return the non-generic builder, so apply them as statements to keep the typed Build().
private static IggyPublisher<T> BuildTypedPublisher<T>(IIggyClient client, TestStreamInfo stream,
- ISerializer<T> serializer, bool background = false, IMessageEncryptor? encryptor = null)
+ ISerializer<T> serializer, bool background = false)
{
IggyPublisherBuilder<T> builder = IggyPublisherBuilder<T>.Create(client, Identifier.String(stream.StreamId),
Identifier.String(stream.TopicId), serializer);
@@ -303,11 +309,6 @@ public class IggyTypedPublisherTests
builder.WithBackgroundSending(true, 1000, 10, TimeSpan.FromMilliseconds(50));
}
- if (encryptor != null)
- {
- builder.WithEncryptor(encryptor);
- }
-
return builder.Build();
}
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
index a8f2972d5..ad13f5d13 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
@@ -31,16 +31,17 @@
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
- <PackageReference Include="Microsoft.Testing.Extensions.Retry" />
- <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
- <PackageReference Include="Shouldly" />
- <PackageReference Include="Testcontainers" />
- <PackageReference Include="TUnit" />
+ <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage"/>
+ <PackageReference Include="Microsoft.Testing.Extensions.Retry"/>
+ <PackageReference Include="Microsoft.Testing.Extensions.TrxReport"/>
+ <PackageReference Include="Shouldly"/>
+ <PackageReference Include="Testcontainers"/>
+ <PackageReference Include="TUnit"/>
</ItemGroup>
<ItemGroup>
- <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj" />
+ <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj"/>
+ <ProjectReference Include="..\Shared\Shared.csproj"/>
</ItemGroup>
<ItemGroup>
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs
new file mode 100644
index 000000000..8e29ec1ba
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text;
+using Apache.Iggy.Consumers;
+using Apache.Iggy.Encryption;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Headers;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Publishers;
+using Apache.Iggy.Tests.Integrations.Fixtures;
+using Shouldly;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.Tests.Integrations;
+
+public class MessageEncryptionIntegrationTests
+{
+ private const int BatchSize = 25;
+
+ [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
+ public required IggyServerFixture Fixture { get; init; }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollRented_WithEncryptor_Should_DecryptBatchOfPayloads(Protocol protocol)
+ {
+ var encryptor = CreateEncryptor();
+ var client = await CreateClient(protocol, encryptor);
+ var (streamId, topicId) = await CreateTestStream(client, protocol);
+
+ await SendBatch(client, streamId, topicId, false);
+
+ using var rental = await client.PollMessagesRentedAsync(streamId, topicId, 0, Consumer.New(1),
+ PollingStrategy.Offset(0), BatchSize, false);
+
+ rental.Messages.Count.ShouldBe(BatchSize);
+ for (var i = 0; i < BatchSize; i++)
+ {
+ var message = rental.Messages[i];
+ Encoding.UTF8.GetString(message.Payload.Span).ShouldBe($"payload-{i}");
+ message.RawUserHeaders.IsEmpty.ShouldBeTrue();
+ message.Header.UserHeadersLength.ShouldBe(0);
+ message.Header.PayloadLength.ShouldBe(Encoding.UTF8.GetByteCount($"payload-{i}"));
+ }
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollRented_WithEncryptor_Should_DecryptBatchWithHeaders(Protocol protocol)
+ {
+ var encryptor = CreateEncryptor();
+ var client = await CreateClient(protocol, encryptor);
+ var (streamId, topicId) = await CreateTestStream(client, protocol);
+
+ // Payload and headers both encrypted: both blobs per message must decrypt into the single rented buffer
+ // at the right offsets.
+ await SendBatch(client, streamId, topicId, true);
+
+ using var rental = await client.PollMessagesRentedAsync(streamId, topicId, 0, Consumer.New(1),
+ PollingStrategy.Offset(0), BatchSize, false);
+
+ rental.Messages.Count.ShouldBe(BatchSize);
+ for (var i = 0; i < BatchSize; i++)
+ {
+ var message = rental.Messages[i];
+ Encoding.UTF8.GetString(message.Payload.Span).ShouldBe($"payload-{i}");
+
+ message.RawUserHeaders.IsEmpty.ShouldBeFalse();
+ Dictionary<HeaderKey, HeaderValue>? headers = message.UserHeaders;
+ headers.ShouldNotBeNull();
+ var typeHeader = headers![new HeaderKey
+ {
+ Kind = HeaderKind.String,
+ Value = "type"u8.ToArray()
+ }];
+ Encoding.UTF8.GetString(typeHeader.Value).ShouldBe("test-message");
+ }
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithEncryptor_Should_DecryptBatch(Protocol protocol)
+ {
+ var encryptor = CreateEncryptor();
+ var client = await CreateClient(protocol, encryptor);
+ var (streamId, topicId) = await CreateTestStream(client, protocol);
+
+ // Materialized poll path (allocates owned arrays): same batch, decrypted before the rental is disposed.
+ await SendBatch(client, streamId, topicId, true);
+
+ var polled = await client.PollMessagesAsync(streamId, topicId, 0, Consumer.New(1), PollingStrategy.Offset(0),
+ BatchSize, false);
+
+ polled.Messages.Count.ShouldBe(BatchSize);
+ for (var i = 0; i < BatchSize; i++)
+ {
+ Encoding.UTF8.GetString(polled.Messages[i].Payload).ShouldBe($"payload-{i}");
+ polled.Messages[i].UserHeaders.ShouldNotBeNull();
+ }
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithWrongKey_Should_ThrowMessageDecryptionException(Protocol protocol)
+ {
+ var publisherEncryptor = CreateEncryptor();
+ var wrongKeyEncryptor = CreateEncryptor();
+
+ var publisherClient = await CreateClient(protocol, publisherEncryptor);
+ var (streamId, topicId) = await CreateTestStream(publisherClient, protocol);
+ await SendBatch(publisherClient, streamId, topicId, false);
+
+ // Same wire bytes, a different key: decryption must fail loudly rather than hand back garbage.
+ var wrongKeyClient = await CreateClient(protocol, wrongKeyEncryptor);
+
+ await Should.ThrowAsync<MessageDecryptionException>(async () =>
+ await wrongKeyClient.PollMessagesAsync(streamId, topicId, 0, Consumer.New(2), PollingStrategy.Offset(0),
+ BatchSize, false));
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveAsync_WithWrongKey_Should_ThrowMessageDecryptionException(Protocol protocol)
+ {
+ var publisherEncryptor = CreateEncryptor();
+ var wrongKeyEncryptor = CreateEncryptor();
+
+ var publisherClient = await CreateClient(protocol, publisherEncryptor);
+ var (streamId, topicId) = await CreateTestStream(publisherClient, protocol);
+ await SendBatch(publisherClient, streamId, topicId, false);
+
+ var wrongKeyClient = await CreateClient(protocol, wrongKeyEncryptor);
+ var consumer = IggyConsumerBuilder
+ .Create(wrongKeyClient, streamId, topicId, Consumer.New(3))
+ .WithPollingStrategy(PollingStrategy.First())
+ .WithBatchSize(10)
+ .WithPartitionId(0)
+ .WithAutoCommitMode(AutoCommitMode.Disabled)
+ .Build();
+
+ await consumer.InitAsync();
+
+ // A bad key is not transient: it must surface out of the stream, not loop re-polling the same offset.
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+ await Should.ThrowAsync<MessageDecryptionException>(async () =>
+ {
+ await foreach (var _ in consumer.ReceiveAsync(cts.Token))
+ {
+ break;
+ }
+ });
+
+ await consumer.DisposeAsync();
+ }
+
+ private async Task SendBatch(IIggyClient client, Identifier streamId, Identifier topicId, bool withHeaders)
+ {
+ var publisher = IggyPublisherBuilder
+ .Create(client, streamId, topicId)
+ .WithPartitioning(Partitioning.PartitionId(0))
+ .Build();
+
+ await publisher.InitAsync();
+
+ var messages = new List<Message>(BatchSize);
+ for (var i = 0; i < BatchSize; i++)
+ {
+ var payload = Encoding.UTF8.GetBytes($"payload-{i}");
+ messages.Add(withHeaders
+ ? new Message(Guid.NewGuid(), payload, CreateTestHeaders())
+ : new Message(Guid.NewGuid(), payload));
+ }
+
+ await publisher.SendMessagesAsync(messages);
+ await publisher.DisposeAsync();
+ }
+
+ private Task<IIggyClient> CreateClient(Protocol protocol, IMessageEncryptor encryptor)
+ {
+ return protocol == Protocol.Tcp
+ ? Fixture.CreateTcpClient(encryptor: encryptor)
+ : Fixture.CreateHttpClient(encryptor: encryptor);
+ }
+
+ private static AesMessageEncryptor CreateEncryptor()
+ {
+ return new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
+ }
+
+ private static Dictionary<HeaderKey, HeaderValue> CreateTestHeaders()
+ {
+ return new Dictionary<HeaderKey, HeaderValue>
+ {
+ {
+ new HeaderKey
+ {
+ Kind = HeaderKind.String,
+ Value = "type"u8.ToArray()
+ },
+ new HeaderValue
+ {
+ Kind = HeaderKind.String,
+ Value = "test-message"u8.ToArray()
+ }
+ }
+ };
+ }
+
+ private static async Task<(Identifier StreamId, Identifier TopicId)> CreateTestStream(IIggyClient client,
+ Protocol protocol)
+ {
+ var streamId = $"enc_msg_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}";
+ var topicId = "enc_msg_topic";
+
+ await client.CreateStreamAsync(streamId);
+ await client.CreateTopicAsync(Identifier.String(streamId), topicId, 1);
+
+ return (Identifier.String(streamId), Identifier.String(topicId));
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs b/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs
index 8c884a7c8..b8c555a81 100644
--- a/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs
+++ b/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
@@ -65,4 +66,19 @@ public sealed class IggyClientConfigurator
/// The logger factory to use.
/// </summary>
public ILoggerFactory LoggerFactory { get; set; } = NullLoggerFactory.Instance;
+
+ /// <summary>
+ /// Optional message encryptor. When set, the client encrypts message payloads and user headers on send
+ /// and decrypts them on poll. Applies to every message on the connection, so topics mixing encrypted
+ /// and plaintext messages are not supported. The caller owns the encryptor and must dispose it
+ /// (e.g. <see cref="AesMessageEncryptor" />) after the client is done with it.
+ /// </summary>
+ public IMessageEncryptor? MessageEncryptor { get; set; }
+
+ /// <summary>
+ /// Opt in to <c>autoCommit: true</c> on the raw poll methods while an encryptor is configured.
+ /// Off by default: the server commits the offset before the client decrypts, so a decryption failure
+ /// would silently skip the batch. Does not affect <see cref="Consumers.IggyConsumer" /> commit modes.
+ /// </summary>
+ public bool AllowAutoCommitWithEncryptor { get; set; }
}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Logging.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Logging.cs
index feb8ba035..e853b2232 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Logging.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Logging.cs
@@ -89,8 +89,8 @@ public partial class IggyConsumer
[LoggerMessage(EventId = 402,
Level = LogLevel.Error,
- Message = "Failed to decrypt message with offset {Offset}")]
- private partial void LogFailedToDecryptMessage(Exception exception, ulong offset);
+ Message = "Failed to decrypt message at offset {Offset}, partition ID {PartitionId}")]
+ private partial void LogFailedToDecryptMessage(Exception exception, ulong offset, uint partitionId);
[LoggerMessage(EventId = 403,
Level = LogLevel.Error,
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
index 8eebde061..e75b60ae3 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
@@ -19,9 +19,7 @@ using System.Runtime.CompilerServices;
using Apache.Iggy.Contracts;
using Apache.Iggy.Enums;
using Apache.Iggy.Exceptions;
-using Apache.Iggy.Headers;
using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
using Microsoft.Extensions.Logging;
namespace Apache.Iggy.Consumers;
@@ -99,7 +97,7 @@ public partial class IggyConsumer
/// <summary>
/// Polls a rented batch from the server and publishes it via <see cref="PublishRentedAsync" />.
- /// Handles decryption, offset tracking, and auto-commit logic. Rental lifetime is managed via
+ /// Handles offset tracking and auto-commit logic. Rental lifetime is managed via
/// <see cref="RentedBatchHandle" /> shared by every produced message.
/// </summary>
protected async Task PollRentedMessagesAsync(CancellationToken ct)
@@ -110,6 +108,8 @@ public partial class IggyConsumer
return;
}
+ ThrowIfAutoCommitWithEncryptor();
+
await _pollingSemaphore.WaitAsync(ct);
PolledMessagesRental? rental = null;
@@ -150,46 +150,11 @@ public partial class IggyConsumer
continue;
}
- var processedMessage = message;
- var status = MessageStatus.Success;
- Exception? error = null;
-
- // TODO: fix encryption allocations by moving it to IggyClient
- if (_config.MessageEncryptor != null)
- {
- try
- {
- var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload.Span);
-
- Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
- if (!message.RawUserHeaders.IsEmpty)
- {
- var decryptedHeaderBytes =
- _config.MessageEncryptor.Decrypt(message.RawUserHeaders.Span);
- decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
- }
-
- processedMessage = new RentedMessageResponse
- {
- Header = message.Header,
- Payload = decryptedPayload,
- RawUserHeaders = ReadOnlyMemory<byte>.Empty,
- UserHeaders = decryptedHeaders
- };
- }
- catch (Exception ex)
- {
- LogFailedToDecryptMessage(ex, message.Header.Offset);
- status = MessageStatus.DecryptionFailed;
- error = ex;
- }
- }
-
currentOffset = message.Header.Offset;
batchHandle.Acquire();
try
{
- await PublishRentedAsync(batchHandle, processedMessage, partitionId, status, error, ct);
+ await PublishRentedAsync(batchHandle, message, partitionId, MessageStatus.Success, null, ct);
}
catch
{
@@ -228,6 +193,17 @@ public partial class IggyConsumer
{
throw;
}
+ catch (MessageDecryptionException ex)
+ {
+ LogFailedToDecryptMessage(ex, ex.Offset, ex.PartitionId);
+ throw;
+ }
+ catch (MalformedResponseException)
+ {
+ // Non-transient poison: rethrow so the generic catch below does not swallow it and re-poll forever.
+ // Base InvalidResponseException (server error status, possibly transient) falls through to retry.
+ throw;
+ }
catch (Exception ex)
{
LogFailedToPollMessages(ex);
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
index 570a6913e..c1201d027 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
@@ -18,13 +18,10 @@
using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using System.Threading.Channels;
-using Apache.Iggy.Contracts;
using Apache.Iggy.Enums;
using Apache.Iggy.Exceptions;
-using Apache.Iggy.Headers;
using Apache.Iggy.IggyClient;
using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
using Apache.Iggy.Utils;
using Microsoft.Extensions.Logging;
@@ -37,7 +34,6 @@ namespace Apache.Iggy.Consumers;
public partial class IggyConsumer : IAsyncDisposable
{
private readonly Channel<ReceivedMessage> _channel;
- private readonly Channel<ReceivedRentedMessage> _rentedChannel;
private readonly IIggyClient _client;
private readonly IggyConsumerConfig _config;
private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1);
@@ -45,6 +41,7 @@ public partial class IggyConsumer : IAsyncDisposable
private readonly ConcurrentDictionary<int, ulong> _lastPolledOffset = new();
private readonly ILogger<IggyConsumer> _logger;
private readonly SemaphoreSlim _pollingSemaphore = new(1, 1);
+ private readonly Channel<ReceivedRentedMessage> _rentedChannel;
private string? _consumerGroupName;
private int _disposeState;
private volatile bool _isInitialized;
@@ -211,7 +208,10 @@ public partial class IggyConsumer : IAsyncDisposable
/// </summary>
/// <param name="offset">The offset to store</param>
/// <param name="partitionId">The partition ID</param>
- /// <param name="resetLastPolled">When true, also advances the cached last-polled offset for the partition so the next poll resumes past the stored offset.</param>
+ /// <param name="resetLastPolled">
+ /// When true, also advances the cached last-polled offset for the partition so the next poll
+ /// resumes past the stored offset.
+ /// </param>
/// <param name="ct">Cancellation token</param>
public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPolled = false,
CancellationToken ct = default)
@@ -343,9 +343,23 @@ public partial class IggyConsumer : IAsyncDisposable
return true;
}
+ /// <summary>
+ /// Guards the wire-driving auto-commit flag against an encrypting client. The builder enforces this via
+ /// AutoCommitMode, but AutoCommit and AutoCommitMode are independent config fields and the consumer is
+ /// constructible without the builder, so the load-bearing check lives here at the poll site.
+ /// </summary>
+ private void ThrowIfAutoCommitWithEncryptor()
+ {
+ if (_config.AutoCommit && _client.MessageEncryptor != null)
+ {
+ throw new InvalidOperationException(
+ "AutoCommit with a message encryptor risks silent message loss: the offset is committed before decryption. Use AutoCommitMode.AfterReceive or AutoCommitMode.Disabled.");
+ }
+ }
+
/// <summary>
/// Polls messages from the server and writes them to the internal channel.
- /// Handles decryption, offset tracking, and auto-commit logic.
+ /// Handles offset tracking and auto-commit logic.
/// Uses semaphore to ensure single concurrent polling operation.
/// </summary>
private async Task PollMessagesAsync(CancellationToken ct)
@@ -356,6 +370,8 @@ public partial class IggyConsumer : IAsyncDisposable
return;
}
+ ThrowIfAutoCommitWithEncryptor();
+
await _pollingSemaphore.WaitAsync(ct);
try
@@ -386,45 +402,13 @@ public partial class IggyConsumer : IAsyncDisposable
continue;
}
- var processedMessage = message;
- var status = MessageStatus.Success;
- Exception? error = null;
-
- if (_config.MessageEncryptor != null)
- {
- try
- {
- var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload);
-
- Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
- if (message.RawUserHeaders is { Length: > 0 })
- {
- var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders);
- decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
- }
-
- processedMessage = new MessageResponse
- {
- Header = message.Header,
- Payload = decryptedPayload,
- UserHeaders = decryptedHeaders
- };
- }
- catch (Exception ex)
- {
- LogFailedToDecryptMessage(ex, message.Header.Offset);
- status = MessageStatus.DecryptionFailed;
- error = ex;
- }
- }
-
var receivedMessage = new ReceivedMessage
{
- Message = processedMessage,
- CurrentOffset = processedMessage.Header.Offset,
+ Message = message,
+ CurrentOffset = message.Header.Offset,
PartitionId = (uint)messages.PartitionId,
- Status = status,
- Error = error
+ Status = MessageStatus.Success,
+ Error = null
};
await _channel.Writer.WriteAsync(receivedMessage, ct);
@@ -452,6 +436,17 @@ public partial class IggyConsumer : IAsyncDisposable
_config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
}
}
+ catch (MessageDecryptionException ex)
+ {
+ LogFailedToDecryptMessage(ex, ex.Offset, ex.PartitionId);
+ throw;
+ }
+ catch (MalformedResponseException)
+ {
+ // Non-transient poison: rethrow so the generic catch below does not swallow it and re-poll forever.
+ // Base InvalidResponseException (server error status, possibly transient) falls through to retry.
+ throw;
+ }
catch (Exception ex)
{
LogFailedToPollMessages(ex);
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs
index e878c85e1..937ca10c5 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs
@@ -31,6 +31,8 @@ namespace Apache.Iggy.Consumers;
/// </summary>
public class IggyConsumerBuilder
{
+ private IMessageEncryptor? _encryptor;
+
internal Func<ConsumerErrorEventArgs, Task>? OnPollingError { get; set; }
internal IggyConsumerConfig Config { get; set; } = new();
internal IIggyClient? IggyClient { get; set; }
@@ -107,13 +109,18 @@ public class IggyConsumerBuilder
}
/// <summary>
- /// Sets a message decryptor for the consumer, enabling decryption of incoming messages.
+ /// Configures message encryption on the client this builder creates. Encryption is a client-level concern:
+ /// the encryptor decrypts on poll (and encrypts on send) for the whole connection. Only valid when the
+ /// builder creates its own client; when an external client is supplied, configure the encryptor on that
+ /// client via <see cref="Configuration.IggyClientConfigurator.MessageEncryptor" /> instead.
+ /// The caller owns the encryptor and must dispose it after the consumer is disposed; the builder never
+ /// disposes it (the same instance may be shared across clients).
/// </summary>
- /// <param name="decryptor">The decryptor implementation to handle message decryption.</param>
+ /// <param name="encryptor">The message encryptor to use.</param>
/// <returns>The current instance of <see cref="IggyConsumerBuilder" /> to allow method chaining.</returns>
- public IggyConsumerBuilder WithDecryptor(IMessageEncryptor decryptor)
+ public IggyConsumerBuilder WithEncryptor(IMessageEncryptor encryptor)
{
- Config.MessageEncryptor = decryptor;
+ _encryptor = encryptor;
return this;
}
@@ -237,8 +244,9 @@ public class IggyConsumerBuilder
BaseAddress = Config.Address,
ReceiveBufferSize = Config.ReceiveBufferSize,
SendBufferSize = Config.SendBufferSize,
- ReconnectionSettings = Config.ReconnectionSettings ?? new(),
- LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance
+ ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(),
+ LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance,
+ MessageEncryptor = _encryptor
});
}
@@ -279,12 +287,25 @@ public class IggyConsumerBuilder
else
{
if (IggyClient == null)
+ {
+ throw new InvalidOperationException("IggyClient must be provided when CreateIggyClient is false.");
+ }
+
+ if (_encryptor != null)
{
throw new InvalidOperationException(
- "IggyClient must be provided when CreateIggyClient is false.");
+ "WithEncryptor is only valid when the builder creates its own client. " +
+ "Configure IggyClientConfigurator.MessageEncryptor on the supplied client instead.");
}
}
+ var hasEncryptor = _encryptor != null || IggyClient?.MessageEncryptor != null;
+ if (hasEncryptor && Config.AutoCommitMode == AutoCommitMode.Auto)
+ {
+ throw new InvalidOperationException(
+ "AutoCommitMode.Auto with a message encryptor risks silent message loss: the offset is committed before decryption. Use AutoCommitMode.AfterReceive or AutoCommitMode.Disabled.");
+ }
+
if (Config.ReceiveBufferSize <= 0)
{
throw new InvalidOperationException("ReceiveBufferSize must be greater than 0.");
@@ -302,8 +323,7 @@ public class IggyConsumerBuilder
if (Config.BatchSize > int.MaxValue / 2)
{
- throw new InvalidOperationException(
- $"BatchSize must be less than or equal to {int.MaxValue / 2}.");
+ throw new InvalidOperationException($"BatchSize must be less than or equal to {int.MaxValue / 2}.");
}
if (Config.PollingIntervalMs < 0)
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerConfig.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerConfig.cs
index f2f445dbe..1418282d3 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerConfig.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerConfig.cs
@@ -16,7 +16,6 @@
// under the License.
using Apache.Iggy.Configuration;
-using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Apache.Iggy.Kinds;
using Microsoft.Extensions.Logging;
@@ -90,11 +89,6 @@ public class IggyConsumerConfig
/// </summary>
public Identifier TopicId { get; set; }
- /// <summary>
- /// Optional message encryptor for decrypting messages
- /// </summary>
- public IMessageEncryptor? MessageEncryptor { get; set; } = null;
-
/// <summary>
/// Optional partition ID to consume from. If null, consumes from all partitions.
/// Note: This is ignored when using consumer groups.
diff --git a/foreign/csharp/Iggy_SDK/Consumers/MessageStatus.cs b/foreign/csharp/Iggy_SDK/Consumers/MessageStatus.cs
index bb16b4d4e..cb3d2eb24 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/MessageStatus.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/MessageStatus.cs
@@ -18,22 +18,21 @@
namespace Apache.Iggy.Consumers;
/// <summary>
-/// Represents the status of a received message
+/// Represents the status of a received message
/// </summary>
public enum MessageStatus
{
/// <summary>
- /// Message was successfully received and processed
+ /// Message was successfully received and processed
/// </summary>
- Success,
+ Success = 0,
/// <summary>
- /// Message decryption failed
+ /// Message deserialization failed
/// </summary>
- DecryptionFailed,
-
- /// <summary>
- /// Message deserialization failed
- /// </summary>
- DeserializationFailed
+ /// <remarks>
+ /// Value 1 was DecryptionFailed (removed; decryption failure now throws MessageDecryptionException).
+ /// Kept at 2 so persisted or compiled-in numeric values stay stable.
+ /// </remarks>
+ DeserializationFailed = 2
}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs
index 24213821c..d147a4bfe 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedMessage.cs
@@ -55,7 +55,7 @@ public sealed class ReceivedMessage<T>
public uint PartitionId { get; init; }
/// <summary>
- /// The status of the message (Success, DecryptionFailed, DeserializationFailed).
+ /// The status of the message (Success, DeserializationFailed).
/// </summary>
public MessageStatus Status { get; init; } = MessageStatus.Success;
@@ -86,7 +86,7 @@ public class ReceivedMessage
public uint PartitionId { get; init; }
/// <summary>
- /// The status of the message (Success, DecryptionFailed, DeserializationFailed)
+ /// The status of the message (Success, DeserializationFailed)
/// </summary>
public MessageStatus Status { get; init; } = MessageStatus.Success;
diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
index 4beaef68d..28a8c8df4 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
@@ -47,7 +47,8 @@ public sealed class ReceivedRentedMessage : IDisposable
public uint PartitionId { get; init; }
/// <summary>
- /// The status of the message (Success, DecryptionFailed).
+ /// The status of the message. The rented path performs no deserialization or decryption, so this is
+ /// always <see cref="MessageStatus.Success" />.
/// </summary>
public MessageStatus Status { get; init; } = MessageStatus.Success;
diff --git a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
index f29ca79fa..6d2b5bc74 100644
--- a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
+++ b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
@@ -82,15 +82,4 @@ public sealed class MessageResponse
_userHeadersInitialized = false;
}
}
-
- internal void ParseUserHeaders()
- {
- if (!_userHeadersInitialized)
- {
- _userHeaders = _rawUserHeaders is { Length: > 0 }
- ? BinaryMapper.TryMapHeaders(_rawUserHeaders)
- : null;
- _userHeadersInitialized = true;
- }
- }
}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
index 188a7b84e..d2dae6aac 100644
--- a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
+++ b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
@@ -25,6 +25,7 @@ namespace Apache.Iggy.Contracts;
public sealed class PolledMessagesRental : IDisposable
{
private readonly IMemoryOwner<byte> _owner;
+ private readonly IMemoryOwner<byte>? _plaintextOwner;
private int _disposed;
/// <summary>
@@ -43,16 +44,21 @@ public sealed class PolledMessagesRental : IDisposable
public required IReadOnlyList<RentedMessageResponse> Messages { get; init; }
/// <summary>
- /// Initializes a new instance of the <see cref="PolledMessagesRental" /> class with the specified memory owner.
+ /// Initializes a new instance of the <see cref="PolledMessagesRental" /> class with the specified memory owner.
/// </summary>
- /// <param name="owner"></param>
- public PolledMessagesRental(IMemoryOwner<byte> owner)
+ /// <param name="owner">Owner of the wire buffer the message slices view into.</param>
+ /// <param name="plaintextOwner">
+ /// Owner of the buffer holding decrypted payloads/headers when an encryptor is configured; null otherwise.
+ /// Tied to the same disposal so decrypted slices stay valid for exactly the rental's lifetime.
+ /// </param>
+ public PolledMessagesRental(IMemoryOwner<byte> owner, IMemoryOwner<byte>? plaintextOwner = null)
{
_owner = owner;
+ _plaintextOwner = plaintextOwner;
}
/// <summary>
- /// Disposes the rental and returns the underlying buffer to the pool.
+ /// Disposes the rental and returns the underlying buffers to the pool.
/// </summary>
public void Dispose()
{
@@ -61,6 +67,7 @@ public sealed class PolledMessagesRental : IDisposable
return;
}
+ _plaintextOwner?.Dispose();
_owner.Dispose();
}
}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs
index c0d3ae1a6..e96e6ddd8 100644
--- a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs
+++ b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs
@@ -15,10 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+using System.Buffers;
using System.Buffers.Binary;
using System.Runtime.CompilerServices;
using System.Text;
using Apache.Iggy.Contracts.Auth;
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Apache.Iggy.Extensions;
using Apache.Iggy.Headers;
@@ -385,8 +387,8 @@ internal static class TcpContracts
bytes[position + 18] = autoCommit ? (byte)1 : (byte)0;
}
- internal static void CreateMessage(Span<byte> bytes, Identifier streamId, Identifier topicId,
- Partitioning partitioning, ReadOnlySpan<Message> messages)
+ internal static int CreateMessage(Span<byte> bytes, Identifier streamId, Identifier topicId,
+ Partitioning partitioning, ReadOnlySpan<Message> messages, IMessageEncryptor? encryptor = null)
{
var metadataLength = 2 + streamId.Length + 2 + topicId.Length + 2 + partitioning.Length + 4;
BinaryPrimitives.WriteInt32LittleEndian(bytes[..4], metadataLength);
@@ -401,49 +403,110 @@ internal static class TcpContracts
position += 16 * messages.Length;
var msgSize = 0;
- foreach (var message in messages)
- {
- ReadOnlyMemory<byte> rawHeaders = message.RawUserHeaders;
- var headersLength = rawHeaders.IsEmpty ? HeadersByteLength(message.UserHeaders) : rawHeaders.Length;
- var payloadLength = message.Payload.Length;
- var header = message.Header;
-
- BinaryPrimitives.WriteUInt64LittleEndian(bytes[position..(position + 8)], 0); // checksum is server-computed
- BinaryPrimitives.WriteUInt128LittleEndian(bytes[(position + 8)..(position + 24)], header.Id);
- BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 24)..(position + 32)], header.Offset);
- BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 32)..(position + 40)],
- DateTimeOffsetUtils.ToUnixTimeMicroSeconds(header.Timestamp));
- BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 40)..(position + 48)],
- header.OriginTimestamp);
- BinaryPrimitives.WriteInt32LittleEndian(bytes[(position + 48)..(position + 52)], headersLength);
- BinaryPrimitives.WriteInt32LittleEndian(bytes[(position + 52)..(position + 56)], payloadLength);
- // Reserved must be zero on the wire; the server rejects non-zero values.
- BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 56)..(position + 64)], 0);
-
- message.Payload.Span.CopyTo(bytes[(position + 64)..(position + 64 + payloadLength)]);
- if (headersLength > 0)
+
+ // One scratch buffer reused across the batch (grown on demand); holds plaintext headers, so it is
+ // returned cleared on every path, including a mid-batch throw from the encryptor or header serialization.
+ byte[]? headerScratch = null;
+ try
+ {
+ foreach (var message in messages)
{
- var headersStart = position + 64 + payloadLength;
- Span<byte> headersDestination = bytes[headersStart..(headersStart + headersLength)];
- if (rawHeaders.IsEmpty)
+ var header = message.Header;
+ var payloadStart = position + 64;
+
+ int payloadLength;
+ if (encryptor is null)
{
- WriteHeadersTo(headersDestination, message.UserHeaders!);
+ payloadLength = message.Payload.Length;
+ message.Payload.Span.CopyTo(bytes[payloadStart..(payloadStart + payloadLength)]);
}
else
{
- rawHeaders.Span.CopyTo(headersDestination);
+ // Bound the destination to this message's reserved share of the buffer so an encryptor that
+ // overruns its GetMaxEncryptedLength contract fails fast here instead of corrupting the batch.
+ Span<byte> payloadDestination =
+ bytes.Slice(payloadStart, encryptor.GetMaxEncryptedLength(message.Payload.Length));
+ payloadLength = encryptor.Encrypt(message.Payload.Span, payloadDestination);
}
- }
- position += 64 + payloadLength + headersLength;
-
- msgSize += 64 + payloadLength + headersLength;
+ ReadOnlyMemory<byte> rawHeaders = message.RawUserHeaders;
+ var plainHeadersLength =
+ rawHeaders.IsEmpty ? HeadersByteLength(message.UserHeaders) : rawHeaders.Length;
+ var headersStart = payloadStart + payloadLength;
+ var headersLength = 0;
+ if (plainHeadersLength > 0)
+ {
+ if (encryptor is null)
+ {
+ headersLength = plainHeadersLength;
+ Span<byte> headersDestination = bytes[headersStart..(headersStart + headersLength)];
+ if (rawHeaders.IsEmpty)
+ {
+ WriteHeadersTo(headersDestination, message.UserHeaders!);
+ }
+ else
+ {
+ rawHeaders.Span.CopyTo(headersDestination);
+ }
+ }
+ else
+ {
+ Span<byte> headersDestination =
+ bytes.Slice(headersStart, encryptor.GetMaxEncryptedLength(plainHeadersLength));
+ if (rawHeaders.IsEmpty)
+ {
+ if (headerScratch is null || headerScratch.Length < plainHeadersLength)
+ {
+ if (headerScratch is not null)
+ {
+ ArrayPool<byte>.Shared.Return(headerScratch, true);
+ }
+
+ headerScratch = ArrayPool<byte>.Shared.Rent(plainHeadersLength);
+ }
+
+ WriteHeadersTo(headerScratch.AsSpan(0, plainHeadersLength), message.UserHeaders!);
+ headersLength = encryptor.Encrypt(headerScratch.AsSpan(0, plainHeadersLength),
+ headersDestination);
+ }
+ else
+ {
+ headersLength = encryptor.Encrypt(rawHeaders.Span, headersDestination);
+ }
+ }
+ }
- BinaryPrimitives.WriteInt32LittleEndian(bytes[indexPosition..(indexPosition + 4)], 0);
- BinaryPrimitives.WriteInt32LittleEndian(bytes[(indexPosition + 4)..(indexPosition + 8)], msgSize);
- BinaryPrimitives.WriteInt64LittleEndian(bytes[(indexPosition + 8)..(indexPosition + 16)], 0);
- indexPosition += 16;
+ BinaryPrimitives.WriteUInt64LittleEndian(bytes[position..(position + 8)], 0);
+ BinaryPrimitives.WriteUInt128LittleEndian(bytes[(position + 8)..(position + 24)], header.Id);
+ BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 24)..(position + 32)], header.Offset);
+ BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 32)..(position + 40)],
+ DateTimeOffsetUtils.ToUnixTimeMicroSeconds(header.Timestamp));
+ BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 40)..(position + 48)],
+ header.OriginTimestamp);
+ BinaryPrimitives.WriteInt32LittleEndian(bytes[(position + 48)..(position + 52)], headersLength);
+ BinaryPrimitives.WriteInt32LittleEndian(bytes[(position + 52)..(position + 56)], payloadLength);
+ // Reserved must be zero on the wire; the server rejects non-zero values.
+ BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 56)..(position + 64)], 0);
+
+ position += 64 + payloadLength + headersLength;
+
+ msgSize += 64 + payloadLength + headersLength;
+
+ BinaryPrimitives.WriteInt32LittleEndian(bytes[indexPosition..(indexPosition + 4)], 0);
+ BinaryPrimitives.WriteInt32LittleEndian(bytes[(indexPosition + 4)..(indexPosition + 8)], msgSize);
+ BinaryPrimitives.WriteInt64LittleEndian(bytes[(indexPosition + 8)..(indexPosition + 16)], 0);
+ indexPosition += 16;
+ }
+ }
+ finally
+ {
+ if (headerScratch is not null)
+ {
+ ArrayPool<byte>.Shared.Return(headerScratch, true);
+ }
}
+
+ return position;
}
internal static int HeadersByteLength(Dictionary<HeaderKey, HeaderValue>? headers)
diff --git a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
index 6de51ce4b..d32eb2b73 100644
--- a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
+++ b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
@@ -22,77 +22,128 @@ namespace Apache.Iggy.Encryption;
/// <summary>
/// AES-GCM based message encryptor for secure message encryption/decryption.
/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity.
+/// Output format: [12-byte nonce][ciphertext][16-byte authentication tag].
/// </summary>
-public sealed class AesMessageEncryptor : IMessageEncryptor
+/// <remarks>
+/// Nonces are random per invocation, so NIST SP 800-38D caps a single key at 2^32 encryptions before
+/// nonce-collision probability grows unacceptable; a message with headers spends two invocations. Rotate
+/// the key well before that bound: a collision breaks confidentiality and integrity for the whole key.
+/// Dispose to release the per-thread cipher contexts and zero the key copy.
+/// </remarks>
+public sealed class AesMessageEncryptor : IMessageEncryptor, IDisposable
{
private const int NonceSize = 12; // 96 bits - recommended for GCM
private const int TagSize = 16; // 128 bits authentication tag
- // AesGcm instance members are not thread-safe (shared native cipher context), so a fresh instance is
- // created per call: a single encryptor can be shared across concurrent publisher/consumer threads.
+ // AesGcm instances are not thread-safe. One per thread lets a shared encryptor serve concurrent threads while
+ // reusing the native key schedule instead of rebuilding a cipher per call. trackAllValues so Dispose can
+ // release every thread's context.
+ private readonly ThreadLocal<AesGcm> _aesGcm;
private readonly byte[] _key;
+ private int _disposed;
/// <summary>
/// Creates a new AES message encryptor with the specified key.
/// </summary>
/// <param name="key">The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively.</param>
+ /// <exception cref="ArgumentNullException">Thrown when key is null</exception>
/// <exception cref="ArgumentException">Thrown when key length is invalid</exception>
public AesMessageEncryptor(byte[] key)
{
+ ArgumentNullException.ThrowIfNull(key);
+
if (key.Length != 16 && key.Length != 24 && key.Length != 32)
{
throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256",
nameof(key));
}
- _key = key;
+ // Own copy: caller mutating its array must not desync lazily-created ciphers, and Dispose can zero this
+ // without clobbering caller memory.
+ _key = (byte[])key.Clone();
+ _aesGcm = new ThreadLocal<AesGcm>(() => new AesGcm(_key, TagSize), true);
}
/// <summary>
- /// Encrypts the provided plain data using AES-GCM.
- /// Format: [12-byte nonce][encrypted data][16-byte authentication tag]
+ /// Releases every thread's native cipher context and zeroes the internal key copy. The encryptor must not
+ /// be used concurrently with or after disposal.
/// </summary>
- /// <param name="plainData">The plain data to encrypt</param>
- /// <returns>The encrypted data with nonce and tag</returns>
- public byte[] Encrypt(ReadOnlySpan<byte> plainData)
+ public void Dispose()
{
- var result = new byte[NonceSize + plainData.Length + TagSize];
- Span<byte> nonce = result.AsSpan(0, NonceSize);
- Span<byte> ciphertext = result.AsSpan(NonceSize, plainData.Length);
- Span<byte> tag = result.AsSpan(NonceSize + plainData.Length, TagSize);
+ if (Interlocked.Exchange(ref _disposed, 1) == 1)
+ {
+ return;
+ }
+
+ foreach (var aesGcm in _aesGcm.Values)
+ {
+ aesGcm.Dispose();
+ }
+
+ _aesGcm.Dispose();
+ CryptographicOperations.ZeroMemory(_key);
+ GC.SuppressFinalize(this);
+ }
+
+ /// <summary>
+ /// Exact ciphertext length for AES-GCM: plaintext plus nonce and tag.
+ /// </summary>
+ public int GetMaxEncryptedLength(int plaintextLength)
+ {
+ return NonceSize + plaintextLength + TagSize;
+ }
+
+ /// <summary>
+ /// Exact plaintext length for AES-GCM: ciphertext minus nonce and tag.
+ /// </summary>
+ public int GetMaxDecryptedLength(int encryptedLength)
+ {
+ return Math.Max(0, encryptedLength - NonceSize - TagSize);
+ }
+
+ /// <summary>
+ /// Encrypts <paramref name="plaintext" /> with AES-GCM into <paramref name="destination" />, returning the
+ /// number of bytes written. Format: [12-byte nonce][encrypted data][16-byte authentication tag].
+ /// </summary>
+ public int Encrypt(ReadOnlySpan<byte> plaintext, Span<byte> destination)
+ {
+ ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
+
+ var totalLength = NonceSize + plaintext.Length + TagSize;
+ Span<byte> nonce = destination[..NonceSize];
+ Span<byte> ciphertext = destination.Slice(NonceSize, plaintext.Length);
+ Span<byte> tag = destination.Slice(NonceSize + plaintext.Length, TagSize);
RandomNumberGenerator.Fill(nonce);
- using var aesGcm = new AesGcm(_key, TagSize);
- aesGcm.Encrypt(nonce, plainData, ciphertext, tag);
- return result;
+ _aesGcm.Value!.Encrypt(nonce, plaintext, ciphertext, tag);
+
+ return totalLength;
}
/// <summary>
- /// Decrypts the provided encrypted data using AES-GCM.
- /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag]
+ /// Decrypts <paramref name="encryptedData" /> into <paramref name="destination" /> using AES-GCM, returning
+ /// the number of plaintext bytes written.
/// </summary>
- /// <param name="encryptedData">The encrypted data with nonce and tag</param>
- /// <returns>The decrypted plain data</returns>
/// <exception cref="ArgumentException">Thrown when encrypted data format is invalid</exception>
/// <exception cref="CryptographicException">Thrown when decryption or authentication fails</exception>
- public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
+ public int Decrypt(ReadOnlySpan<byte> encryptedData, Span<byte> destination)
{
+ ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
+
if (encryptedData.Length < NonceSize + TagSize)
{
throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData));
}
- var ciphertextLength = encryptedData.Length - NonceSize - TagSize;
+ var plaintextLength = encryptedData.Length - NonceSize - TagSize;
ReadOnlySpan<byte> nonce = encryptedData[..NonceSize];
- ReadOnlySpan<byte> ciphertext = encryptedData.Slice(NonceSize, ciphertextLength);
- ReadOnlySpan<byte> tag = encryptedData.Slice(NonceSize + ciphertextLength, TagSize);
+ ReadOnlySpan<byte> ciphertext = encryptedData.Slice(NonceSize, plaintextLength);
+ ReadOnlySpan<byte> tag = encryptedData.Slice(NonceSize + plaintextLength, TagSize);
- var plaintext = new byte[ciphertextLength];
- using var aesGcm = new AesGcm(_key, TagSize);
- aesGcm.Decrypt(nonce, ciphertext, tag, plaintext);
+ _aesGcm.Value!.Decrypt(nonce, ciphertext, tag, destination[..plaintextLength]);
- return plaintext;
+ return plaintextLength;
}
/// <summary>
@@ -100,6 +151,8 @@ public sealed class AesMessageEncryptor : IMessageEncryptor
/// </summary>
/// <param name="base64Key">The base64-encoded encryption key; must decode to 16, 24, or 32 bytes</param>
/// <returns>A new AesMessageEncryptor instance</returns>
+ /// <exception cref="FormatException">Thrown when the key is not valid base64</exception>
+ /// <exception cref="ArgumentException">Thrown when the decoded key length is invalid</exception>
public static AesMessageEncryptor FromBase64Key(string base64Key)
{
return new AesMessageEncryptor(Convert.FromBase64String(base64Key));
diff --git a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
index 55ab75768..65fa293a2 100644
--- a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
+++ b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
@@ -18,25 +18,43 @@
namespace Apache.Iggy.Encryption;
/// <summary>
-/// Interface for encrypting and decrypting message payloads in Iggy messaging system.
-/// Implementations of this interface can be used with IggyPublisher and IggyConsumer
-/// to provide end-to-end encryption of message data.
+/// Encrypts and decrypts message payloads (and user headers) at the wire boundary: the client encrypts
+/// directly into the outgoing wire buffer on send and decrypts into a pooled buffer on poll, so neither
+/// direction allocates per message. Implementations write into a caller-provided destination sized by the
+/// corresponding GetMax*Length bound and return the number of bytes actually written.
/// </summary>
+/// <remarks>
+/// Implementations must be thread-safe: a single instance is shared by every publisher and consumer on the
+/// client and may encrypt or decrypt on concurrent threads. Nothing enforces this at the boundary: an
+/// implementation that shares mutable cipher state across calls will silently corrupt ciphertext or crash
+/// under concurrent send/poll. See <see cref="AesMessageEncryptor" /> for the built-in AEAD implementation.
+/// </remarks>
public interface IMessageEncryptor
{
/// <summary>
- /// Encrypts the provided plain data. Takes a span so the send path can encrypt
- /// straight from pooled or rented payload memory.
+ /// A safe upper bound on the ciphertext length produced by encrypting <paramref name="plaintextLength" />
+ /// bytes, used to size the wire buffer before encryption runs. Exact for AEAD ciphers
+ /// (plaintext + nonce + tag).
/// </summary>
- /// <param name="plainData">The plain data to encrypt</param>
- /// <returns>The encrypted data</returns>
- byte[] Encrypt(ReadOnlySpan<byte> plainData);
+ int GetMaxEncryptedLength(int plaintextLength);
/// <summary>
- /// Decrypts the provided encrypted data. Takes a span so the consumer path can
- /// decrypt straight from rented payload memory.
+ /// A safe upper bound on the plaintext length produced by decrypting <paramref name="encryptedLength" />
+ /// bytes, used to size the pooled buffer the poll path decrypts into.
/// </summary>
- /// <param name="encryptedData">The encrypted data to decrypt</param>
- /// <returns>The decrypted plain data</returns>
- byte[] Decrypt(ReadOnlySpan<byte> encryptedData);
+ int GetMaxDecryptedLength(int encryptedLength);
+
+ /// <summary>
+ /// Encrypts <paramref name="plaintext" /> into <paramref name="destination" /> and returns the number of
+ /// bytes written. The destination is at least <see cref="GetMaxEncryptedLength" /> bytes and does not
+ /// overlap the source.
+ /// </summary>
+ int Encrypt(ReadOnlySpan<byte> plaintext, Span<byte> destination);
+
+ /// <summary>
+ /// Decrypts <paramref name="encryptedData" /> into <paramref name="destination" /> and returns the number
+ /// of bytes written. The destination is at least <see cref="GetMaxDecryptedLength" /> bytes and does not
+ /// overlap the source. Throws on authentication failure or malformed input.
+ /// </summary>
+ int Decrypt(ReadOnlySpan<byte> encryptedData, Span<byte> destination);
}
diff --git a/foreign/csharp/Iggy_SDK/Exceptions/InvalidResponseException.cs b/foreign/csharp/Iggy_SDK/Exceptions/InvalidResponseException.cs
index bcb8f7452..d0133002c 100644
--- a/foreign/csharp/Iggy_SDK/Exceptions/InvalidResponseException.cs
+++ b/foreign/csharp/Iggy_SDK/Exceptions/InvalidResponseException.cs
@@ -18,9 +18,11 @@
namespace Apache.Iggy.Exceptions;
/// <summary>
-/// Exception thrown when the response from the server is invalid.
+/// Exception thrown when the response from the server is invalid. Covers server-rejected commands
+/// (error status responses), which may be transient; see <see cref="MalformedResponseException" /> for
+/// unparseable frames the consumer treats as non-retryable poison.
/// </summary>
-public sealed class InvalidResponseException : Exception
+public class InvalidResponseException : Exception
{
/// <summary>
/// Initializes a new instance of the <see cref="InvalidResponseException" /> class.
diff --git a/foreign/csharp/Iggy_SDK/Exceptions/InvalidResponseException.cs b/foreign/csharp/Iggy_SDK/Exceptions/MalformedResponseException.cs
similarity index 62%
copy from foreign/csharp/Iggy_SDK/Exceptions/InvalidResponseException.cs
copy to foreign/csharp/Iggy_SDK/Exceptions/MalformedResponseException.cs
index bcb8f7452..f22246eaa 100644
--- a/foreign/csharp/Iggy_SDK/Exceptions/InvalidResponseException.cs
+++ b/foreign/csharp/Iggy_SDK/Exceptions/MalformedResponseException.cs
@@ -18,14 +18,14 @@
namespace Apache.Iggy.Exceptions;
/// <summary>
-/// Exception thrown when the response from the server is invalid.
+/// Thrown when a server response frame cannot be parsed (e.g. a negative length field). Unlike its base
+/// <see cref="InvalidResponseException" />, which also covers server-rejected commands that may succeed on
+/// retry, a malformed frame is non-transient poison: the consumer re-throws it out of the receive
+/// enumerator instead of retrying the poll forever.
/// </summary>
-public sealed class InvalidResponseException : Exception
+public sealed class MalformedResponseException : InvalidResponseException
{
- /// <summary>
- /// Initializes a new instance of the <see cref="InvalidResponseException" /> class.
- /// </summary>
- internal InvalidResponseException(string message) : base(message)
+ internal MalformedResponseException(string message) : base(message)
{
}
}
diff --git a/foreign/csharp/Iggy_SDK/Exceptions/MessageDecryptionException.cs b/foreign/csharp/Iggy_SDK/Exceptions/MessageDecryptionException.cs
new file mode 100644
index 000000000..3ce31035f
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK/Exceptions/MessageDecryptionException.cs
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.Iggy.Exceptions;
+
+/// <summary>
+/// Thrown when a polled message cannot be decrypted. The whole batch is aborted and its offset is not
+/// committed, so the consumer stays stuck at the poison message until it is skipped with
+/// <c>StoreOffsetAsync(<see cref="Offset" /> + 1, <see cref="PartitionId" />)</c> (effective only for
+/// server-tracked strategies such as <c>PollingStrategy.Next()</c> or consumer groups).
+/// WARNING: the abort also discards messages before <see cref="Offset" /> that decrypted fine, so skipping
+/// past the failure silently skips those too; re-poll the range below <see cref="Offset" /> first to recover
+/// them.
+/// </summary>
+public sealed class MessageDecryptionException : Exception
+{
+ /// <summary>
+ /// Offset of the message that failed to decrypt.
+ /// </summary>
+ public ulong Offset { get; }
+
+ /// <summary>
+ /// Partition the failing message was polled from.
+ /// </summary>
+ public uint PartitionId { get; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="MessageDecryptionException" /> class.
+ /// </summary>
+ /// <param name="offset">Offset of the message that failed to decrypt.</param>
+ /// <param name="partitionId">Partition the failing message was polled from.</param>
+ /// <param name="innerException">The underlying cryptographic failure.</param>
+ public MessageDecryptionException(ulong offset, uint partitionId, Exception innerException)
+ : base($"Failed to decrypt message at offset {offset} in partition {partitionId}.", innerException)
+ {
+ Offset = offset;
+ PartitionId = partitionId;
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs b/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs
index f489e1164..430046490 100644
--- a/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs
+++ b/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs
@@ -24,19 +24,26 @@ using Apache.Iggy.IggyClient.Implementations;
namespace Apache.Iggy.Factory;
/// <summary>
-/// A static factory for creating instances of <see cref="IIggyClient"/>.
+/// A static factory for creating instances of <see cref="IIggyClient" />.
/// </summary>
/// <remarks>
-/// The factory determines the appropriate implementation of the <see cref="IIggyClient"/> based on the specified protocol in the configurator options.
+/// The factory determines the appropriate implementation of the <see cref="IIggyClient" /> based on the specified
+/// protocol in the configurator options.
/// </remarks>
public static class IggyClientFactory
{
/// <summary>
- /// Creates and returns an instance of <see cref="IIggyClient"/> based on the provided configuration options.
+ /// Creates and returns an instance of <see cref="IIggyClient" /> based on the provided configuration options.
/// </summary>
- /// <param name="options">The configuration options for creating the Iggy client, including protocol, base address, and buffer sizes.</param>
- /// <returns>An instance of <see cref="IIggyClient"/> configured according to the specified options.</returns>
- /// <exception cref="InvalidEnumArgumentException">Thrown when the specified protocol in <paramref name="options"/> is not supported.</exception>
+ /// <param name="options">
+ /// The configuration options for creating the Iggy client, including protocol, base address, and
+ /// buffer sizes.
+ /// </param>
+ /// <returns>An instance of <see cref="IIggyClient" /> configured according to the specified options.</returns>
+ /// <exception cref="InvalidEnumArgumentException">
+ /// Thrown when the specified protocol in <paramref name="options" /> is not
+ /// supported.
+ /// </exception>
public static IIggyClient CreateClient(IggyClientConfigurator options)
{
return options.Protocol switch
@@ -54,7 +61,8 @@ public static class IggyClientFactory
private static IIggyClient CreateIggyHttpClient(IggyClientConfigurator options)
{
- return new HttpMessageStream(CreateHttpClient(options));
+ return new HttpMessageStream(CreateHttpClient(options), options.MessageEncryptor,
+ options.AllowAutoCommitWithEncryptor);
}
private static HttpClient CreateHttpClient(IggyClientConfigurator options)
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyClient.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyClient.cs
index d8b28296c..bcc7a0a26 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyClient.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyClient.cs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+using Apache.Iggy.Encryption;
+
namespace Apache.Iggy.IggyClient;
/// <summary>
@@ -25,6 +27,13 @@ namespace Apache.Iggy.IggyClient;
public interface IIggyClient : IIggyPublisher, IIggyStream, IIggyTopic, IIggyConsumer, IIggyOffset, IIggyConsumerGroup,
IIggySystem, IIggyPartition, IIggySegment, IIggyUsers, IIggyPersonalAccessToken, IDisposable
{
+ /// <summary>
+ /// The message encryptor configured on this client, or null when encryption is disabled. When set, the client
+ /// encrypts on send and decrypts on poll for the whole connection. Defaults to null so existing
+ /// implementations keep compiling.
+ /// </summary>
+ IMessageEncryptor? MessageEncryptor => null;
+
/// <summary>
/// Subscribes to connection state change events.
/// </summary>
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs
index 453090f1f..483814a63 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs
@@ -35,6 +35,11 @@ public interface IIggyPublisher
/// - PartitionId: Messages are sent to a specific partition.
/// - MessagesKey: Partition is selected based on a key value, ensuring all messages with the same key go to the same
/// partition.
+ /// <para>
+ /// When the client is configured with an encryptor, each <see cref="Message" /> payload (and user
+ /// headers) is encrypted at the wire boundary while the outgoing buffer is serialized; the caller's
+ /// instances are never mutated and keep their plaintext.
+ /// </para>
/// </remarks>
/// <param name="streamId">The stream identifier (numeric ID or name).</param>
/// <param name="topicId">The topic identifier (numeric ID or name).</param>
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
index 9b1ce3905..96bd38520 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+using System.Buffers;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
@@ -25,6 +26,8 @@ using Apache.Iggy.Contracts;
using Apache.Iggy.Contracts.Auth;
using Apache.Iggy.Contracts.Http;
using Apache.Iggy.Contracts.Http.Auth;
+using Apache.Iggy.Contracts.Tcp;
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Apache.Iggy.Exceptions;
using Apache.Iggy.Kinds;
@@ -43,6 +46,7 @@ public class HttpMessageStream : IIggyClient
{
private const string Context = "csharp-sdk";
+ private readonly bool _allowAutoCommitWithEncryptor;
private readonly HttpClient _httpClient;
//TODO - create mechanism for refreshing jwt token
@@ -50,9 +54,12 @@ public class HttpMessageStream : IIggyClient
//TODO - the error handling pattern is pretty ugly, look into moving it into an extension method
private readonly JsonSerializerOptions _jsonSerializerOptions;
- internal HttpMessageStream(HttpClient httpClient)
+ internal HttpMessageStream(HttpClient httpClient, IMessageEncryptor? encryptor = null,
+ bool allowAutoCommitWithEncryptor = false)
{
_httpClient = httpClient;
+ MessageEncryptor = encryptor;
+ _allowAutoCommitWithEncryptor = allowAutoCommitWithEncryptor;
_jsonSerializerOptions = new JsonSerializerOptions
{
@@ -248,6 +255,17 @@ public class HttpMessageStream : IIggyClient
IList<Message> messages,
CancellationToken token = default)
{
+ if (MessageEncryptor is not null)
+ {
+ var encrypted = new List<Message>(messages.Count);
+ foreach (var message in messages)
+ {
+ encrypted.Add(EncryptCopy(message, MessageEncryptor));
+ }
+
+ messages = encrypted;
+ }
+
var request = new MessageSendRequest
{
StreamId = streamId,
@@ -287,6 +305,15 @@ public class HttpMessageStream : IIggyClient
Consumer consumer,
PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
{
+ if (autoCommit && MessageEncryptor is not null && !_allowAutoCommitWithEncryptor)
+ {
+ // Server-side autoCommit commits the batch offset before the client decrypts, so a decryption failure
+ // would permanently skip the whole batch. IggyConsumer guards this too, but the raw poll is public and
+ // bypasses that path. Opt out via IggyClientConfigurator.AllowAutoCommitWithEncryptor.
+ throw new InvalidOperationException(
+ "AutoCommit with a message encryptor risks silent message loss: the offset is committed before decryption. Poll with autoCommit false, or set AllowAutoCommitWithEncryptor.");
+ }
+
var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" +
$"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}");
@@ -297,6 +324,11 @@ public class HttpMessageStream : IIggyClient
var pollMessages = await response.Content.ReadFromJsonAsync<PolledMessages>(_jsonSerializerOptions, token)
?? PolledMessages.Empty;
+ if (MessageEncryptor is not null)
+ {
+ DecryptMessages(pollMessages.Messages, (uint)pollMessages.PartitionId);
+ }
+
return pollMessages;
}
@@ -831,12 +863,109 @@ public class HttpMessageStream : IIggyClient
{
}
+ /// <inheritdoc />
+ public IMessageEncryptor? MessageEncryptor { get; }
+
/// <inheritdoc />
public string GetCurrentAddress()
{
return _httpClient.BaseAddress?.ToString() ?? string.Empty;
}
+ private void DecryptMessages(IReadOnlyList<MessageResponse> messages, uint partitionId)
+ {
+ foreach (var message in messages)
+ {
+ byte[] payload;
+ byte[]? rawHeaders = null;
+ try
+ {
+ payload = Decrypt(MessageEncryptor!, message.Payload);
+ if (message.RawUserHeaders is { Length: > 0 })
+ {
+ rawHeaders = Decrypt(MessageEncryptor!, message.RawUserHeaders);
+ }
+ }
+ catch (Exception ex)
+ {
+ throw new MessageDecryptionException(message.Header.Offset, partitionId, ex);
+ }
+
+ message.Payload = payload;
+ message.Header = message.Header with { PayloadLength = payload.Length };
+
+ if (rawHeaders is not null)
+ {
+ message.RawUserHeaders = rawHeaders;
+ message.Header = message.Header with { UserHeadersLength = rawHeaders.Length };
+ }
+ }
+ }
+
+ /// <summary>
+ /// Builds an encrypted copy of <paramref name="message" /> for the send path, leaving the caller's
+ /// instance untouched with its plaintext payload. HTTP serializes messages to JSON, so it needs
+ /// standalone ciphertext arrays rather than a slice of a wire buffer.
+ /// </summary>
+ internal static Message EncryptCopy(Message message, IMessageEncryptor encryptor)
+ {
+ var payload = Encrypt(encryptor, message.Payload.Span);
+ var copy = new Message
+ {
+ Header = message.Header with { PayloadLength = payload.Length },
+ Payload = payload
+ };
+
+ if (!message.RawUserHeaders.IsEmpty)
+ {
+ var rawUserHeaders = Encrypt(encryptor, message.RawUserHeaders.Span);
+ copy.RawUserHeaders = rawUserHeaders;
+ copy.Header = copy.Header with { UserHeadersLength = rawUserHeaders.Length };
+ }
+ else if (message.UserHeaders is { Count: > 0 })
+ {
+ var length = TcpContracts.HeadersByteLength(message.UserHeaders);
+ var scratch = ArrayPool<byte>.Shared.Rent(length);
+ try
+ {
+ TcpContracts.WriteHeadersTo(scratch.AsSpan(0, length), message.UserHeaders);
+ var rawUserHeaders = Encrypt(encryptor, scratch.AsSpan(0, length));
+ copy.RawUserHeaders = rawUserHeaders;
+ copy.Header = copy.Header with { UserHeadersLength = rawUserHeaders.Length };
+ }
+ finally
+ {
+ ArrayPool<byte>.Shared.Return(scratch, true);
+ }
+ }
+
+ return copy;
+ }
+
+ private static byte[] Encrypt(IMessageEncryptor encryptor, ReadOnlySpan<byte> data)
+ {
+ var ciphertext = new byte[encryptor.GetMaxEncryptedLength(data.Length)];
+ var written = encryptor.Encrypt(data, ciphertext);
+ if (written != ciphertext.Length)
+ {
+ Array.Resize(ref ciphertext, written);
+ }
+
+ return ciphertext;
+ }
+
+ private static byte[] Decrypt(IMessageEncryptor encryptor, ReadOnlySpan<byte> data)
+ {
+ var plaintext = new byte[encryptor.GetMaxDecryptedLength(data.Length)];
+ var written = encryptor.Decrypt(data, plaintext);
+ if (written != plaintext.Length)
+ {
+ Array.Resize(ref plaintext, written);
+ }
+
+ return plaintext;
+ }
+
private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false)
{
if ((int)response.StatusCode > 300
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
index 937ac723a..373488ec1 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
@@ -28,6 +28,7 @@ using Apache.Iggy.ConnectionStream;
using Apache.Iggy.Contracts;
using Apache.Iggy.Contracts.Auth;
using Apache.Iggy.Contracts.Tcp;
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Apache.Iggy.Exceptions;
using Apache.Iggy.Kinds;
@@ -91,6 +92,9 @@ public sealed class TcpMessageStream : IIggyClient
_connectionEvents.Unsubscribe(callback);
}
+ /// <inheritdoc />
+ public IMessageEncryptor? MessageEncryptor => _configuration.MessageEncryptor;
+
/// <inheritdoc />
public string GetCurrentAddress()
{
@@ -274,7 +278,9 @@ public sealed class TcpMessageStream : IIggyClient
/// <inheritdoc />
public Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
IList<Message> messages, CancellationToken token = default)
- => SendMessagesCoreAsync(streamId, topicId, partitioning, AsSpan(messages), token);
+ {
+ return SendMessagesCoreAsync(streamId, topicId, partitioning, AsSpan(messages), token);
+ }
/// <inheritdoc />
public Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
@@ -284,50 +290,6 @@ public sealed class TcpMessageStream : IIggyClient
return SendMessagesCoreAsync(streamId, topicId, partitioning, span, token);
}
- private Task SendMessagesCoreAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
- ReadOnlySpan<Message> messages, CancellationToken token)
- {
- var metadataLength = 2 + streamId.Length + 2 + topicId.Length
- + 2 + partitioning.Length + 4 + 4;
- var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages)
- + metadataLength;
- var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
-
- IMemoryOwner<byte> payloadBuffer = MemoryPool<byte>.Shared.Rent(payloadBufferSize);
- try
- {
- FillSendMessagesPayload(payloadBuffer.Memory.Span, messageBufferSize, streamId, topicId,
- partitioning, messages);
- }
- catch
- {
- payloadBuffer.Dispose();
- throw;
- }
-
- return SendAckAndDisposeAsync(payloadBuffer, payloadBufferSize, token);
- }
-
- private async Task SendAckAndDisposeAsync(IMemoryOwner<byte> payloadBuffer, int payloadBufferSize,
- CancellationToken token)
- {
- try
- {
- await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token);
- }
- finally
- {
- payloadBuffer.Dispose();
- }
- }
-
- private static ReadOnlySpan<Message> AsSpan(IList<Message> messages) => messages switch
- {
- Message[] array => array,
- List<Message> list => CollectionsMarshal.AsSpan(list),
- _ => messages.ToArray()
- };
-
/// <inheritdoc />
public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
CancellationToken token = default)
@@ -356,6 +318,8 @@ public sealed class TcpMessageStream : IIggyClient
Consumer consumer,
PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
{
+ ThrowIfAutoCommitWithEncryptor(autoCommit);
+
var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer);
var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize);
var payload = ArrayPool<byte>.Shared.Rent(payloadBufferSize);
@@ -369,7 +333,8 @@ public sealed class TcpMessageStream : IIggyClient
BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE);
responseBuffer = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token);
- return BinaryMapper.MapRentedMessages(responseBuffer.Memory, responseBuffer);
+ return BinaryMapper.MapRentedMessages(responseBuffer.Memory, responseBuffer,
+ _configuration.MessageEncryptor);
}
catch
{
@@ -892,13 +857,81 @@ public sealed class TcpMessageStream : IIggyClient
return new AuthResponse(userId, null);
}
+ // Server-side autoCommit commits the batch offset before the client decrypts, so a decryption failure
+ // would permanently skip the whole batch. IggyConsumer guards this too, but the raw poll is public and
+ // bypasses that path. Opt out via IggyClientConfigurator.AllowAutoCommitWithEncryptor.
+ private void ThrowIfAutoCommitWithEncryptor(bool autoCommit)
+ {
+ if (autoCommit && _configuration.MessageEncryptor is not null && !_configuration.AllowAutoCommitWithEncryptor)
+ {
+ throw new InvalidOperationException(
+ "AutoCommit with a message encryptor risks silent message loss: the offset is committed before decryption. Poll with autoCommit false, or set AllowAutoCommitWithEncryptor.");
+ }
+ }
+
+ private Task SendMessagesCoreAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
+ ReadOnlySpan<Message> messages, CancellationToken token)
+ {
+ var encryptor = _configuration.MessageEncryptor;
+
+ // With an encryptor this is an upper bound; the fill reports the size actually written and only that
+ // prefix is sent.
+ var metadataLength = 2 + streamId.Length + 2 + topicId.Length
+ + 2 + partitioning.Length + 4 + 4;
+ var maxMessageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages, encryptor)
+ + metadataLength;
+ var maxPayloadBufferSize = CalculatePayloadBufferSize(maxMessageBufferSize);
+
+ IMemoryOwner<byte> payloadBuffer = MemoryPool<byte>.Shared.Rent(maxPayloadBufferSize);
+ int payloadBufferSize;
+ try
+ {
+ var messageBufferSize = FillSendMessagesPayload(payloadBuffer.Memory.Span, maxMessageBufferSize,
+ streamId, topicId, partitioning, messages, encryptor);
+ payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize);
+ }
+ catch
+ {
+ payloadBuffer.Dispose();
+ throw;
+ }
+
+ return SendAckAndDisposeAsync(payloadBuffer, payloadBufferSize, token);
+ }
+
+ private async Task SendAckAndDisposeAsync(IMemoryOwner<byte> payloadBuffer, int payloadBufferSize,
+ CancellationToken token)
+ {
+ try
+ {
+ await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token);
+ }
+ finally
+ {
+ payloadBuffer.Dispose();
+ }
+ }
+
+ private static ReadOnlySpan<Message> AsSpan(IList<Message> messages)
+ {
+ return messages switch
+ {
+ Message[] array => array,
+ List<Message> list => CollectionsMarshal.AsSpan(list),
+ _ => messages.ToArray()
+ };
+ }
+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void FillSendMessagesPayload(Span<byte> buffer, int messageBufferSize,
- Identifier streamId, Identifier topicId, Partitioning partitioning, ReadOnlySpan<Message> messages)
+ private static int FillSendMessagesPayload(Span<byte> buffer, int maxMessageBufferSize,
+ Identifier streamId, Identifier topicId, Partitioning partitioning, ReadOnlySpan<Message> messages,
+ IMessageEncryptor? encryptor)
{
- TcpContracts.CreateMessage(buffer.Slice(8, messageBufferSize), streamId, topicId, partitioning, messages);
+ var messageBufferSize = TcpContracts.CreateMessage(buffer.Slice(8, maxMessageBufferSize), streamId, topicId,
+ partitioning, messages, encryptor);
BinaryPrimitives.WriteInt32LittleEndian(buffer[..4], messageBufferSize + 4);
BinaryPrimitives.WriteInt32LittleEndian(buffer[4..8], CommandCodes.SEND_MESSAGES_CODE);
+ return messageBufferSize;
}
private async Task TryEstablishConnectionAsync(CancellationToken token)
@@ -1206,7 +1239,8 @@ public sealed class TcpMessageStream : IIggyClient
}
/// <summary>
- /// Sets the connection state and publishes a ConnectionStateChangedEventArgs to subscribers via the connection event aggregator.
+ /// Sets the connection state and publishes a ConnectionStateChangedEventArgs to subscribers via the connection event
+ /// aggregator.
/// </summary>
/// <param name="newState">The new connection state</param>
private void SetConnectionStateAsync(ConnectionState newState)
@@ -1337,12 +1371,12 @@ public sealed class TcpMessageStream : IIggyClient
internal static class ArrayPoolHelper
{
- public static SlicedMemoryOwner Rent(int minimumLength)
+ public static SlicedMemoryOwner Rent(int minimumLength, bool clearOnReturn = false)
{
- return new SlicedMemoryOwner(minimumLength);
+ return new SlicedMemoryOwner(minimumLength, clearOnReturn);
}
- internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner<byte>
+ internal sealed class SlicedMemoryOwner(int minimumLength, bool clearOnReturn = false) : IMemoryOwner<byte>
{
private readonly byte[] _value = ArrayPool<byte>.Shared.Rent(minimumLength);
private int _disposed;
@@ -1356,7 +1390,7 @@ internal static class ArrayPoolHelper
return;
}
- ArrayPool<byte>.Shared.Return(_value);
+ ArrayPool<byte>.Shared.Return(_value, clearOnReturn);
}
}
}
diff --git a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
index fc82971a4..e4b821d99 100644
--- a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
+++ b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
@@ -27,7 +27,7 @@
<TargetFrameworks>net8.0;net10.0</TargetFrameworks>
<AssemblyName>Apache.Iggy</AssemblyName>
<RootNamespace>Apache.Iggy</RootNamespace>
- <Version>0.8.1-edge.2</Version>
+ <Version>0.8.1-edge.3</Version>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
index f952a6483..e65a4ef90 100644
--- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
+++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
@@ -20,7 +20,9 @@ using System.Buffers.Binary;
using System.Text;
using Apache.Iggy.Contracts;
using Apache.Iggy.Contracts.Auth;
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
using Apache.Iggy.Extensions;
using Apache.Iggy.Headers;
using Apache.Iggy.IggyClient.Implementations;
@@ -334,7 +336,7 @@ internal static class BinaryMapper
}
internal static PolledMessagesRental MapRentedMessages(ReadOnlyMemory<byte> payload,
- IMemoryOwner<byte> payloadOwner)
+ IMemoryOwner<byte> payloadOwner, IMessageEncryptor? encryptor = null)
{
ReadOnlySpan<byte> span = payload.Span;
var length = payload.Length;
@@ -352,72 +354,178 @@ internal static class BinaryMapper
};
}
- var maxMessages = (length - 16) / PropertiesSize;
- var capacity = (int)Math.Min(messagesCount, (uint)maxMessages);
- List<RentedMessageResponse> messages = new(capacity);
-
- while (position < length)
+ ArrayPoolHelper.SlicedMemoryOwner? plaintextOwner = null;
+ try
{
- var checksum = BinaryPrimitives.ReadUInt64LittleEndian(span[position..(position + 8)]);
- var id = BinaryPrimitives.ReadUInt128LittleEndian(span[(position + 8)..(position + 24)]);
- var offset = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 24)..(position + 32)]);
- var timestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 32)..(position + 40)]);
- var originTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 40)..(position + 48)]);
- var headersLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 48)..(position + 52)]);
- var payloadLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 52)..(position + 56)]);
- var reserved = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 56)..(position + 64)]);
-
- var wireHeadersLength = headersLength;
- if (headersLength < 0)
+ // One plaintext buffer rented up front (sized by a header-only pre-pass) keeps the rented path
+ // allocation-free per message. Tied to the rental's disposal so decrypted slices stay valid.
+ var plaintext = Memory<byte>.Empty;
+ var plainCursor = 0;
+ if (encryptor is not null)
{
- throw new ArgumentOutOfRangeException();
+ var total = SumMaxDecryptedLength(span, length, encryptor);
+ if (total > 0)
+ {
+ plaintextOwner = ArrayPoolHelper.Rent(total, true);
+ plaintext = plaintextOwner.Memory;
+ }
}
- var payloadRangeStart = position + 64;
- var payloadRangeEnd = position + 64 + payloadLength;
- var headersRangeStart = payloadRangeEnd;
- var headersRangeEnd = headersRangeStart + headersLength;
- if (payloadRangeStart > length || payloadRangeEnd > length || headersRangeStart > length ||
- headersRangeEnd > length)
+ var maxMessages = (length - 16) / PropertiesSize;
+ var capacity = (int)Math.Min(messagesCount, (uint)maxMessages);
+ List<RentedMessageResponse> messages = new(capacity);
+
+ while (position < length)
{
- break;
+ if (!TryReadFrameLengths(span, length, position, out var headersLength, out var payloadLength))
+ {
+ break;
+ }
+
+ var checksum = BinaryPrimitives.ReadUInt64LittleEndian(span[position..(position + 8)]);
+ var id = BinaryPrimitives.ReadUInt128LittleEndian(span[(position + 8)..(position + 24)]);
+ var offset = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 24)..(position + 32)]);
+ var timestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 32)..(position + 40)]);
+ var originTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 40)..(position + 48)]);
+ var reserved = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 56)..(position + 64)]);
+
+ var payloadRangeStart = position + 64;
+ var headersRangeStart = payloadRangeStart + payloadLength;
+
+ ReadOnlyMemory<byte> payloadSlice = payload.Slice(payloadRangeStart, payloadLength);
+ ReadOnlyMemory<byte> rawHeaders = headersLength > 0
+ ? payload.Slice(headersRangeStart, headersLength)
+ : ReadOnlyMemory<byte>.Empty;
+
+ // Decrypt into the shared buffer so the message looks like plaintext downstream. Wire lengths
+ // still drive the cursor advance; only the decrypted lengths land on the header.
+ var storedPayloadLength = payloadLength;
+ var storedHeadersLength = headersLength;
+ if (encryptor is not null)
+ {
+ try
+ {
+ // Bound each destination to this message's reserved slice so an encryptor that overruns
+ // its contract fails fast here instead of corrupting the next message's region.
+ Memory<byte> payloadDest =
+ plaintext.Slice(plainCursor, encryptor.GetMaxDecryptedLength(payloadLength));
+ var writtenPayload = encryptor.Decrypt(payloadSlice.Span, payloadDest.Span);
+ payloadSlice = payloadDest.Slice(0, writtenPayload);
+ storedPayloadLength = writtenPayload;
+ plainCursor += writtenPayload;
+
+ if (!rawHeaders.IsEmpty)
+ {
+ Memory<byte> headersDest =
+ plaintext.Slice(plainCursor, encryptor.GetMaxDecryptedLength(headersLength));
+ var writtenHeaders = encryptor.Decrypt(rawHeaders.Span, headersDest.Span);
+ rawHeaders = headersDest.Slice(0, writtenHeaders);
+ storedHeadersLength = writtenHeaders;
+ plainCursor += writtenHeaders;
+ }
+ }
+ catch (Exception ex)
+ {
+ throw new MessageDecryptionException(offset, (uint)partitionId, ex);
+ }
+ }
+
+ messages.Add(new RentedMessageResponse
+ {
+ Header = new MessageHeader
+ {
+ Checksum = checksum,
+ Id = id,
+ Offset = offset,
+ OriginTimestamp = originTimestamp,
+ PayloadLength = storedPayloadLength,
+ Timestamp = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(timestamp),
+ UserHeadersLength = storedHeadersLength,
+ Reserved = reserved
+ },
+ RawUserHeaders = rawHeaders,
+ Payload = payloadSlice
+ });
+
+ position += 64 + payloadLength + headersLength;
+ if (position + PropertiesSize >= length)
+ {
+ break;
+ }
}
- ReadOnlyMemory<byte> payloadSlice = payload.Slice(payloadRangeStart, payloadLength);
- ReadOnlyMemory<byte> rawHeaders = headersLength > 0
- ? payload.Slice(headersRangeStart, headersLength)
- : ReadOnlyMemory<byte>.Empty;
+ return new PolledMessagesRental(payloadOwner, plaintextOwner)
+ {
+ PartitionId = partitionId,
+ CurrentOffset = currentOffset,
+ Messages = messages
+ };
+ }
+ catch
+ {
+ plaintextOwner?.Dispose();
+ throw;
+ }
+ }
+
+ // Shared by the decrypt sizing pre-pass and the main map loop so both agree on which frames are included;
+ // drift would mis-size the shared plaintext buffer. Returns false at buffer end or on a frame running past
+ // the buffer; throws on a negative length so a poison frame surfaces instead of being re-polled forever.
+ private static bool TryReadFrameLengths(ReadOnlySpan<byte> span, int length, int position,
+ out int headersLength, out int payloadLength)
+ {
+ headersLength = 0;
+ payloadLength = 0;
+ if (position + PropertiesSize > length)
+ {
+ return false;
+ }
- messages.Add(new RentedMessageResponse
+ headersLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 48)..(position + 52)]);
+ payloadLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 52)..(position + 56)]);
+ if (headersLength < 0 || payloadLength < 0)
+ {
+ throw new MalformedResponseException(
+ $"Malformed message frame at byte {position}: negative payload ({payloadLength}) or header " +
+ $"({headersLength}) length.");
+ }
+
+ // Overflow-safe: server-controlled lengths can approach int.MaxValue, so compute the bound in long.
+ if ((long)position + 64 + payloadLength + headersLength > length)
+ {
+ return false;
+ }
+
+ return true;
+ }
+
+ // Pre-pass summing upper-bound plaintext length so the shared buffer is rented exactly once. Same
+ // TryReadFrameLengths walk as the main loop, so both agree on which messages are included.
+ private static int SumMaxDecryptedLength(ReadOnlySpan<byte> span, int length, IMessageEncryptor encryptor)
+ {
+ var position = 16;
+ var total = 0;
+ while (position < length)
+ {
+ if (!TryReadFrameLengths(span, length, position, out var headersLength, out var payloadLength))
{
- Header = new MessageHeader
- {
- Checksum = checksum,
- Id = id,
- Offset = offset,
- OriginTimestamp = originTimestamp,
- PayloadLength = payloadLength,
- Timestamp = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(timestamp),
- UserHeadersLength = headersLength,
- Reserved = reserved
- },
- RawUserHeaders = rawHeaders,
- Payload = payloadSlice
- });
+ break;
+ }
- position += 64 + payloadLength + wireHeadersLength;
+ total += encryptor.GetMaxDecryptedLength(payloadLength);
+ if (headersLength > 0)
+ {
+ total += encryptor.GetMaxDecryptedLength(headersLength);
+ }
+
+ position += 64 + payloadLength + headersLength;
if (position + PropertiesSize >= length)
{
break;
}
}
- return new PolledMessagesRental(payloadOwner)
- {
- PartitionId = partitionId,
- CurrentOffset = currentOffset,
- Messages = messages
- };
+ return total;
}
internal static PolledMessages MaterializeMessages(PolledMessagesRental rental)
diff --git a/foreign/csharp/Iggy_SDK/Messages/Message.cs b/foreign/csharp/Iggy_SDK/Messages/Message.cs
index 175c79ee6..5a128b460 100644
--- a/foreign/csharp/Iggy_SDK/Messages/Message.cs
+++ b/foreign/csharp/Iggy_SDK/Messages/Message.cs
@@ -26,7 +26,8 @@ namespace Apache.Iggy.Messages;
/// <summary>
/// A message which can be sent to the server. A payload backed by caller-owned (e.g. pooled) memory
/// must stay alive until the send copies it to the wire: for a direct send, until the send call
-/// completes; for a background/queued send, until <see cref="Publishers.IggyPublisher.WaitUntilAllSendsAsync" /> returns.
+/// completes; for a background/queued send, until <see cref="Publishers.IggyPublisher.WaitUntilAllSendsAsync" />
+/// returns.
/// </summary>
[JsonConverter(typeof(MessageConverter))]
public class Message
@@ -41,14 +42,6 @@ public class Message
/// </summary>
public required ReadOnlyMemory<byte> Payload { get; set; }
- /// <summary>
- /// Whether <see cref="Payload" /> (and any user headers) already hold their encrypted form. Set by the
- /// publisher when its encryptor runs; marked messages are skipped, so re-sending one (e.g. a failed-batch
- /// snapshot) cannot double-encrypt it. Set it yourself to send an already-encrypted payload past a
- /// configured encryptor untouched.
- /// </summary>
- public bool Encrypted { get; set; }
-
/// <summary>
/// User defined headers.
/// </summary>
diff --git a/foreign/csharp/Iggy_SDK/Messages/MessageHeader.cs b/foreign/csharp/Iggy_SDK/Messages/MessageHeader.cs
index d858ab3df..6288dd8c6 100644
--- a/foreign/csharp/Iggy_SDK/Messages/MessageHeader.cs
+++ b/foreign/csharp/Iggy_SDK/Messages/MessageHeader.cs
@@ -28,6 +28,8 @@ public readonly record struct MessageHeader
{
/// <summary>
/// Message checksum. Computed by the server on append and populated on poll; ignored on publish.
+ /// With a message encryptor configured, the server computes it over the stored ciphertext, so it
+ /// does not match the decrypted payload the client exposes.
/// </summary>
public ulong Checksum { get; init; }
diff --git a/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs b/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs
index 133ac1615..969118e40 100644
--- a/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs
+++ b/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs
@@ -31,29 +31,29 @@ namespace Apache.Iggy.Publishers;
/// </summary>
internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable
{
- private readonly int _payloadBufferBaselineCapacity;
- private readonly int _payloadBufferCapacityLimit;
private readonly EventAggregator<PublisherErrorEventArgs> _backgroundErrorAggregator;
private readonly CancellationTokenSource _cancellationTokenSource;
- private readonly CancellationToken _disposalToken;
private readonly IIggyClient _client;
private readonly IggyPublisherConfig _config;
+ private readonly CancellationToken _disposalToken;
private readonly ILogger<BackgroundMessageProcessor> _logger;
private readonly EventAggregator<MessageBatchFailedEventArgs> _messageBatchErrorAggregator;
+ private readonly int _payloadBufferBaselineCapacity;
+ private readonly int _payloadBufferCapacityLimit;
private readonly ChannelReader<SendUnit> _reader;
private readonly List<SendUnit> _units;
private readonly List<Message> _wire;
private readonly ChannelWriter<SendUnit> _writer;
- private PooledBufferWriter _payloadBuffer;
- private HashSet<SendUnit>? _failed;
private Task? _backgroundTask;
private volatile bool _canSend = true;
private Exception? _crashException;
private bool _disposed;
private TaskCompletionSource _drainedSignal = CreateCompletedSignal();
+ private HashSet<SendUnit>? _failed;
// Mutated only via Interlocked; OnUnitsCompleted resolves _drainedSignal on the 1->0 transition.
private int _inFlight;
+ private PooledBufferWriter _payloadBuffer;
public BackgroundMessageProcessor(IIggyClient client, IggyPublisherConfig config, ILoggerFactory loggerFactory)
{
@@ -346,8 +346,8 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable
/// Builds the reused wire-ready <see cref="Message" /> list from the units serialized by
/// <see cref="AccumulateBatch" />. Runs as a separate pass because the shared payload buffer can move while
/// it grows during serialization, so the slices are stable only once every unit has serialized. Units that
- /// failed serialization are skipped; a unit whose encryptor throws here is rolled back out and dropped via
- /// <see cref="ReportUnitDropped" />.
+ /// failed to materialize are rolled back out and dropped via <see cref="ReportUnitDropped" />. Encryption is
+ /// applied later by the client at send time, so an encryptor failure surfaces as a batch send failure.
/// </summary>
private List<Message> MaterializeWire()
{
@@ -375,7 +375,7 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable
return _wire;
}
- // Drops only the offending unit so a throwing serializer/encryptor cannot kill the loop and discard the queue.
+ // Drops only the offending unit so a throwing serializer cannot kill the loop and discard the queue.
private void ReportUnitDropped(SendUnit unit, Exception ex)
{
LogFailedToMaterializeUnit(ex);
@@ -383,7 +383,7 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable
if (_backgroundErrorAggregator.HasSubscribers)
{
_backgroundErrorAggregator.Publish(new PublisherErrorEventArgs(ex,
- "Failed to serialize or encrypt a queued message; the unit was dropped", unit.DroppedValues));
+ "Failed to serialize a queued message; the unit was dropped", unit.DroppedValues));
}
}
@@ -482,8 +482,7 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable
LogFailedToSendBatch(ex, wire.Count);
if (_messageBatchErrorAggregator.HasSubscribers)
{
- _messageBatchErrorAggregator.Publish(
- new MessageBatchFailedEventArgs(ex, SnapshotForFailure(wire)));
+ _messageBatchErrorAggregator.Publish(new MessageBatchFailedEventArgs(ex, SnapshotForFailure(wire)));
}
}
@@ -554,7 +553,6 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable
{
Header = source.Header,
Payload = source.Payload.ToArray(),
- Encrypted = source.Encrypted,
UserHeaders = source.UserHeaders,
RawUserHeaders = source.RawUserHeaders.IsEmpty ? default : source.RawUserHeaders.ToArray()
};
@@ -567,7 +565,6 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable
{
Header = source.Header,
Payload = default,
- Encrypted = source.Encrypted,
UserHeaders = source.UserHeaders
};
}
diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs
index 4aa966c33..c90bedfa6 100644
--- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs
+++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs
@@ -115,9 +115,9 @@ public partial class IggyPublisher : IAsyncDisposable
}
/// <summary>
- /// Subscribe to background error events. A queued message whose serializer or encryptor throws at flush
- /// time is dropped and surfaced here, carrying the original values in
- /// <see cref="PublisherErrorEventArgs.DroppedValues" />; transport failures are reported via
+ /// Subscribe to background error events. A queued message whose serializer throws at flush time is dropped
+ /// and surfaced here, carrying the original values in
+ /// <see cref="PublisherErrorEventArgs.DroppedValues" />; transport and encryption failures are reported via
/// <see cref="SubscribeOnMessageBatchFailed" /> instead.
/// </summary>
/// <param name="handler">The handler to invoke on background errors</param>
@@ -136,9 +136,9 @@ public partial class IggyPublisher : IAsyncDisposable
}
/// <summary>
- /// Subscribe to message batch failed events, raised when a batch fails to send over the transport (after
- /// retries, if enabled) with a snapshot of the failed messages for inspection or re-send. Serializer and
- /// encryptor drops are reported via <see cref="SubscribeOnBackgroundError" /> instead.
+ /// Subscribe to message batch failed events, raised when a batch fails to send over the transport or to
+ /// encrypt (after retries, if enabled) with a snapshot of the failed messages for inspection or re-send.
+ /// Serializer drops are reported via <see cref="SubscribeOnBackgroundError" /> instead.
/// </summary>
/// <param name="handler">The handler to invoke on message batch failures</param>
public void SubscribeOnMessageBatchFailed(Func<MessageBatchFailedEventArgs, Task> handler)
@@ -268,10 +268,9 @@ public partial class IggyPublisher : IAsyncDisposable
/// Otherwise, messages are sent immediately.
/// </summary>
/// <remarks>
- /// When an encryptor is configured, each message's payload (and user headers) are encrypted IN PLACE:
- /// the caller's <see cref="Message" /> instances come back with ciphertext payloads and
- /// <see cref="Message.Encrypted" /> set. Already-marked messages are skipped, so a re-send cannot
- /// double-encrypt. Copy the messages first if the plaintext instances must stay usable.
+ /// When the client is configured with an encryptor, payloads (and user headers) are encrypted at the wire
+ /// boundary while the outgoing buffer is serialized: the caller's <see cref="Message" /> instances are never
+ /// mutated and keep their plaintext. A retry re-encrypts from the plaintext with a fresh nonce.
/// <para>
/// With background sending enabled this overload queues the message references (not a payload copy),
/// so a payload backed by caller-owned pooled memory must stay valid until
@@ -296,11 +295,6 @@ public partial class IggyPublisher : IAsyncDisposable
return;
}
- if (Config.MessageEncryptor != null)
- {
- PublisherEncryption.EncryptAll(messages, Config.MessageEncryptor);
- }
-
if (Config.EnableBackgroundSending && BackgroundProcessor != null)
{
LogQueuingMessages(messages.Count);
@@ -341,23 +335,6 @@ public partial class IggyPublisher : IAsyncDisposable
return;
}
- // Encryption replaces each payload with a fresh array, so the rented buffer can be released right after
- // (and must be released even when the encryptor throws mid-batch).
- if (Config.MessageEncryptor != null)
- {
- try
- {
- PublisherEncryption.EncryptAll(messages, Config.MessageEncryptor);
- }
- finally
- {
- batch.Dispose();
- }
-
- await SendReadyAsync(messages, null, ct);
- return;
- }
-
await SendReadyAsync(messages, batch, ct);
}
diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs
index a294fe564..47fa6f09d 100644
--- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs
+++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs
@@ -33,6 +33,8 @@ namespace Apache.Iggy.Publishers;
/// </summary>
public class IggyPublisherBuilder
{
+ private IMessageEncryptor? _encryptor;
+
internal Func<PublisherErrorEventArgs, Task>? OnBackgroundError { get; set; }
internal Func<MessageBatchFailedEventArgs, Task>? OnMessageBatchFailed { get; set; }
@@ -161,7 +163,10 @@ public class IggyPublisherBuilder
/// <param name="topicPartitionsCount">The number of partitions for the topic. Default is 1.</param>
/// <param name="compressionAlgorithm">The compression algorithm to use for messages in the topic. Default is None.</param>
/// <param name="replicationFactor">The replication factor for the topic. Null means server default.</param>
- /// <param name="messageExpiry">The message expiry time. TimeSpan.Zero uses the server default, TimeSpan.MaxValue never expires. Default is TimeSpan.Zero.</param>
+ /// <param name="messageExpiry">
+ /// The message expiry time. TimeSpan.Zero uses the server default, TimeSpan.MaxValue never
+ /// expires. Default is TimeSpan.Zero.
+ /// </param>
/// <param name="maxTopicSize">The maximum size of the topic in bytes (0 for unlimited). Default is 0.</param>
/// <returns>The builder instance for method chaining.</returns>
public IggyPublisherBuilder CreateTopicIfNotExists(string name, uint topicPartitionsCount = 1,
@@ -180,13 +185,18 @@ public class IggyPublisherBuilder
}
/// <summary>
- /// Configures message encryption using the specified encryptor.
+ /// Configures message encryption on the client this builder creates. Encryption is a client-level concern:
+ /// the encryptor encrypts on send and decrypts on poll for the whole connection. Only valid when the
+ /// builder creates its own client; when an external client is supplied, configure the encryptor on that
+ /// client via <see cref="Configuration.IggyClientConfigurator.MessageEncryptor" /> instead.
+ /// The caller owns the encryptor and must dispose it after the publisher is disposed; the builder never
+ /// disposes it (the same instance may be shared across clients).
/// </summary>
- /// <param name="encryptor">The message encryptor to use for encrypting message payloads.</param>
+ /// <param name="encryptor">The message encryptor to use.</param>
/// <returns>The builder instance for method chaining.</returns>
public IggyPublisherBuilder WithEncryptor(IMessageEncryptor encryptor)
{
- Config.MessageEncryptor = encryptor;
+ _encryptor = encryptor;
return this;
}
@@ -242,7 +252,10 @@ public class IggyPublisherBuilder
/// When enabled, messages are queued and sent in batches for improved throughput.
/// </summary>
/// <param name="enabled">Whether background sending is enabled. Default is true.</param>
- /// <param name="queueCapacity">The maximum number of queued send calls; each send occupies one slot regardless of how many messages its batch contains. Default is 10,000.</param>
+ /// <param name="queueCapacity">
+ /// The maximum number of queued send calls; each send occupies one slot regardless of how many
+ /// messages its batch contains. Default is 10,000.
+ /// </param>
/// <param name="batchSize">The number of messages to send in each batch. Default is 100.</param>
/// <param name="flushInterval">The interval at which to flush pending messages. Default is 100ms.</param>
/// <param name="disposalTimeout">
@@ -292,7 +305,8 @@ public class IggyPublisherBuilder
ReceiveBufferSize = Config.ReceiveBufferSize,
SendBufferSize = Config.SendBufferSize,
ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(),
- LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance
+ LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance,
+ MessageEncryptor = _encryptor
});
}
@@ -342,6 +356,13 @@ public class IggyPublisherBuilder
{
throw new InvalidOperationException("IggyClient must be provided when CreateIggyClient is false.");
}
+
+ if (_encryptor != null)
+ {
+ throw new InvalidOperationException(
+ "WithEncryptor is only valid when the builder creates its own client. " +
+ "Configure IggyClientConfigurator.MessageEncryptor on the supplied client instead.");
+ }
}
if (Config.CreateStream && string.IsNullOrWhiteSpace(Config.StreamName))
diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs
index 09c19c9ac..5354069af 100644
--- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs
+++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs
@@ -16,7 +16,6 @@
// under the License.
using Apache.Iggy.Configuration;
-using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Microsoft.Extensions.Logging;
using Partitioning = Apache.Iggy.Kinds.Partitioning;
@@ -130,12 +129,6 @@ public class IggyPublisherConfig
/// </summary>
public string? TopicName { get; set; }
- /// <summary>
- /// Gets or sets the message encryptor for encrypting message payloads.
- /// When set, all message payloads will be encrypted before sending.
- /// </summary>
- public IMessageEncryptor? MessageEncryptor { get; set; } = null;
-
/// <summary>
/// Gets or sets the logger factory for diagnostic logging.
/// When set, enables detailed logging of publisher operations.
diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs
index 7134acdf5..ac8c790c6 100644
--- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs
+++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs
@@ -69,8 +69,7 @@ public class IggyPublisher<T> : IggyPublisher
if (BackgroundProcessor != null)
{
- await BackgroundProcessor.EnqueueAsync(
- TypedUnit<T>.Single(data, id, userHeaders, _serializer, Config.MessageEncryptor), ct);
+ await BackgroundProcessor.EnqueueAsync(TypedUnit<T>.Single(data, id, userHeaders, _serializer), ct);
return;
}
@@ -79,11 +78,6 @@ public class IggyPublisher<T> : IggyPublisher
{
_serializer.Serialize(data, writer);
var message = new Message(id, writer.Written, userHeaders);
- if (Config.MessageEncryptor != null)
- {
- PublisherEncryption.Encrypt(message, Config.MessageEncryptor);
- }
-
await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, message, ct);
}
finally
@@ -103,7 +97,7 @@ public class IggyPublisher<T> : IggyPublisher
if (BackgroundProcessor != null)
{
- TypedUnit<T> unit = TypedUnit<T>.Batch(data, _serializer, Config.MessageEncryptor);
+ TypedUnit<T> unit = TypedUnit<T>.Batch(data, _serializer);
if (unit.Count > 0)
{
await BackgroundProcessor.EnqueueAsync(unit, ct);
@@ -135,7 +129,7 @@ public class IggyPublisher<T> : IggyPublisher
if (BackgroundProcessor != null)
{
- TypedUnit<T> unit = TypedUnit<T>.Batch(items, _serializer, Config.MessageEncryptor);
+ TypedUnit<T> unit = TypedUnit<T>.Batch(items, _serializer);
if (unit.Count > 0)
{
await BackgroundProcessor.EnqueueAsync(unit, ct);
@@ -166,13 +160,6 @@ public class IggyPublisher<T> : IggyPublisher
return;
}
- if (Config.MessageEncryptor != null)
- {
- // Payloads are fresh arrays after encryption, so return the rented buffer now instead of holding plaintext through the send.
- PublisherEncryption.EncryptAll(messages, Config.MessageEncryptor);
- batch.Dispose();
- }
-
await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct);
}
finally
diff --git a/foreign/csharp/Iggy_SDK/Publishers/MessageBatchFailedEventArgs.cs b/foreign/csharp/Iggy_SDK/Publishers/MessageBatchFailedEventArgs.cs
index 46cb96e3a..371601554 100644
--- a/foreign/csharp/Iggy_SDK/Publishers/MessageBatchFailedEventArgs.cs
+++ b/foreign/csharp/Iggy_SDK/Publishers/MessageBatchFailedEventArgs.cs
@@ -30,9 +30,8 @@ public class MessageBatchFailedEventArgs : EventArgs
public Exception Exception { get; }
/// <summary>
- /// Gets the array of messages that failed to send. Payloads are owned copies, safe to inspect or re-send.
- /// With an encryptor configured they are already ciphertext (<see cref="Message.Encrypted" /> is set), and
- /// re-sending through the publisher will not encrypt them again.
+ /// Gets the array of messages that failed to send. Payloads are owned plaintext copies, safe to inspect or
+ /// re-send; with an encryptor configured a re-send encrypts them again at the wire boundary.
/// </summary>
public Message[] FailedMessages { get; }
diff --git a/foreign/csharp/Iggy_SDK/Publishers/PublisherEncryption.cs b/foreign/csharp/Iggy_SDK/Publishers/PublisherEncryption.cs
deleted file mode 100644
index e88c70788..000000000
--- a/foreign/csharp/Iggy_SDK/Publishers/PublisherEncryption.cs
+++ /dev/null
@@ -1,81 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Buffers;
-using Apache.Iggy.Contracts.Tcp;
-using Apache.Iggy.Encryption;
-using Apache.Iggy.Messages;
-
-namespace Apache.Iggy.Publishers;
-
-/// <summary>
-/// Applies an <see cref="IMessageEncryptor" /> to a message in place, replacing the payload (and any user
-/// headers) with ciphertext. The ciphertext is a freshly allocated buffer the message owns, so any pooled
-/// payload memory can be released once encryption has run. Idempotent: a message already marked
-/// <see cref="Message.Encrypted" /> is left untouched, so a re-sent failure snapshot is never double-encrypted.
-/// </summary>
-internal static class PublisherEncryption
-{
- public static void Encrypt(Message message, IMessageEncryptor encryptor)
- {
- if (message.Encrypted)
- {
- return;
- }
-
- // Stage into locals and mutate the message only after every encryptor call succeeded: a throw halfway
- // must not leave a ciphertext payload with Encrypted still false, or a retried send would double-encrypt.
- ReadOnlyMemory<byte> payload = encryptor.Encrypt(message.Payload.Span);
-
- ReadOnlyMemory<byte> rawUserHeaders = default;
- var encryptedHeaders = message.UserHeaders is { Count: > 0 };
- if (encryptedHeaders)
- {
- var length = TcpContracts.HeadersByteLength(message.UserHeaders!);
- var scratch = ArrayPool<byte>.Shared.Rent(length);
- try
- {
- TcpContracts.WriteHeadersTo(scratch.AsSpan(0, length), message.UserHeaders!);
- rawUserHeaders = encryptor.Encrypt(scratch.AsSpan(0, length));
- }
- finally
- {
- ArrayPool<byte>.Shared.Return(scratch, true);
- }
- }
-
- message.Payload = payload;
- message.Header = message.Header with { PayloadLength = payload.Length };
-
- if (encryptedHeaders)
- {
- message.RawUserHeaders = rawUserHeaders;
- message.Header = message.Header with { UserHeadersLength = rawUserHeaders.Length };
- message.UserHeaders = null;
- }
-
- message.Encrypted = true;
- }
-
- public static void EncryptAll(IList<Message> messages, IMessageEncryptor encryptor)
- {
- foreach (var message in messages)
- {
- Encrypt(message, encryptor);
- }
- }
-}
diff --git a/foreign/csharp/Iggy_SDK/Publishers/SendUnit.cs b/foreign/csharp/Iggy_SDK/Publishers/SendUnit.cs
index b3937a983..d965a712d 100644
--- a/foreign/csharp/Iggy_SDK/Publishers/SendUnit.cs
+++ b/foreign/csharp/Iggy_SDK/Publishers/SendUnit.cs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Iggy.Encryption;
using Apache.Iggy.Extensions;
using Apache.Iggy.Headers;
using Apache.Iggy.Messages;
@@ -118,7 +117,8 @@ internal sealed class ReadyUnit : SendUnit
/// <summary>
/// A unit of typed values serialized at flush time into the processor's shared payload buffer. Holds the
-/// serializer and optional encryptor it was created with, so the non-generic processor loop can drive it.
+/// serializer it was created with, so the non-generic processor loop can drive it. Encryption is applied by
+/// the client at send time, not here.
/// </summary>
/// <remarks>
/// Because serialization is deferred, a value is captured by reference when queued: mutating a queued mutable
@@ -126,7 +126,6 @@ internal sealed class ReadyUnit : SendUnit
/// </remarks>
internal sealed class TypedUnit<T> : SendUnit
{
- private readonly IMessageEncryptor? _encryptor;
private readonly Item[] _items;
private readonly ISerializer<T> _serializer;
@@ -146,15 +145,14 @@ internal sealed class TypedUnit<T> : SendUnit
}
}
- private TypedUnit(Item[] items, ISerializer<T> serializer, IMessageEncryptor? encryptor)
+ private TypedUnit(Item[] items, ISerializer<T> serializer)
{
_items = items;
_serializer = serializer;
- _encryptor = encryptor;
}
public static TypedUnit<T> Single(T value, UInt128 id, Dictionary<HeaderKey, HeaderValue>? headers,
- ISerializer<T> serializer, IMessageEncryptor? encryptor)
+ ISerializer<T> serializer)
{
return new TypedUnit<T>([
new Item
@@ -163,29 +161,26 @@ internal sealed class TypedUnit<T> : SendUnit
Id = id,
Headers = headers
}
- ], serializer, encryptor);
+ ], serializer);
}
- public static TypedUnit<T> Batch(IEnumerable<T> values, ISerializer<T> serializer, IMessageEncryptor? encryptor)
+ public static TypedUnit<T> Batch(IEnumerable<T> values, ISerializer<T> serializer)
{
- var items = Materialize(values, static value => new Item
- {
- Value = value
- });
- return new TypedUnit<T>(items, serializer, encryptor);
+ Item[] items = Materialize(values, static value => new Item { Value = value });
+ return new TypedUnit<T>(items, serializer);
}
public static TypedUnit<T> Batch(
IEnumerable<(T data, Guid? messageId, Dictionary<HeaderKey, HeaderValue>? userHeaders)> values,
- ISerializer<T> serializer, IMessageEncryptor? encryptor)
+ ISerializer<T> serializer)
{
- var items = Materialize(values, static value => new Item
+ Item[] items = Materialize(values, static value => new Item
{
Value = value.data,
Id = value.messageId?.ToUInt128() ?? UInt128.Zero,
Headers = value.userHeaders
});
- return new TypedUnit<T>(items, serializer, encryptor);
+ return new TypedUnit<T>(items, serializer);
}
private static Item[] Materialize<TSource>(IEnumerable<TSource> values, Func<TSource, Item> toItem)
@@ -228,11 +223,6 @@ internal sealed class TypedUnit<T> : SendUnit
{
ref var item = ref _items[i];
var message = new Message(item.Id, payloadBuffer.Slice(item.Offset, item.Length), item.Headers);
- if (_encryptor != null)
- {
- PublisherEncryption.Encrypt(message, _encryptor);
- }
-
wire.Add(message);
}
}
diff --git a/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs b/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs
index 6af890198..6f0dc135c 100644
--- a/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs
+++ b/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs
@@ -18,6 +18,7 @@
using System.Buffers.Binary;
using System.Runtime.CompilerServices;
using Apache.Iggy.Contracts.Tcp;
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
using Apache.Iggy.Messages;
@@ -43,19 +44,26 @@ internal static class TcpMessageStreamHelpers
return (status, length);
}
- internal static int CalculateMessageBytesCount(ReadOnlySpan<Message> messages)
+ internal static int CalculateMessageBytesCount(ReadOnlySpan<Message> messages, IMessageEncryptor? encryptor)
{
var bytesCount = 0;
foreach (var message in messages)
{
- bytesCount += 16 + 64 + message.Payload.Length;
- if (!message.RawUserHeaders.IsEmpty)
+ var payloadLength = message.Payload.Length;
+ var headersLength = message.RawUserHeaders.IsEmpty
+ ? TcpContracts.HeadersByteLength(message.UserHeaders)
+ : message.RawUserHeaders.Length;
+
+ if (encryptor is not null)
{
- bytesCount += message.RawUserHeaders.Length;
- continue;
+ payloadLength = encryptor.GetMaxEncryptedLength(payloadLength);
+ if (headersLength > 0)
+ {
+ headersLength = encryptor.GetMaxEncryptedLength(headersLength);
+ }
}
- bytesCount += TcpContracts.HeadersByteLength(message.UserHeaders);
+ bytesCount += 16 + 64 + payloadLength + headersLength;
}
return bytesCount;
diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/RawClientEncryptionTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/RawClientEncryptionTests.cs
new file mode 100644
index 000000000..e6b563b4b
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK_Tests/ClientTests/RawClientEncryptionTests.cs
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Net;
+using System.Text;
+using Apache.Iggy.Configuration;
+using Apache.Iggy.Encryption;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.IggyClient.Implementations;
+using Apache.Iggy.Kinds;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Apache.Iggy.Tests.ClientTests;
+
+public class RawClientEncryptionTests
+{
+ private const string EmptyPollResponse = "{\"partition_id\":0,\"current_offset\":0,\"messages\":[]}";
+ private static readonly Identifier StreamId = Identifier.Numeric(1);
+ private static readonly Identifier TopicId = Identifier.Numeric(1);
+
+ [Fact]
+ public async Task PollMessages_WithEncryptorAndAutoCommit_Throws_ByDefault()
+ {
+ var client = CreateHttpClient(allowAutoCommitWithEncryptor: false);
+
+ var ex = await Assert.ThrowsAsync<InvalidOperationException>(() =>
+ client.PollMessagesAsync(StreamId, TopicId, 0, Consumer.New(1), PollingStrategy.Next(), 10, true,
+ TestContext.Current.CancellationToken));
+
+ Assert.Contains("autoCommit", ex.Message);
+ }
+
+ [Fact]
+ public async Task PollMessages_WithEncryptorAndAutoCommit_IsAllowed_WhenOptedIn()
+ {
+ var client = CreateHttpClient(allowAutoCommitWithEncryptor: true);
+
+ var polled = await client.PollMessagesAsync(StreamId, TopicId, 0, Consumer.New(1), PollingStrategy.Next(), 10,
+ true, TestContext.Current.CancellationToken);
+
+ Assert.Empty(polled.Messages);
+ }
+
+ [Fact]
+ public async Task PollMessages_WithoutAutoCommit_Succeeds_WithEncryptorRegardlessOfOptIn()
+ {
+ var client = CreateHttpClient(allowAutoCommitWithEncryptor: false);
+
+ var polled = await client.PollMessagesAsync(StreamId, TopicId, 0, Consumer.New(1), PollingStrategy.Next(), 10,
+ false, TestContext.Current.CancellationToken);
+
+ Assert.Empty(polled.Messages);
+ }
+
+ [Fact]
+ public async Task TcpPollMessages_WithEncryptorAndAutoCommit_Throws_ByDefault()
+ {
+ var configuration = new IggyClientConfigurator
+ {
+ BaseAddress = "127.0.0.1:8090",
+ Protocol = Protocol.Tcp,
+ MessageEncryptor = new AesMessageEncryptor(AesMessageEncryptor.GenerateKey())
+ };
+ using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance);
+
+ var ex = await Assert.ThrowsAsync<InvalidOperationException>(() =>
+ client.PollMessagesAsync(StreamId, TopicId, 0, Consumer.New(1), PollingStrategy.Next(), 10, true,
+ TestContext.Current.CancellationToken));
+
+ Assert.Contains("autoCommit", ex.Message);
+ }
+
+ [Fact]
+ public async Task PollMessages_WithEncryptor_UndecryptablePayload_ThrowsMessageDecryptionException()
+ {
+ var garbage = new byte[44];
+ Random.Shared.NextBytes(garbage);
+ var pollResponse = "{\"partition_id\":3,\"current_offset\":42,\"messages\":[{" +
+ "\"header\":{\"checksum\":0,\"id\":1,\"offset\":42,\"timestamp\":0," +
+ "\"origin_timestamp\":0,\"user_headers_length\":0,\"payload_length\":44,\"reserved\":0}," +
+ $"\"payload\":\"{Convert.ToBase64String(garbage)}\"}}]}}";
+ var client = CreateHttpClient(allowAutoCommitWithEncryptor: false, pollResponse);
+
+ var ex = await Assert.ThrowsAsync<MessageDecryptionException>(() =>
+ client.PollMessagesAsync(StreamId, TopicId, 0, Consumer.New(1), PollingStrategy.Next(), 10, false,
+ TestContext.Current.CancellationToken));
+
+ Assert.Equal(42ul, ex.Offset);
+ Assert.Equal(3u, ex.PartitionId);
+ }
+
+ private static HttpMessageStream CreateHttpClient(bool allowAutoCommitWithEncryptor,
+ string pollResponse = EmptyPollResponse)
+ {
+ var httpClient = new HttpClient(new StubHandler(pollResponse))
+ {
+ BaseAddress = new Uri("http://localhost")
+ };
+ var encryptor = new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
+ return new HttpMessageStream(httpClient, encryptor, allowAutoCommitWithEncryptor);
+ }
+
+ private sealed class StubHandler(string json) : HttpMessageHandler
+ {
+ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
+ {
+ return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ Content = new StringContent(json, Encoding.UTF8, "application/json")
+ });
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs
new file mode 100644
index 000000000..db0af7c3f
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Consumers;
+using Apache.Iggy.Encryption;
+using Apache.Iggy.Enums;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Moq;
+
+namespace Apache.Iggy.Tests.ConsumerTests;
+
+public class IggyConsumerBuilderTests
+{
+ private static readonly Identifier StreamId = Identifier.Numeric(1);
+ private static readonly Identifier TopicId = Identifier.Numeric(1);
+
+ [Fact]
+ public void Build_WithEncryptorAndAutoCommit_Throws()
+ {
+ var builder = IggyConsumerBuilder
+ .Create(StreamId, TopicId, Consumer.New(1))
+ .WithConnection(Protocol.Tcp, "127.0.0.1:8090", "user", "pass")
+ .WithEncryptor(new AesMessageEncryptor(AesMessageEncryptor.GenerateKey()))
+ .WithAutoCommitMode(AutoCommitMode.Auto);
+
+ var ex = Assert.Throws<InvalidOperationException>(() => builder.Build());
+ Assert.Contains("AutoCommitMode.Auto", ex.Message);
+ }
+
+ [Fact]
+ public void Build_WithExternalClientEncryptorAndAutoCommit_Throws()
+ {
+ var client = new Mock<IIggyClient>();
+ client.SetupGet(c => c.MessageEncryptor)
+ .Returns(new AesMessageEncryptor(AesMessageEncryptor.GenerateKey()));
+
+ var builder = IggyConsumerBuilder
+ .Create(client.Object, StreamId, TopicId, Consumer.New(1))
+ .WithAutoCommitMode(AutoCommitMode.Auto);
+
+ var ex = Assert.Throws<InvalidOperationException>(() => builder.Build());
+ Assert.Contains("AutoCommitMode.Auto", ex.Message);
+ }
+
+ [Fact]
+ public void Build_WithEncryptorAndAfterReceiveCommit_DoesNotThrow()
+ {
+ var consumer = IggyConsumerBuilder
+ .Create(StreamId, TopicId, Consumer.New(1))
+ .WithConnection(Protocol.Tcp, "127.0.0.1:8090", "user", "pass")
+ .WithEncryptor(new AesMessageEncryptor(AesMessageEncryptor.GenerateKey()))
+ .WithAutoCommitMode(AutoCommitMode.AfterReceive)
+ .Build();
+
+ Assert.NotNull(consumer);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
index 5b4c55eb8..7d65beed3 100644
--- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
@@ -130,39 +130,49 @@ public class RentedConsumerTests
}
[Fact]
- public async Task ReceiveRentedAsync_Should_YieldDecryptionFailed_AndStillReleaseBuffer_WhenEncryptorThrows()
+ public async Task ReceiveRentedAsync_Should_SurfaceDecryptionException_FromClient()
{
- var owner = new TrackingMemoryOwner(1024);
- IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 1);
- var rental = new PolledMessagesRental(owner)
+ var client = new Mock<IIggyClient>(MockBehavior.Loose);
+ client.Setup(c => c.ConnectAsync(It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
+ client.Setup(c => c.PollMessagesRentedAsync(It.IsAny<Identifier>(), It.IsAny<Identifier>(),
+ It.IsAny<uint?>(), It.IsAny<Consumer>(), It.IsAny<PollingStrategy>(), It.IsAny<uint>(),
+ It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+ .ThrowsAsync(new MessageDecryptionException(0, 1, new InvalidOperationException("decrypt fail")));
+ var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance);
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ await Assert.ThrowsAsync<MessageDecryptionException>(async () =>
{
- PartitionId = 1,
- CurrentOffset = 0,
- Messages = messages
- };
- Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
- var encryptor = new ThrowingEncryptor();
+ await foreach (var _ in consumer.ReceiveRentedAsync(cts.Token))
+ {
+ }
+ });
+
+ await consumer.DisposeAsync();
+ }
+
+ [Fact]
+ public async Task ReceiveRentedAsync_Should_Throw_WhenAutoCommitCombinedWithEncryptor()
+ {
+ var client = new Mock<IIggyClient>(MockBehavior.Loose);
+ client.Setup(c => c.ConnectAsync(It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
+ client.SetupGet(c => c.MessageEncryptor)
+ .Returns(new AesMessageEncryptor(AesMessageEncryptor.GenerateKey()));
+
var config = BuildConfig();
- config.MessageEncryptor = encryptor;
+ config.AutoCommit = true;
var consumer = new IggyConsumer(client.Object, config, NullLoggerFactory.Instance);
await consumer.InitAsync(TestContext.Current.CancellationToken);
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
- ReceivedRentedMessage? got = null;
- await using (IAsyncEnumerator<ReceivedRentedMessage> e = consumer.ReceiveRentedAsync(cts.Token)
- .GetAsyncEnumerator(TestContext.Current.CancellationToken))
+ var ex = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
{
- Assert.True(await e.MoveNextAsync());
- got = e.Current;
- }
-
- Assert.NotNull(got);
- Assert.Equal(MessageStatus.DecryptionFailed, got!.Status);
- Assert.IsType<InvalidOperationException>(got.Error);
-
- Assert.Equal(0, owner.DisposeCount);
- got.Dispose();
- Assert.Equal(1, owner.DisposeCount);
+ await foreach (var _ in consumer.ReceiveRentedAsync(cts.Token))
+ {
+ }
+ });
+ Assert.Contains("AutoCommit", ex.Message);
await consumer.DisposeAsync();
}
@@ -401,19 +411,6 @@ public class RentedConsumerTests
}
}
- private sealed class ThrowingEncryptor : IMessageEncryptor
- {
- public byte[] Encrypt(ReadOnlySpan<byte> plainData)
- {
- throw new NotSupportedException();
- }
-
- public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
- {
- throw new InvalidOperationException("decrypt fail");
- }
- }
-
private sealed class FailingPublishConsumer : IggyConsumer
{
private int _calls;
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
index 580d905f8..c86fb463d 100644
--- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
@@ -15,11 +15,16 @@
// specific language governing permissions and limitations
// under the License.
+using System.Buffers.Binary;
+using System.Security.Cryptography;
using Apache.Iggy.Contracts;
using Apache.Iggy.Contracts.Auth;
+using Apache.Iggy.Encryption;
using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
using Apache.Iggy.Extensions;
using Apache.Iggy.IggyClient.Implementations;
+using Apache.Iggy.Shared;
using Apache.Iggy.Tests.Utils;
using Apache.Iggy.Tests.Utils.Groups;
using Apache.Iggy.Tests.Utils.Messages;
@@ -342,4 +347,110 @@ public sealed class BinaryMapper
Assert.Equal(stats.OsVersion, stats.OsVersion);
Assert.Equal(stats.KernelVersion, response.KernelVersion);
}
+
+ [Fact]
+ public void MapRentedMessages_WithEncryptor_DecryptsPayloadsAndHeadersIntoPooledBuffer()
+ {
+ var encryptor = new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
+ var payload1 = "first-secret-payload"u8.ToArray();
+ var headers1 = "first-secret-headers"u8.ToArray();
+ var payload2 = "second-secret-payload"u8.ToArray();
+
+ var frame1 = BuildEncryptedFrame(encryptor, 100, payload1, headers1);
+ var frame2 = BuildEncryptedFrame(encryptor, 101, payload2, ReadOnlySpan<byte>.Empty);
+
+ var combined = new byte[16 + frame1.Length + frame2.Length];
+ BinaryPrimitives.WriteInt32LittleEndian(combined.AsSpan(0, 4), 7);
+ BinaryPrimitives.WriteUInt64LittleEndian(combined.AsSpan(4, 8), 101);
+ BinaryPrimitives.WriteUInt32LittleEndian(combined.AsSpan(12, 4), 2);
+ frame1.CopyTo(combined.AsSpan(16));
+ frame2.CopyTo(combined.AsSpan(16 + frame1.Length));
+
+ using var rental = Mappers.BinaryMapper.MapRentedMessages(combined, TcpMessageStream.EmptyMemoryOwner.Instance,
+ encryptor);
+
+ Assert.Equal(7, rental.PartitionId);
+ Assert.Equal(101ul, rental.CurrentOffset);
+ Assert.Equal(2, rental.Messages.Count);
+
+ var first = rental.Messages[0];
+ Assert.Equal(payload1, first.Payload.ToArray());
+ Assert.Equal(headers1, first.RawUserHeaders.ToArray());
+ Assert.Equal(payload1.Length, first.Header.PayloadLength);
+ Assert.Equal(headers1.Length, first.Header.UserHeadersLength);
+
+ var second = rental.Messages[1];
+ Assert.Equal(payload2, second.Payload.ToArray());
+ Assert.True(second.RawUserHeaders.IsEmpty);
+ Assert.Equal(payload2.Length, second.Header.PayloadLength);
+ Assert.Equal(0, second.Header.UserHeadersLength);
+
+ var materialized = Mappers.BinaryMapper.MaterializeMessages(rental);
+ Assert.Equal(payload1, materialized.Messages[0].Payload);
+ Assert.Equal(headers1, materialized.Messages[0].RawUserHeaders);
+ Assert.Equal(payload2, materialized.Messages[1].Payload);
+ }
+
+ [Fact]
+ public void MapRentedMessages_WithEncryptor_NegativePayloadLength_ThrowsInsteadOfSpinning()
+ {
+ var encryptor = new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
+
+ var frame = new byte[64];
+ BinaryPrimitives.WriteInt32LittleEndian(frame.AsSpan(48, 4), 0); // headersLength
+ BinaryPrimitives.WriteInt32LittleEndian(frame.AsSpan(52, 4), -64); // payloadLength
+
+ var combined = new byte[16 + frame.Length];
+ BinaryPrimitives.WriteInt32LittleEndian(combined.AsSpan(0, 4), 7);
+ BinaryPrimitives.WriteUInt64LittleEndian(combined.AsSpan(4, 8), 1);
+ BinaryPrimitives.WriteUInt32LittleEndian(combined.AsSpan(12, 4), 1);
+ frame.CopyTo(combined.AsSpan(16));
+
+ Assert.Throws<MalformedResponseException>(() =>
+ Mappers.BinaryMapper.MapRentedMessages(combined, TcpMessageStream.EmptyMemoryOwner.Instance, encryptor));
+ }
+
+ [Fact]
+ public void MapRentedMessages_WithEncryptor_TamperedCiphertext_ThrowsMessageDecryptionException()
+ {
+ var encryptor = new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
+ var frame = BuildEncryptedFrame(encryptor, 42, "secret-payload"u8, ReadOnlySpan<byte>.Empty);
+
+ frame[64 + 12] ^= 0xFF;
+
+ var combined = new byte[16 + frame.Length];
+ BinaryPrimitives.WriteInt32LittleEndian(combined.AsSpan(0, 4), 7);
+ BinaryPrimitives.WriteUInt64LittleEndian(combined.AsSpan(4, 8), 42);
+ BinaryPrimitives.WriteUInt32LittleEndian(combined.AsSpan(12, 4), 1);
+ frame.CopyTo(combined.AsSpan(16));
+
+ var ex = Assert.Throws<MessageDecryptionException>(() =>
+ Mappers.BinaryMapper.MapRentedMessages(combined, TcpMessageStream.EmptyMemoryOwner.Instance, encryptor));
+
+ Assert.Equal(42ul, ex.Offset);
+ Assert.Equal(7u, ex.PartitionId);
+ Assert.IsAssignableFrom<CryptographicException>(ex.InnerException);
+ }
+
+ private static byte[] BuildEncryptedFrame(AesMessageEncryptor encryptor, ulong offset,
+ ReadOnlySpan<byte> plainPayload, ReadOnlySpan<byte> plainHeaders)
+ {
+ var cipherPayload = encryptor.EncryptToArray(plainPayload);
+ var cipherHeaders = plainHeaders.Length > 0 ? encryptor.EncryptToArray(plainHeaders) : [];
+
+ var frame = new byte[64 + cipherPayload.Length + cipherHeaders.Length];
+ Span<byte> span = frame.AsSpan();
+ BinaryPrimitives.WriteUInt64LittleEndian(span[..8], 0);
+ BinaryPrimitives.WriteUInt128LittleEndian(span[8..24], Guid.NewGuid().ToUInt128());
+ BinaryPrimitives.WriteUInt64LittleEndian(span[24..32], offset);
+ BinaryPrimitives.WriteUInt64LittleEndian(span[32..40], 12345);
+ BinaryPrimitives.WriteUInt64LittleEndian(span[40..48], 12345);
+ BinaryPrimitives.WriteInt32LittleEndian(span[48..52], cipherHeaders.Length);
+ BinaryPrimitives.WriteInt32LittleEndian(span[52..56], cipherPayload.Length);
+ BinaryPrimitives.WriteUInt64LittleEndian(span[56..64], 0);
+ cipherPayload.CopyTo(span[64..]);
+ cipherHeaders.CopyTo(span[(64 + cipherPayload.Length)..]);
+
+ return frame;
+ }
}
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/HeaderEncryptionTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/HeaderEncryptionTests.cs
index 1f5465d66..8d8a62e04 100644
--- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/HeaderEncryptionTests.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/HeaderEncryptionTests.cs
@@ -16,6 +16,8 @@
// under the License.
using Apache.Iggy.Contracts.Tcp;
+using Apache.Iggy.Encryption;
+using Apache.Iggy.Shared;
namespace Apache.Iggy.Tests.MapperTests;
@@ -24,8 +26,8 @@ public class HeaderEncryptionTests
[Fact]
public void Headers_should_survive_encrypt_decrypt_roundtrip()
{
- var key = Encryption.AesMessageEncryptor.GenerateKey();
- var encryptor = new Encryption.AesMessageEncryptor(key);
+ var key = AesMessageEncryptor.GenerateKey();
+ var encryptor = new AesMessageEncryptor(key);
var originalHeaders = new Dictionary<Headers.HeaderKey, Headers.HeaderValue>
{
@@ -37,7 +39,7 @@ public class HeaderEncryptionTests
var headerBytes = HeadersToBytes(originalHeaders);
Assert.NotEmpty(headerBytes);
- var encrypted = encryptor.Encrypt(headerBytes);
+ var encrypted = encryptor.EncryptToArray(headerBytes);
Assert.NotEqual(headerBytes, encrypted);
Assert.True(encrypted.Length > headerBytes.Length);
@@ -45,7 +47,7 @@ public class HeaderEncryptionTests
var parsed = Mappers.BinaryMapper.TryMapHeaders(encrypted);
Assert.Null(parsed);
- var decrypted = encryptor.Decrypt(encrypted);
+ var decrypted = encryptor.DecryptToArray(encrypted);
Assert.Equal(headerBytes, decrypted);
var roundTripped = Mappers.BinaryMapper.MapHeaders(decrypted);
diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs
index f88f89888..61776aec4 100644
--- a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs
@@ -18,7 +18,6 @@
using System.Buffers;
using System.Text;
using Apache.Iggy.Contracts;
-using Apache.Iggy.Encryption;
using Apache.Iggy.Exceptions;
using Apache.Iggy.Extensions;
using Apache.Iggy.Headers;
@@ -147,34 +146,6 @@ public class IggyTypedPublisherTests
partitioning.Kind.Should().Be(Enums.Partitioning.PartitionId);
}
- [Fact]
- public async Task SendSingle_Direct_Should_EncryptPayload_WhenEncryptorConfigured()
- {
- var recorder = new SendRecorder();
- await using IggyPublisher<string> publisher
- = await CreatePublisherAsync(BuildClient(recorder), Config(encryptor: new ReverseEncryptor()));
-
- await publisher.SendAsync("hello", ct: Ct);
-
- var sent = recorder.AllMessages.Should().ContainSingle().Subject;
- sent.Payload.Should().Equal(Reverse(Encoding.UTF8.GetBytes("hello")));
- }
-
- [Fact]
- public async Task SendBatch_Direct_Should_EncryptEveryPayload_WhenEncryptorConfigured()
- {
- var recorder = new SendRecorder();
- await using IggyPublisher<string> publisher
- = await CreatePublisherAsync(BuildClient(recorder), Config(encryptor: new ReverseEncryptor()));
-
- await publisher.SendAsync(new[] { "ab", "cd" }, Ct);
-
- recorder.AllMessages.Select(m => m.Payload)
- .Should().BeEquivalentTo(
- new[] { Reverse(Encoding.UTF8.GetBytes("ab")), Reverse(Encoding.UTF8.GetBytes("cd")) },
- o => o.WithStrictOrdering());
- }
-
[Fact]
public async Task SendSingle_Should_Throw_WhenNotInitialized()
{
@@ -266,20 +237,6 @@ public class IggyTypedPublisherTests
recorder.Calls.Should().BeEmpty();
}
- [Fact]
- public async Task SendSingle_Background_Should_EncryptPayload_WhenEncryptorConfigured()
- {
- var recorder = new SendRecorder();
- await using IggyPublisher<string> publisher
- = await CreatePublisherAsync(BuildClient(recorder), Config(true, new ReverseEncryptor()));
-
- await publisher.SendAsync("hello", ct: Ct);
- await publisher.WaitUntilAllSendsAsync(Ct);
-
- var sent = recorder.AllMessages.Should().ContainSingle().Subject;
- sent.Payload.Should().Equal(Reverse(Encoding.UTF8.GetBytes("hello")));
- }
-
[Fact]
public async Task SendBackground_Should_DrainEveryMessage_WhenManyProducersEnqueueConcurrently()
{
@@ -311,14 +268,7 @@ public class IggyTypedPublisherTests
return Encoding.UTF8.GetString(message.Payload);
}
- private static byte[] Reverse(byte[] bytes)
- {
- var copy = (byte[])bytes.Clone();
- Array.Reverse(copy);
- return copy;
- }
-
- private static IggyPublisherConfig<string> Config(bool background = false, IMessageEncryptor? encryptor = null)
+ private static IggyPublisherConfig<string> Config(bool background = false)
{
return new IggyPublisherConfig<string>
{
@@ -326,7 +276,6 @@ public class IggyTypedPublisherTests
TopicId = Topic,
Partitioning = Partitioning.None(),
Serializer = new StringSerializer(),
- MessageEncryptor = encryptor,
EnableBackgroundSending = background,
BackgroundFlushInterval = TimeSpan.FromMilliseconds(15),
BackgroundBatchSize = 100,
@@ -436,21 +385,4 @@ public class IggyTypedPublisherTests
writer.Write(Encoding.UTF8.GetBytes(data));
}
}
-
- private sealed class ReverseEncryptor : IMessageEncryptor
- {
- public byte[] Encrypt(ReadOnlySpan<byte> plainData)
- {
- var copy = plainData.ToArray();
- Array.Reverse(copy);
- return copy;
- }
-
- public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
- {
- var copy = encryptedData.ToArray();
- Array.Reverse(copy);
- return copy;
- }
- }
}
diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/MessageEncryptionTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/MessageEncryptionTests.cs
new file mode 100644
index 000000000..7e64a2bf3
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/MessageEncryptionTests.cs
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Buffers.Binary;
+using Apache.Iggy.Contracts.Tcp;
+using Apache.Iggy.Encryption;
+using Apache.Iggy.Headers;
+using Apache.Iggy.IggyClient.Implementations;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Shared;
+using Apache.Iggy.Utils;
+
+namespace Apache.Iggy.Tests.PublisherTests;
+
+public class MessageEncryptionTests
+{
+ private static AesMessageEncryptor CreateEncryptor()
+ {
+ return new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
+ }
+
+ private static Dictionary<HeaderKey, HeaderValue> CreateHeaders()
+ {
+ return new Dictionary<HeaderKey, HeaderValue>
+ {
+ {
+ new HeaderKey
+ {
+ Kind = HeaderKind.String,
+ Value = "key"u8.ToArray()
+ },
+ new HeaderValue
+ {
+ Kind = HeaderKind.String,
+ Value = "value"u8.ToArray()
+ }
+ }
+ };
+ }
+
+ [Fact]
+ public void EncryptCopy_LeavesOriginalUntouched_AndRoundtrips()
+ {
+ var encryptor = CreateEncryptor();
+ var message = new Message(Guid.NewGuid(), "payload"u8.ToArray());
+
+ var copy = HttpMessageStream.EncryptCopy(message, encryptor);
+
+ Assert.Equal("payload"u8.ToArray(), message.Payload.ToArray());
+ Assert.Equal("payload"u8.Length, message.Header.PayloadLength);
+
+ Assert.Equal(copy.Payload.Length, copy.Header.PayloadLength);
+ Assert.Equal(encryptor.GetMaxEncryptedLength("payload"u8.Length), copy.Payload.Length);
+ Assert.Equal("payload"u8.ToArray(), encryptor.DecryptToArray(copy.Payload.Span));
+ }
+
+ [Fact]
+ public void EncryptCopy_MovesUserHeadersToEncryptedRawForm()
+ {
+ var encryptor = CreateEncryptor();
+ Dictionary<HeaderKey, HeaderValue> headers = CreateHeaders();
+ var message = new Message(Guid.NewGuid(), "payload"u8.ToArray(), headers);
+
+ var copy = HttpMessageStream.EncryptCopy(message, encryptor);
+
+ Assert.Same(headers, message.UserHeaders);
+ Assert.True(message.RawUserHeaders.IsEmpty);
+
+ Assert.Null(copy.UserHeaders);
+ Assert.False(copy.RawUserHeaders.IsEmpty);
+ Assert.Equal(copy.RawUserHeaders.Length, copy.Header.UserHeadersLength);
+
+ var plainHeaders = encryptor.DecryptToArray(copy.RawUserHeaders.Span);
+ Dictionary<HeaderKey, HeaderValue> roundTripped = BinaryMapper.MapHeaders(plainHeaders);
+ Assert.Single(roundTripped);
+ }
+
+ [Fact]
+ public void Encrypt_ProducesFreshNoncePerCall()
+ {
+ var encryptor = CreateEncryptor();
+
+ var first = encryptor.EncryptToArray("payload"u8);
+ var second = encryptor.EncryptToArray("payload"u8);
+
+ Assert.False(first.AsSpan(0, 12).SequenceEqual(second.AsSpan(0, 12)));
+ Assert.False(first.AsSpan().SequenceEqual(second));
+ Assert.Equal("payload"u8.ToArray(), encryptor.DecryptToArray(first));
+ Assert.Equal("payload"u8.ToArray(), encryptor.DecryptToArray(second));
+ }
+
+ [Fact]
+ public void Dispose_ReleasesCiphers_DoesNotClobberCallerKey_AndIsIdempotent()
+ {
+ var key = AesMessageEncryptor.GenerateKey();
+ var keySnapshot = (byte[])key.Clone();
+
+ var encryptor = new AesMessageEncryptor(key);
+ var ciphertext = encryptor.EncryptToArray("payload"u8);
+
+ encryptor.Dispose();
+ encryptor.Dispose();
+
+ Assert.Equal(keySnapshot, key);
+ Assert.Throws<ObjectDisposedException>(() => encryptor.EncryptToArray("payload"u8));
+
+ using var reopened = new AesMessageEncryptor(key);
+ Assert.Equal("payload"u8.ToArray(), reopened.DecryptToArray(ciphertext));
+ }
+
+ [Fact]
+ public void CreateMessage_WithEncryptor_EncryptsIntoWireBuffer_AndReportsWrittenSize()
+ {
+ var encryptor = CreateEncryptor();
+ var streamId = Identifier.Numeric(1);
+ var topicId = Identifier.Numeric(1);
+ var partitioning = Partitioning.None();
+ var messages = new[]
+ {
+ new Message(Guid.NewGuid(), "first-payload"u8.ToArray()),
+ new Message(Guid.NewGuid(), "second-payload"u8.ToArray(), CreateHeaders())
+ };
+
+ var maxSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages, encryptor)
+ + 2 + streamId.Length + 2 + topicId.Length + 2 + partitioning.Length + 4 + 4;
+ var buffer = new byte[maxSize];
+
+ var written = TcpContracts.CreateMessage(buffer, streamId, topicId, partitioning, messages, encryptor);
+
+ Assert.Equal(maxSize, written);
+
+ Assert.Equal("first-payload"u8.ToArray(), messages[0].Payload.ToArray());
+ Assert.Equal("second-payload"u8.ToArray(), messages[1].Payload.ToArray());
+ Assert.NotNull(messages[1].UserHeaders);
+
+ Span<byte> span = buffer.AsSpan(0, written);
+ var metadataLength = BinaryPrimitives.ReadInt32LittleEndian(span[..4]);
+ var position = 4 + metadataLength + 16 * messages.Length;
+
+ var firstPayloadLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 52)..(position + 56)]);
+ Assert.Equal(encryptor.GetMaxEncryptedLength("first-payload"u8.Length), firstPayloadLength);
+ Assert.Equal("first-payload"u8.ToArray(),
+ encryptor.DecryptToArray(span.Slice(position + 64, firstPayloadLength)));
+
+ position += 64 + firstPayloadLength;
+ var secondHeadersLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 48)..(position + 52)]);
+ var secondPayloadLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 52)..(position + 56)]);
+ Assert.Equal("second-payload"u8.ToArray(),
+ encryptor.DecryptToArray(span.Slice(position + 64, secondPayloadLength)));
+
+ var plainHeaders = encryptor.DecryptToArray(
+ span.Slice(position + 64 + secondPayloadLength, secondHeadersLength));
+ Assert.Single(BinaryMapper.MapHeaders(plainHeaders));
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/PublisherEncryptionTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/PublisherEncryptionTests.cs
deleted file mode 100644
index b22ce5bef..000000000
--- a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/PublisherEncryptionTests.cs
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Encryption;
-using Apache.Iggy.Headers;
-using Apache.Iggy.Messages;
-using Apache.Iggy.Publishers;
-
-namespace Apache.Iggy.Tests.PublisherTests;
-
-public class PublisherEncryptionTests
-{
- private static AesMessageEncryptor CreateEncryptor()
- {
- return new AesMessageEncryptor(AesMessageEncryptor.GenerateKey());
- }
-
- [Fact]
- public void Encrypt_MarksMessageEncrypted_AndSyncsHeaderPayloadLength()
- {
- var encryptor = CreateEncryptor();
- var message = new Message(Guid.NewGuid(), "payload"u8.ToArray());
-
- PublisherEncryption.Encrypt(message, encryptor);
-
- Assert.True(message.Encrypted);
- Assert.Equal(message.Payload.Length, message.Header.PayloadLength);
- Assert.Equal("payload"u8.ToArray(), encryptor.Decrypt(message.Payload.Span));
- }
-
- [Fact]
- public void Encrypt_SkipsAlreadyEncryptedMessage()
- {
- var encryptor = CreateEncryptor();
- var message = new Message(Guid.NewGuid(), "payload"u8.ToArray());
-
- PublisherEncryption.Encrypt(message, encryptor);
- var ciphertext = message.Payload;
-
- PublisherEncryption.Encrypt(message, encryptor);
-
- Assert.True(ciphertext.Span.SequenceEqual(message.Payload.Span));
- Assert.Equal("payload"u8.ToArray(), encryptor.Decrypt(message.Payload.Span));
- }
-
- [Fact]
- public void Encrypt_MovesUserHeadersToEncryptedRawForm()
- {
- var encryptor = CreateEncryptor();
- var headers = new Dictionary<HeaderKey, HeaderValue>
- {
- { new HeaderKey { Kind = HeaderKind.String, Value = "key"u8.ToArray() }, new HeaderValue { Kind = HeaderKind.String, Value = "value"u8.ToArray() } }
- };
- var message = new Message(Guid.NewGuid(), "payload"u8.ToArray(), headers);
-
- PublisherEncryption.Encrypt(message, encryptor);
-
- Assert.Null(message.UserHeaders);
- Assert.False(message.RawUserHeaders.IsEmpty);
- Assert.True(message.Encrypted);
- }
-}
diff --git a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs
index 0475da2df..c7095f312 100644
--- a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs
@@ -17,7 +17,6 @@
using System.Buffers;
using System.Diagnostics;
-using Apache.Iggy.Encryption;
using Apache.Iggy.Extensions;
using Apache.Iggy.IggyClient;
using Apache.Iggy.Messages;
@@ -35,8 +34,7 @@ public class SendUnitTests
{
var messages = new[]
{
- new Message(Guid.NewGuid(), new byte[] { 1 }),
- new Message(Guid.NewGuid(), new byte[] { 2, 3 })
+ new Message(Guid.NewGuid(), new byte[] { 1 }), new Message(Guid.NewGuid(), new byte[] { 2, 3 })
};
var unit = new ReadyUnit(messages, null);
@@ -64,7 +62,7 @@ public class SendUnitTests
[Fact]
public void TypedUnit_ReadyBytes_IsZero_BecauseBytesComeFromPayloadBuffer()
{
- var unit = TypedUnit<byte[]>.Batch([new byte[10]], new PassthroughSerializer(), null);
+ TypedUnit<byte[]> unit = TypedUnit<byte[]>.Batch([new byte[10]], new PassthroughSerializer());
Assert.Equal(0, unit.ReadyBytes);
}
@@ -81,7 +79,7 @@ public class SendUnitTests
public void TypedUnit_ExposesOriginalValuesAsDroppedValues()
{
var payloads = new[] { new byte[] { 1 }, new byte[] { 2, 3 } };
- var unit = TypedUnit<byte[]>.Batch(payloads, new PassthroughSerializer(), null);
+ TypedUnit<byte[]> unit = TypedUnit<byte[]>.Batch(payloads, new PassthroughSerializer());
Assert.Equal(payloads.Cast<object?>().ToArray(), unit.DroppedValues);
}
@@ -111,7 +109,7 @@ public class SendUnitTests
public void TypedUnit_Batch_SerializesIntoPayloadBuffer_AndSlicesInOrder()
{
var payloads = new[] { new byte[] { 1, 2 }, new byte[] { 3, 4, 5 }, new byte[] { 6 } };
- var unit = TypedUnit<byte[]>.Batch(payloads, new PassthroughSerializer(), null);
+ TypedUnit<byte[]> unit = TypedUnit<byte[]>.Batch(payloads, new PassthroughSerializer());
using var payloadBuffer = new PooledBufferWriter(2);
unit.Serialize(payloadBuffer);
@@ -129,7 +127,7 @@ public class SendUnitTests
public void TypedUnit_Single_PreservesIdAndHeaders()
{
var id = Guid.NewGuid().ToUInt128();
- var unit = TypedUnit<byte[]>.Single(new byte[] { 9, 9 }, id, null, new PassthroughSerializer(), null);
+ TypedUnit<byte[]> unit = TypedUnit<byte[]>.Single(new byte[] { 9, 9 }, id, null, new PassthroughSerializer());
Assert.Equal(1, unit.Count);
@@ -142,26 +140,12 @@ public class SendUnitTests
Assert.Equal(new byte[] { 9, 9 }, wire[0].Payload.ToArray());
}
- [Fact]
- public void TypedUnit_WithEncryptor_EncryptsAtFlush()
- {
- var unit = TypedUnit<byte[]>.Single(new byte[] { 1, 2, 3 }, Guid.NewGuid().ToUInt128(), null,
- new PassthroughSerializer(), new XorEncryptor());
-
- using var payloadBuffer = new PooledBufferWriter();
- unit.Serialize(payloadBuffer);
- var wire = new List<Message>();
- unit.AppendWire(wire, payloadBuffer.Written);
-
- Assert.Equal(new byte[] { 1 ^ 0xFF, 2 ^ 0xFF, 3 ^ 0xFF }, wire[0].Payload.ToArray());
- }
-
[Fact]
public async Task Processor_SendsUnitWhole_AndDisposesOwnerOnce()
{
var sentCounts = new List<int>();
- var client = MockClient(sentCounts);
- var config = BackgroundConfig(batchSize: 10);
+ Mock<IIggyClient> client = MockClient(sentCounts);
+ var config = BackgroundConfig(10);
await using var processor = new BackgroundMessageProcessor(client.Object, config, NullLoggerFactory.Instance);
processor.Start();
@@ -185,9 +169,9 @@ public class SendUnitTests
public async Task Processor_FlushesOnByteLimit_BeforeCountLimit()
{
var sentCounts = new List<int>();
- var client = MockClient(sentCounts);
+ Mock<IIggyClient> client = MockClient(sentCounts);
// 100-byte messages, 250-byte cap => at most 3 messages per flush; the count limit never trips.
- var config = BackgroundConfig(batchSize: 1000);
+ var config = BackgroundConfig(1000);
config.BackgroundMaxBatchBytes = 250;
await using var processor = new BackgroundMessageProcessor(client.Object, config, NullLoggerFactory.Instance);
@@ -195,8 +179,8 @@ public class SendUnitTests
// Enqueue before Start so the first flush drains the full backlog in cap-sized chunks deterministically.
for (var i = 0; i < 10; i++)
{
- await processor.EnqueueAsync(
- new ReadyUnit([new Message(Guid.NewGuid(), new byte[100])], null), CancellationToken.None);
+ await processor.EnqueueAsync(new ReadyUnit([new Message(Guid.NewGuid(), new byte[100])], null),
+ CancellationToken.None);
}
processor.Start();
@@ -209,14 +193,14 @@ public class SendUnitTests
[Fact]
public async Task Processor_Dispose_DisposesQueuedOwners()
{
- var client = MockClient(new List<int>());
- var config = BackgroundConfig(batchSize: 10);
+ Mock<IIggyClient> client = MockClient(new List<int>());
+ var config = BackgroundConfig(10);
var processor = new BackgroundMessageProcessor(client.Object, config, NullLoggerFactory.Instance);
var owner = new TrackingDisposable();
- await processor.EnqueueAsync(
- new ReadyUnit([new Message(Guid.NewGuid(), new byte[] { 1 })], owner), CancellationToken.None);
+ await processor.EnqueueAsync(new ReadyUnit([new Message(Guid.NewGuid(), new byte[] { 1 })], owner),
+ CancellationToken.None);
await processor.DisposeAsync();
@@ -232,13 +216,13 @@ public class SendUnitTests
client.Setup(c => c.SendMessagesAsync(It.IsAny<Identifier>(), It.IsAny<Identifier>(),
It.IsAny<Partitioning>(), It.IsAny<IList<Message>>(), It.IsAny<CancellationToken>()))
.Returns(gate.Task);
- var config = BackgroundConfig(batchSize: 10);
+ var config = BackgroundConfig(10);
await using var processor = new BackgroundMessageProcessor(client.Object, config, NullLoggerFactory.Instance);
processor.Start();
- await processor.EnqueueAsync(
- new ReadyUnit([new Message(Guid.NewGuid(), new byte[] { 1 })], null), CancellationToken.None);
+ await processor.EnqueueAsync(new ReadyUnit([new Message(Guid.NewGuid(), new byte[] { 1 })], null),
+ CancellationToken.None);
var drain = processor.WaitForDrainAsync(CancellationToken.None);
@@ -260,16 +244,19 @@ public class SendUnitTests
return client;
}
- private static IggyPublisherConfig BackgroundConfig(int batchSize) => new()
+ private static IggyPublisherConfig BackgroundConfig(int batchSize)
{
- StreamId = Identifier.Numeric(1),
- TopicId = Identifier.Numeric(1),
- BackgroundBatchSize = batchSize,
- BackgroundQueueCapacity = 1000,
- BackgroundFlushInterval = TimeSpan.FromMilliseconds(10),
- BackgroundDisposalTimeout = TimeSpan.FromSeconds(5),
- EnableRetry = false
- };
+ return new IggyPublisherConfig
+ {
+ StreamId = Identifier.Numeric(1),
+ TopicId = Identifier.Numeric(1),
+ BackgroundBatchSize = batchSize,
+ BackgroundQueueCapacity = 1000,
+ BackgroundFlushInterval = TimeSpan.FromMilliseconds(10),
+ BackgroundDisposalTimeout = TimeSpan.FromSeconds(5),
+ EnableRetry = false
+ };
+ }
private static async Task WaitUntil(Func<bool> condition)
{
@@ -289,7 +276,10 @@ public class SendUnitTests
{
public int DisposeCount { get; private set; }
- public void Dispose() => DisposeCount++;
+ public void Dispose()
+ {
+ DisposeCount++;
+ }
}
private sealed class PassthroughSerializer : ISerializer<byte[]>
@@ -300,20 +290,4 @@ public class SendUnitTests
writer.Advance(data.Length);
}
}
-
- private sealed class XorEncryptor : IMessageEncryptor
- {
- public byte[] Encrypt(ReadOnlySpan<byte> plainData)
- {
- var result = plainData.ToArray();
- for (var i = 0; i < result.Length; i++)
- {
- result[i] ^= 0xFF;
- }
-
- return result;
- }
-
- public byte[] Decrypt(ReadOnlySpan<byte> encryptedData) => Encrypt(encryptedData);
- }
}
diff --git a/foreign/csharp/Shared/EncryptorTestExtensions.cs b/foreign/csharp/Shared/EncryptorTestExtensions.cs
new file mode 100644
index 000000000..dca15be0d
--- /dev/null
+++ b/foreign/csharp/Shared/EncryptorTestExtensions.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Encryption;
+
+namespace Apache.Iggy.Shared;
+
+/// <summary>
+/// Test-support allocating wrappers over the span-based <see cref="IMessageEncryptor" /> contract.
+/// </summary>
+public static class EncryptorTestExtensions
+{
+ public static byte[] EncryptToArray(this IMessageEncryptor encryptor, ReadOnlySpan<byte> plaintext)
+ {
+ var ciphertext = new byte[encryptor.GetMaxEncryptedLength(plaintext.Length)];
+ var written = encryptor.Encrypt(plaintext, ciphertext);
+ return written == ciphertext.Length ? ciphertext : ciphertext.AsSpan(0, written).ToArray();
+ }
+
+ public static byte[] DecryptToArray(this IMessageEncryptor encryptor, ReadOnlySpan<byte> encryptedData)
+ {
+ var plaintext = new byte[encryptor.GetMaxDecryptedLength(encryptedData.Length)];
+ var written = encryptor.Decrypt(encryptedData, plaintext);
+ return written == plaintext.Length ? plaintext : plaintext.AsSpan(0, written).ToArray();
+ }
+}
diff --git a/foreign/csharp/Shared/Shared.csproj b/foreign/csharp/Shared/Shared.csproj
index 781a6b54e..0cf43f449 100644
--- a/foreign/csharp/Shared/Shared.csproj
+++ b/foreign/csharp/Shared/Shared.csproj
@@ -26,10 +26,12 @@
<Nullable>enable</Nullable>
<AssemblyName>Apache.Iggy.Shared</AssemblyName>
<RootNamespace>Apache.Iggy.Shared</RootNamespace>
+ <!-- Test-support and example shared types only; never published to NuGet. -->
+ <IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
- <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj" />
+ <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj"/>
</ItemGroup>
</Project>