You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by lz...@apache.org on 2026/05/12 21:24:46 UTC
(iggy) 01/02: feat(csharp): implement rented message polling and deserialization in Iggy consumer
This is an automated email from the ASF dual-hosted git repository.
lukaszzborek pushed a commit to branch feat/dotnet-memory-usage
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit 0e20bc488a965363265c6468c329f15eef9bed8e
Author: Łukasz Zborek <lu...@gmail.com>
AuthorDate: Tue May 12 23:08:24 2026 +0200
feat(csharp): implement rented message polling and deserialization in Iggy consumer
---
foreign/csharp/Directory.Packages.props | 53 +-
.../FetchMessagesTests.cs | 398 +--
.../IggyTypedConsumerTests.cs | 272 +++
.../Iggy_SDK.Tests.Integration.csproj | 89 +-
.../Iggy_SDK/Consumers/DeserializedMessage.cs | 65 +
foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs | 104 +-
.../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 250 ++
foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 1081 +++++----
.../csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs | 300 ++-
.../Iggy_SDK/Consumers/ReceivedRentedMessage.cs | 126 +
.../csharp/Iggy_SDK/Contracts/MessageResponse.cs | 148 +-
.../Iggy_SDK/Contracts/PolledMessagesRental.cs | 71 +
.../Iggy_SDK/Contracts/RentedMessageResponse.cs | 71 +
.../Iggy_SDK/Encryption/AesMessageEncryptor.cs | 257 +-
.../Iggy_SDK/Encryption/IMessageEncryptor.cs | 80 +-
.../Iggy_SDK/Extensions/IggyClientExtension.cs | 148 +-
.../csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs | 148 +-
.../Implementations/HttpMessageStream.cs | 1712 ++++++-------
.../IggyClient/Implementations/TcpMessageStream.cs | 2556 ++++++++++----------
foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 2 +-
.../JsonConverters/MessageResponseConverter.cs | 204 +-
foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 244 +-
.../ConsumerTests/RentedConsumerTests.cs | 483 ++++
.../ConsumerTests/RentedTypedConsumerTests.cs | 183 ++
.../Iggy_SDK_Tests/MapperTests/BinaryMapper.cs | 4 +-
.../MapperTests/MessageResponseConverterTests.cs | 1 +
26 files changed, 5470 insertions(+), 3580 deletions(-)
diff --git a/foreign/csharp/Directory.Packages.props b/foreign/csharp/Directory.Packages.props
index b2dee5c2a..cd888e5b8 100644
--- a/foreign/csharp/Directory.Packages.props
+++ b/foreign/csharp/Directory.Packages.props
@@ -1,27 +1,26 @@
-<Project>
- <PropertyGroup>
- <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
- <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
- </PropertyGroup>
- <ItemGroup>
- <PackageVersion Include="FluentAssertions" Version="6.12.0" />
- <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
- <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
- <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
- <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.102" />
- <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.3.2" />
- <PackageVersion Include="Microsoft.Testing.Extensions.Retry" Version="2.2.1" />
- <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.0.2" />
- <PackageVersion Include="Moq" Version="4.20.72" />
- <PackageVersion Include="Reqnroll.xunit.v3" Version="3.3.3" />
- <PackageVersion Include="Shouldly" Version="4.3.0" />
- <PackageVersion Include="System.IO.Hashing" Version="8.0.0" />
- <PackageVersion Include="Testcontainers" Version="4.10.0" />
- <PackageVersion Include="TUnit" Version="1.12.93" />
- <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5">
- <PrivateAssets>all</PrivateAssets>
- <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
- </PackageVersion>
- <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
- </ItemGroup>
-</Project>
+<Project>
+ <PropertyGroup>
+ <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+ <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
+ </PropertyGroup>
+ <ItemGroup>
+ <PackageVersion Include="FluentAssertions" Version="6.12.0" />
+ <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
+ <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
+ <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+ <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.102" />
+ <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.3.2" />
+ <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.0.2" />
+ <PackageVersion Include="Moq" Version="4.20.72" />
+ <PackageVersion Include="Reqnroll.xunit.v3" Version="3.3.3" />
+ <PackageVersion Include="Shouldly" Version="4.3.0" />
+ <PackageVersion Include="System.IO.Hashing" Version="8.0.0" />
+ <PackageVersion Include="Testcontainers" Version="4.10.0" />
+ <PackageVersion Include="TUnit" Version="1.12.93" />
+ <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5">
+ <PrivateAssets>all</PrivateAssets>
+ <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+ </PackageVersion>
+ <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
+ </ItemGroup>
+</Project>
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs
index 4a32eff1f..91768ada4 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs
@@ -1,184 +1,214 @@
-// // Licensed to the Apache Software Foundation (ASF) under one
-// // or more contributor license agreements. See the NOTICE file
-// // distributed with this work for additional information
-// // regarding copyright ownership. The ASF licenses this file
-// // to you under the Apache License, Version 2.0 (the
-// // "License"); you may not use this file except in compliance
-// // with the License. You may obtain a copy of the License at
-// //
-// // http://www.apache.org/licenses/LICENSE-2.0
-// //
-// // Unless required by applicable law or agreed to in writing,
-// // software distributed under the License is distributed on an
-// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// // KIND, either express or implied. See the License for the
-// // specific language governing permissions and limitations
-// // under the License.
-
-using System.Text;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Headers;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Messages;
-using Apache.Iggy.Tests.Integrations.Fixtures;
-using Shouldly;
-using Partitioning = Apache.Iggy.Kinds.Partitioning;
-
-namespace Apache.Iggy.Tests.Integrations;
-
-public class FetchMessagesTests
-{
- private const int MessageCount = 20;
- private const string TopicName = "topic";
- private const string HeadersTopicName = "headers-topic";
-
- [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
- public required IggyServerFixture Fixture { get; init; }
-
- private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol)
- {
- var client = await Fixture.CreateAuthenticatedClient(protocol);
-
- var streamName = $"fetch-msg-{Guid.NewGuid():N}";
-
- await client.CreateStreamAsync(streamName);
- await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1);
- await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1);
-
- await client.SendMessagesAsync(Identifier.String(streamName),
- Identifier.String(TopicName), Partitioning.None(),
- CreateMessagesWithoutHeader(MessageCount));
-
- await client.SendMessagesAsync(Identifier.String(streamName),
- Identifier.String(HeadersTopicName), Partitioning.None(),
- CreateMessagesWithHeader(MessageCount));
-
- return (client, streamName);
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol)
- {
- var (client, streamName) = await CreateStreamWithMessages(protocol);
-
- var response = await client.PollMessagesAsync(new MessageFetchRequest
- {
- Count = 10,
- AutoCommit = true,
- Consumer = Consumer.New(1),
- PartitionId = 0,
- PollingStrategy = PollingStrategy.Next(),
- StreamId = Identifier.String(streamName),
- TopicId = Identifier.String(TopicName)
- });
-
- response.Messages.Count.ShouldBe(10);
- response.PartitionId.ShouldBe(0);
- response.CurrentOffset.ShouldBe(19u);
-
- foreach (var responseMessage in response.Messages)
- {
- responseMessage.UserHeaders.ShouldBeNull();
- responseMessage.Payload.ShouldNotBeNull();
- responseMessage.Payload.Length.ShouldBeGreaterThan(0);
- }
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol)
- {
- var (client, streamName) = await CreateStreamWithMessages(protocol);
-
- var invalidFetchRequest = new MessageFetchRequest
- {
- Count = 10,
- AutoCommit = true,
- Consumer = Consumer.New(1),
- PartitionId = 0,
- PollingStrategy = PollingStrategy.Next(),
- StreamId = Identifier.String(streamName),
- TopicId = Identifier.Numeric(2137)
- };
-
- await Should.ThrowAsync<IggyInvalidStatusCodeException>(() =>
- client.PollMessagesAsync(invalidFetchRequest));
- }
-
- [Test]
- [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
- public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol)
- {
- var (client, streamName) = await CreateStreamWithMessages(protocol);
-
- var headersMessageFetchRequest = new MessageFetchRequest
- {
- Count = 10,
- AutoCommit = true,
- Consumer = Consumer.New(1),
- PartitionId = 0,
- PollingStrategy = PollingStrategy.Next(),
- StreamId = Identifier.String(streamName),
- TopicId = Identifier.String(HeadersTopicName)
- };
-
- var response = await client.PollMessagesAsync(headersMessageFetchRequest);
- response.Messages.Count.ShouldBe(10);
- response.PartitionId.ShouldBe(0);
- response.CurrentOffset.ShouldBe(19u);
- foreach (var responseMessage in response.Messages)
- {
- responseMessage.UserHeaders.ShouldNotBeNull();
- responseMessage.UserHeaders.Count.ShouldBe(2);
- responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1");
- responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0);
- }
- }
-
- private static Message[] CreateMessagesWithoutHeader(int count)
- {
- var messages = new List<Message>();
- for (var i = 0; i < count; i++)
- {
- var dummyJson = $$"""
- {
- "userId": {{i + 1}},
- "id": {{i + 1}},
- "title": "delete",
- "completed": false
- }
- """;
- messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson)));
- }
-
- return messages.ToArray();
- }
-
- private static Message[] CreateMessagesWithHeader(int count)
- {
- var messages = new List<Message>();
- for (var i = 0; i < count; i++)
- {
- var dummyJson = $$"""
- {
- "userId": {{i + 1}},
- "id": {{i + 1}},
- "title": "delete",
- "completed": false
- }
- """;
- messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson),
- new Dictionary<HeaderKey, HeaderValue>
- {
- { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") },
- { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) }
- }));
- }
-
- return messages.ToArray();
- }
-}
+// // Licensed to the Apache Software Foundation (ASF) under one
+// // or more contributor license agreements. See the NOTICE file
+// // distributed with this work for additional information
+// // regarding copyright ownership. The ASF licenses this file
+// // to you under the Apache License, Version 2.0 (the
+// // "License"); you may not use this file except in compliance
+// // with the License. You may obtain a copy of the License at
+// //
+// // http://www.apache.org/licenses/LICENSE-2.0
+// //
+// // Unless required by applicable law or agreed to in writing,
+// // software distributed under the License is distributed on an
+// // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// // KIND, either express or implied. See the License for the
+// // specific language governing permissions and limitations
+// // under the License.
+
+using System.Text;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Headers;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Tests.Integrations.Fixtures;
+using Shouldly;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.Tests.Integrations;
+
+public class FetchMessagesTests
+{
+ private const int MessageCount = 20;
+ private const string TopicName = "topic";
+ private const string HeadersTopicName = "headers-topic";
+
+ [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
+ public required IggyServerFixture Fixture { get; init; }
+
+ private async Task<(IIggyClient client, string streamName)> CreateStreamWithMessages(Protocol protocol)
+ {
+ var client = await Fixture.CreateAuthenticatedClient(protocol);
+
+ var streamName = $"fetch-msg-{Guid.NewGuid():N}";
+
+ await client.CreateStreamAsync(streamName);
+ await client.CreateTopicAsync(Identifier.String(streamName), TopicName, 1);
+ await client.CreateTopicAsync(Identifier.String(streamName), HeadersTopicName, 1);
+
+ await client.SendMessagesAsync(Identifier.String(streamName),
+ Identifier.String(TopicName), Partitioning.None(),
+ CreateMessagesWithoutHeader(MessageCount));
+
+ await client.SendMessagesAsync(Identifier.String(streamName),
+ Identifier.String(HeadersTopicName), Partitioning.None(),
+ CreateMessagesWithHeader(MessageCount));
+
+ return (client, streamName);
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var response = await client.PollMessagesAsync(new MessageFetchRequest
+ {
+ Count = 10,
+ AutoCommit = true,
+ Consumer = Consumer.New(1),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.Next(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.String(TopicName)
+ });
+
+ response.Messages.Count.ShouldBe(10);
+ response.PartitionId.ShouldBe(0);
+ response.CurrentOffset.ShouldBe(19u);
+
+ foreach (var responseMessage in response.Messages)
+ {
+ responseMessage.UserHeaders.ShouldBeNull();
+ responseMessage.Payload.ShouldNotBeNull();
+ responseMessage.Payload.Length.ShouldBeGreaterThan(0);
+ }
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var invalidFetchRequest = new MessageFetchRequest
+ {
+ Count = 10,
+ AutoCommit = true,
+ Consumer = Consumer.New(1),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.Next(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.Numeric(2137)
+ };
+
+ await Should.ThrowAsync<IggyInvalidStatusCodeException>(() =>
+ client.PollMessagesAsync(invalidFetchRequest));
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var headersMessageFetchRequest = new MessageFetchRequest
+ {
+ Count = 10,
+ AutoCommit = true,
+ Consumer = Consumer.New(1),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.Next(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.String(HeadersTopicName)
+ };
+
+ var response = await client.PollMessagesAsync(headersMessageFetchRequest);
+ response.Messages.Count.ShouldBe(10);
+ response.PartitionId.ShouldBe(0);
+ response.CurrentOffset.ShouldBe(19u);
+ foreach (var responseMessage in response.Messages)
+ {
+ responseMessage.UserHeaders.ShouldNotBeNull();
+ responseMessage.UserHeaders.Count.ShouldBe(2);
+ responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1");
+ responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0);
+ }
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task PollMessages_WithHeaders_Should_AutoParseUserHeadersFromRaw(Protocol protocol)
+ {
+ var (client, streamName) = await CreateStreamWithMessages(protocol);
+
+ var response = await client.PollMessagesAsync(new MessageFetchRequest
+ {
+ Count = 1,
+ AutoCommit = true,
+ Consumer = Consumer.New(2),
+ PartitionId = 0,
+ PollingStrategy = PollingStrategy.First(),
+ StreamId = Identifier.String(streamName),
+ TopicId = Identifier.String(HeadersTopicName)
+ });
+
+ response.Messages.Count.ShouldBe(1);
+ var msg = response.Messages[0];
+
+ Dictionary<HeaderKey, HeaderValue>? parsed = msg.UserHeaders;
+ parsed.ShouldNotBeNull();
+ parsed!.Count.ShouldBe(2);
+ parsed[HeaderKey.FromString("header1")].ToString().ShouldBe("value1");
+ parsed[HeaderKey.FromString("header2")].ToInt32().ShouldBe(14);
+
+ Dictionary<HeaderKey, HeaderValue>? second = msg.UserHeaders;
+ second.ShouldBeSameAs(parsed);
+ }
+
+ private static Message[] CreateMessagesWithoutHeader(int count)
+ {
+ var messages = new List<Message>();
+ for (var i = 0; i < count; i++)
+ {
+ var dummyJson = $$"""
+ {
+ "userId": {{i + 1}},
+ "id": {{i + 1}},
+ "title": "delete",
+ "completed": false
+ }
+ """;
+ messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson)));
+ }
+
+ return messages.ToArray();
+ }
+
+ private static Message[] CreateMessagesWithHeader(int count)
+ {
+ var messages = new List<Message>();
+ for (var i = 0; i < count; i++)
+ {
+ var dummyJson = $$"""
+ {
+ "userId": {{i + 1}},
+ "id": {{i + 1}},
+ "title": "delete",
+ "completed": false
+ }
+ """;
+ messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson),
+ new Dictionary<HeaderKey, HeaderValue>
+ {
+ { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") },
+ { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) }
+ }));
+ }
+
+ return messages.ToArray();
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
new file mode 100644
index 000000000..8dd3160ff
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs
@@ -0,0 +1,272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text;
+using Apache.Iggy.Consumers;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Tests.Integrations.Fixtures;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.Tests.Integrations;
+
+public class IggyTypedConsumerTests
+{
+ [ClassDataSource<IggyServerFixture>(Shared = SharedType.PerAssembly)]
+ public required IggyServerFixture Fixture { get; init; }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 200 : 300;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new Utf8StringDeserializer());
+
+ await consumer.InitAsync();
+
+ var received = new List<DeserializedMessage<string>>();
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+
+ await foreach (DeserializedMessage<string> msg in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ msg.ShouldNotBeNull();
+ msg.Status.ShouldBe(MessageStatus.Success);
+ msg.Data.ShouldNotBeNull();
+ msg.Data.ShouldStartWith("Test message");
+ msg.PartitionId.ShouldBe(1u);
+ msg.Error.ShouldBeNull();
+ received.Add(msg);
+ if (received.Count >= 10)
+ {
+ break;
+ }
+ }
+
+ received.Count.ShouldBeGreaterThanOrEqualTo(10);
+ await consumer.DisposeAsync();
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException(
+ Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 201 : 301;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new Utf8StringDeserializer());
+
+ await Should.ThrowAsync<ConsumerNotInitializedException>(async () =>
+ {
+ await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync())
+ {
+ }
+ });
+
+ await consumer.DisposeAsync();
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 202 : 302;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.AfterReceive, new Utf8StringDeserializer(),
+ PollingStrategy.First());
+
+ await consumer.InitAsync();
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+ var count = 0;
+
+ await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ count++;
+ if (count >= 5)
+ {
+ break;
+ }
+ }
+
+ await consumer.DisposeAsync();
+
+ var offset = await client.GetOffsetAsync(Consumer.New(consumerId),
+ Identifier.String(testStream.StreamId),
+ Identifier.String(testStream.TopicId),
+ 1u);
+
+ offset.ShouldNotBeNull();
+ offset.StoredOffset.ShouldBe(3ul);
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed(
+ Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 203 : 303;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new FailingDeserializer());
+
+ await consumer.InitAsync();
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+
+ await foreach (DeserializedMessage<string> msg in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ msg.Status.ShouldBe(MessageStatus.DeserializationFailed);
+ msg.Data.ShouldBeNull();
+ msg.Error.ShouldNotBeNull();
+ msg.Error.ShouldBeOfType<InvalidOperationException>();
+ break;
+ }
+
+ await consumer.DisposeAsync();
+ }
+
+ [Test]
+ [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))]
+ public async Task ReceiveRentedDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol)
+ {
+ var client = protocol == Protocol.Tcp
+ ? await Fixture.CreateTcpClient()
+ : await Fixture.CreateHttpClient();
+
+ var testStream = await CreateTestStreamWithMessages(client, protocol);
+
+ var consumerId = protocol == Protocol.Tcp ? 204 : 304;
+ IggyConsumer<string> consumer = BuildTypedConsumer(client, testStream, Consumer.New(consumerId),
+ AutoCommitMode.Disabled, new Utf8StringDeserializer());
+
+ await consumer.InitAsync();
+
+ using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
+
+ await Should.NotThrowAsync(async () =>
+ {
+ try
+ {
+ await foreach (DeserializedMessage<string> _ in consumer.ReceiveRentedDeserializedAsync(cts.Token))
+ {
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ }
+ });
+
+ await consumer.DisposeAsync();
+ }
+
+ private static IggyConsumer<string> BuildTypedConsumer(IIggyClient client,
+ TestStreamInfo stream,
+ Consumer consumer,
+ AutoCommitMode autoCommitMode,
+ IDeserializer<string> deserializer,
+ PollingStrategy? pollingStrategy = null)
+ {
+ var config = new IggyConsumerConfig<string>
+ {
+ StreamId = Identifier.String(stream.StreamId),
+ TopicId = Identifier.String(stream.TopicId),
+ Consumer = consumer,
+ PollingStrategy = pollingStrategy ?? PollingStrategy.Next(),
+ BatchSize = 10,
+ PartitionId = 1,
+ AutoCommitMode = autoCommitMode,
+ AutoCommit = autoCommitMode != AutoCommitMode.Disabled,
+ PollingIntervalMs = 0,
+ Deserializer = deserializer
+ };
+ return new IggyConsumer<string>(client, config, NullLoggerFactory.Instance);
+ }
+
+ private async Task<TestStreamInfo> CreateTestStreamWithMessages(IIggyClient client, Protocol protocol,
+ uint partitionsCount = 5, int messagesPerPartition = 100)
+ {
+ var streamId = $"typed_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}";
+ var topicId = "test_topic";
+
+ await client.CreateStreamAsync(streamId);
+ await client.CreateTopicAsync(Identifier.String(streamId), topicId, partitionsCount);
+
+ for (uint partitionId = 0; partitionId < partitionsCount; partitionId++)
+ {
+ var messages = new List<Message>();
+ for (var i = 0; i < messagesPerPartition; i++)
+ {
+ messages.Add(new Message(Guid.NewGuid(),
+ Encoding.UTF8.GetBytes($"Test message {i} for partition {partitionId}")));
+ }
+
+ await client.SendMessagesAsync(Identifier.String(streamId),
+ Identifier.String(topicId),
+ Partitioning.PartitionId((int)partitionId),
+ messages);
+ }
+
+ return new TestStreamInfo(streamId, topicId, partitionsCount, messagesPerPartition);
+ }
+
+ private record TestStreamInfo(string StreamId, string TopicId, uint PartitionsCount, int MessagesPerPartition);
+
+ private sealed class Utf8StringDeserializer : IDeserializer<string>
+ {
+ public string Deserialize(ReadOnlyMemory<byte> data)
+ {
+ return Encoding.UTF8.GetString(data.Span);
+ }
+ }
+
+ private sealed class FailingDeserializer : IDeserializer<string>
+ {
+ public string Deserialize(ReadOnlyMemory<byte> data)
+ {
+ throw new InvalidOperationException("Intentional deserialization failure");
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
index 1b229b82c..a0bc77def 100644
--- a/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
+++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Iggy_SDK.Tests.Integration.csproj
@@ -1,45 +1,44 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
- <PropertyGroup>
- <ImplicitUsings>enable</ImplicitUsings>
- <Nullable>enable</Nullable>
- <OutputType>Exe</OutputType>
- <TargetFrameworks>net8.0;net10.0</TargetFrameworks>
- <AssemblyName>Apache.Iggy.Tests.Integrations</AssemblyName>
- <RootNamespace>Apache.Iggy.Tests.Integrations</RootNamespace>
- <IsTestingPlatformApplication>true</IsTestingPlatformApplication>
- </PropertyGroup>
-
- <ItemGroup>
- <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
- <PackageReference Include="Microsoft.Testing.Extensions.Retry" />
- <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
- <PackageReference Include="Shouldly" />
- <PackageReference Include="Testcontainers" />
- <PackageReference Include="TUnit" />
- </ItemGroup>
-
- <ItemGroup>
- <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj" />
- </ItemGroup>
-
- <ItemGroup>
- <Content Include="..\..\..\core\certs\iggy.pfx">
- <Link>Certs\iggy.pfx</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- <Content Include="..\..\..\core\certs\iggy_ca_cert.pem">
- <Link>Certs\iggy_ca_cert.pem</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- <Content Include="..\..\..\core\certs\iggy_cert.pem">
- <Link>Certs\iggy_cert.pem</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- <Content Include="..\..\..\core\certs\iggy_key.pem">
- <Link>Certs\iggy_key.pem</Link>
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </Content>
- </ItemGroup>
-
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <ImplicitUsings>enable</ImplicitUsings>
+ <Nullable>enable</Nullable>
+ <OutputType>Exe</OutputType>
+ <TargetFrameworks>net8.0;net10.0</TargetFrameworks>
+ <AssemblyName>Apache.Iggy.Tests.Integrations</AssemblyName>
+ <RootNamespace>Apache.Iggy.Tests.Integrations</RootNamespace>
+ <IsTestingPlatformApplication>true</IsTestingPlatformApplication>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Microsoft.Testing.Extensions.CodeCoverage" />
+ <PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
+ <PackageReference Include="Shouldly" />
+ <PackageReference Include="Testcontainers" />
+ <PackageReference Include="TUnit" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\Iggy_SDK\Iggy_SDK.csproj" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <Content Include="..\..\..\core\certs\iggy.pfx">
+ <Link>Certs\iggy.pfx</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ <Content Include="..\..\..\core\certs\iggy_ca_cert.pem">
+ <Link>Certs\iggy_ca_cert.pem</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ <Content Include="..\..\..\core\certs\iggy_cert.pem">
+ <Link>Certs\iggy_cert.pem</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ <Content Include="..\..\..\core\certs\iggy_key.pem">
+ <Link>Certs\iggy_key.pem</Link>
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </Content>
+ </ItemGroup>
+
+</Project>
diff --git a/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs
new file mode 100644
index 000000000..b7e8cdbb8
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK/Consumers/DeserializedMessage.cs
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Headers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Represents a message whose payload was deserialized directly from rented memory. The rented buffer has
+/// already been returned to the pool by the time this message is yielded, so only the header, user headers,
+/// and the deserialized <see cref="Data" /> are available; the raw payload bytes are not retained.
+/// </summary>
+/// <typeparam name="T">The deserialized payload type.</typeparam>
+public sealed class DeserializedMessage<T>
+{
+ /// <summary>
+ /// Message header.
+ /// </summary>
+ public required MessageHeader Header { get; init; }
+
+ /// <summary>
+ /// The deserialized payload. Null if <see cref="Status" /> is not <see cref="MessageStatus.Success" />.
+ /// </summary>
+ public T? Data { get; init; }
+
+ /// <summary>
+ /// Parsed user headers, if present.
+ /// </summary>
+ public Dictionary<HeaderKey, HeaderValue>? UserHeaders { get; init; }
+
+ /// <summary>
+ /// The current offset of this message in the partition.
+ /// </summary>
+ public required ulong CurrentOffset { get; init; }
+
+ /// <summary>
+ /// The partition ID from which this message was consumed.
+ /// </summary>
+ public uint PartitionId { get; init; }
+
+ /// <summary>
+ /// The status of the message (Success, DecryptionFailed, DeserializationFailed).
+ /// </summary>
+ public MessageStatus Status { get; init; } = MessageStatus.Success;
+
+ /// <summary>
+ /// The exception that occurred during processing, if any.
+ /// </summary>
+ public Exception? Error { get; init; }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs
index 4a82d1108..e22372144 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IDeserializer.cs
@@ -1,54 +1,50 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// Interface for deserializing message payloads from byte arrays to type T.
-/// <para>
-/// No type constraints are enforced on T to provide maximum flexibility.
-/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the target type.
-/// </para>
-/// </summary>
-/// <typeparam name="T">
-/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable.
-/// The deserializer implementation must be able to produce instances of the specific type.
-/// </typeparam>
-/// <remarks>
-/// Implementations should throw appropriate exceptions (e.g., <see cref="System.FormatException" />,
-/// <see cref="System.ArgumentException" />, or <see cref="System.InvalidOperationException" />)
-/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by
-/// <see cref="IggyConsumer{T}" /> during message processing.
-/// </remarks>
-public interface IDeserializer<out T>
-{
- /// <summary>
- /// Deserializes a byte array into an instance of type T.
- /// </summary>
- /// <param name="data">The byte array containing the serialized data to deserialize.</param>
- /// <returns>An instance of type T representing the deserialized data.</returns>
- /// <exception cref="System.FormatException">
- /// Thrown when the data format is invalid and cannot be deserialized.
- /// </exception>
- /// <exception cref="System.ArgumentException">
- /// Thrown when the data cannot be deserialized due to invalid content or structure.
- /// </exception>
- /// <exception cref="System.InvalidOperationException">
- /// Thrown when the deserialization operation fails due to state issues.
- /// </exception>
- T Deserialize(byte[] data);
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Interface for deserializing message payloads from byte arrays to type T.
+/// <para>
+/// No type constraints are enforced on T to provide maximum flexibility.
+/// Implementations are responsible for ensuring that the provided byte data can be properly deserialized to the
+/// target type.
+/// </para>
+/// </summary>
+/// <typeparam name="T">
+/// The target type for deserialization. Can be any type - reference or value type, nullable or non-nullable.
+/// The deserializer implementation must be able to produce instances of the specific type.
+/// </typeparam>
+/// <remarks>
+/// Implementations should throw appropriate exceptions (e.g., <see cref="System.FormatException" />,
+/// <see cref="System.ArgumentException" />, or <see cref="System.InvalidOperationException" />)
+/// if the provided data cannot be deserialized to type T. These exceptions will be caught and logged by
+/// <see cref="IggyConsumer{T}" /> during message processing.
+/// </remarks>
+public interface IDeserializer<out T>
+{
+ /// <summary>
+ /// Deserializes a read-only memory into an instance of type T. Callers may pass a <c>byte[]</c> directly
+ /// thanks to the implicit conversion to <see cref="ReadOnlyMemory{T}" />.
+ /// </summary>
+ /// <param name="data">
+ /// Read-only memory containing the serialized data. The implementation MUST NOT retain a reference to
+ /// the span after returning.
+ /// </param>
+ /// <returns>An instance of type T representing the deserialized data.</returns>
+ T Deserialize(ReadOnlyMemory<byte> data);
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
new file mode 100644
index 000000000..a18f4c784
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs
@@ -0,0 +1,250 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Headers;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Microsoft.Extensions.Logging;
+
+namespace Apache.Iggy.Consumers;
+
+public partial class IggyConsumer
+{
+ private readonly Channel<ReceivedRentedMessage> _rentedChannel = Channel.CreateUnbounded<ReceivedRentedMessage>();
+
+ /// <summary>
+ /// Receives messages asynchronously as an async stream of rented messages. Each yielded
+ /// <see cref="ReceivedRentedMessage" /> shares its underlying pooled buffer with the other messages from the
+ /// same poll and MUST be disposed by the caller when processing is complete. The buffer is returned to the
+ /// pool once every message of its batch has been disposed.
+ /// </summary>
+ /// <param name="ct">Cancellation token to stop receiving messages.</param>
+ /// <returns>An async enumerable of rented messages.</returns>
+ /// <exception cref="ConsumerNotInitializedException">Thrown when <see cref="InitAsync" /> has not been called.</exception>
+ public async IAsyncEnumerable<ReceivedRentedMessage> ReceiveRentedAsync(
+ [EnumeratorCancellation] CancellationToken ct = default)
+ {
+ if (!_isInitialized)
+ {
+ throw new ConsumerNotInitializedException();
+ }
+
+ do
+ {
+ if (!_rentedChannel.Reader.TryRead(out var message))
+ {
+ await PollRentedMessagesAsync(ct);
+ continue;
+ }
+
+ yield return message;
+
+ if (_config.AutoCommitMode == AutoCommitMode.AfterReceive)
+ {
+ await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct);
+ }
+ } while (!ct.IsCancellationRequested);
+ }
+
+ /// <summary>
+ /// Publishes a single rented message from a polled batch to the consumer channel. Called once per
+ /// message during <see cref="PollRentedMessagesAsync" />. The caller acquires a reference on
+ /// <paramref name="rental" /> before invocation; the channel reader is expected to release that
+ /// reference by disposing the produced <see cref="ReceivedRentedMessage" />. Override to redirect
+ /// rented batches to a different sink (e.g. typed deserialization) — overrides must ensure the
+ /// acquired reference is released exactly once on every path.
+ /// </summary>
+ /// <param name="rental">Reference-counted handle around the polled batch. Caller has already acquired one reference.</param>
+ /// <param name="message">
+ /// The rented message to publish. Payload and raw headers are slices of pooled memory tied to
+ /// <paramref name="rental" />.
+ /// </param>
+ /// <param name="partitionId">Partition the message was polled from.</param>
+ /// <param name="status">Outcome of any prior processing (e.g. decryption).</param>
+ /// <param name="error">Exception captured if <paramref name="status" /> is non-success; otherwise null.</param>
+ /// <param name="ct">Cancellation token.</param>
+ protected virtual async Task PublishRentedAsync(RentedBatchHandle rental,
+ RentedMessageResponse message,
+ uint partitionId,
+ MessageStatus status,
+ Exception? error,
+ CancellationToken ct)
+ {
+ await _rentedChannel.Writer.WriteAsync(new ReceivedRentedMessage
+ {
+ Handle = rental,
+ Message = message,
+ CurrentOffset = message.Header.Offset,
+ PartitionId = partitionId,
+ Status = status,
+ Error = error
+ }, ct);
+ }
+
+ /// <summary>
+ /// Polls a rented batch from the server and publishes it via <see cref="PublishRentedAsync" />.
+ /// Handles decryption, offset tracking, and auto-commit logic. Rental lifetime is managed via
+ /// <see cref="RentedBatchHandle" /> shared by every produced message.
+ /// </summary>
+ protected async Task PollRentedMessagesAsync(CancellationToken ct)
+ {
+ if (!_joinedConsumerGroup)
+ {
+ LogConsumerGroupNotJoinedYetSkippingPolling();
+ return;
+ }
+
+ await _pollingSemaphore.WaitAsync(ct);
+
+ PolledMessagesRental? rental = null;
+ RentedBatchHandle? batchHandle = null;
+ try
+ {
+ if (_config.PollingIntervalMs > 0)
+ {
+ await WaitBeforePollingAsync(ct);
+ }
+
+ rental = await _client.PollMessagesRentedAsync(_config.StreamId, _config.TopicId,
+ _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize,
+ _config.AutoCommit, ct);
+
+ if (rental.Messages.Count == 0)
+ {
+ if (_logger.IsEnabled(LogLevel.Debug))
+ {
+ _logger.LogDebug("No messages received from poll for partition {PartitionId}", rental.PartitionId);
+ }
+
+ return;
+ }
+
+ var partitionId = (uint)rental.PartitionId;
+
+ var hasLastOffset = _lastPolledOffset.TryGetValue(rental.PartitionId, out var lastPolledPartitionOffset);
+
+ var currentOffset = 0ul;
+
+ batchHandle = new RentedBatchHandle(rental);
+ var anyNewMessages = false;
+ foreach (var message in rental.Messages)
+ {
+ if (hasLastOffset && message.Header.Offset <= lastPolledPartitionOffset)
+ {
+ continue;
+ }
+
+ var processedMessage = message;
+ var status = MessageStatus.Success;
+ Exception? error = null;
+
+ if (_config.MessageEncryptor != null)
+ {
+ try
+ {
+ var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload.Span);
+
+ Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
+ if (!message.RawUserHeaders.IsEmpty)
+ {
+ var decryptedHeaderBytes =
+ _config.MessageEncryptor.Decrypt(message.RawUserHeaders.Span);
+ decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
+ }
+
+ processedMessage = new RentedMessageResponse
+ {
+ Header = message.Header,
+ Payload = decryptedPayload,
+ RawUserHeaders = ReadOnlyMemory<byte>.Empty,
+ UserHeaders = decryptedHeaders
+ };
+ }
+ catch (Exception ex)
+ {
+ LogFailedToDecryptMessage(ex, message.Header.Offset);
+ status = MessageStatus.DecryptionFailed;
+ error = ex;
+ }
+ }
+
+ currentOffset = message.Header.Offset;
+ batchHandle.Acquire();
+ try
+ {
+ await PublishRentedAsync(batchHandle, processedMessage, partitionId, status, error, ct);
+ }
+ catch
+ {
+ batchHandle.Release();
+ throw;
+ }
+
+ anyNewMessages = true;
+ }
+
+ if (!anyNewMessages
+ && _config.AutoCommitMode != AutoCommitMode.Disabled)
+ {
+ if (_logger.IsEnabled(LogLevel.Debug))
+ {
+ _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}",
+ lastPolledPartitionOffset, rental.PartitionId);
+ }
+
+ await StoreOffsetAsync(lastPolledPartitionOffset, partitionId, false, ct);
+ }
+
+ if (anyNewMessages)
+ {
+ _lastPolledOffset.AddOrUpdate(rental.PartitionId, currentOffset, (_, _) => currentOffset);
+ }
+
+ if (_config.PollingStrategy.Kind == MessagePolling.Offset)
+ {
+ _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
+ }
+ }
+ catch (OperationCanceledException) when (ct.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception ex)
+ {
+ LogFailedToPollMessages(ex);
+ _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages"));
+ }
+ finally
+ {
+ if (batchHandle is not null)
+ {
+ batchHandle.Release();
+ }
+ else
+ {
+ rental?.Dispose();
+ }
+
+ _pollingSemaphore.Release();
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
index 76264f988..47349f9cf 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs
@@ -1,536 +1,545 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Collections.Concurrent;
-using System.Runtime.CompilerServices;
-using System.Threading.Channels;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Headers;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
-using Apache.Iggy.Utils;
-using Microsoft.Extensions.Logging;
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// High-level consumer for receiving messages from Iggy streams.
-/// Provides automatic polling, offset management, and consumer group support.
-/// </summary>
-public partial class IggyConsumer : IAsyncDisposable
-{
- private readonly Channel<ReceivedMessage> _channel;
- private readonly IIggyClient _client;
- private readonly IggyConsumerConfig _config;
- private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1);
- private readonly EventAggregator<ConsumerErrorEventArgs> _consumerErrorEvents;
- private readonly ConcurrentDictionary<int, ulong> _lastPolledOffset = new();
- private readonly ILogger<IggyConsumer> _logger;
- private readonly SemaphoreSlim _pollingSemaphore = new(1, 1);
- private string? _consumerGroupName;
- private int _disposeState;
- private volatile bool _isInitialized;
- private volatile bool _joinedConsumerGroup;
- private long _lastPolledAtMs;
-
- /// <summary>
- /// Initializes a new instance of the <see cref="IggyConsumer" /> class
- /// </summary>
- /// <param name="client">The Iggy client for server communication</param>
- /// <param name="config">Consumer configuration settings</param>
- /// <param name="loggerFactory">Logger for creating loggers</param>
- public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory)
- {
- _client = client;
- _config = config;
- _logger = loggerFactory.CreateLogger<IggyConsumer>();
-
- _channel = Channel.CreateUnbounded<ReceivedMessage>();
- _consumerErrorEvents = new EventAggregator<ConsumerErrorEventArgs>(loggerFactory);
- }
-
- /// <summary>
- /// Disposes the consumer, leaving consumer groups and logging out if applicable
- /// </summary>
- public async ValueTask DisposeAsync()
- {
- if (Interlocked.Exchange(ref _disposeState, 1) == 1)
- {
- return;
- }
-
- if (_isInitialized)
- {
- _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
- }
-
- if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized)
- {
- try
- {
- await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId,
- Identifier.String(_consumerGroupName));
-
- LogLeftConsumerGroup(_consumerGroupName);
- }
- catch (Exception e)
- {
- LogFailedToLeaveConsumerGroup(e, _consumerGroupName);
- }
- }
-
- if (_config.CreateIggyClient && _isInitialized)
- {
- try
- {
- await _client.LogoutUserAsync();
- _client.Dispose();
- }
- catch (Exception e)
- {
- LogFailedToLogoutOrDispose(e);
- }
- }
-
- _consumerErrorEvents.Clear();
- _pollingSemaphore.Dispose();
- _connectionStateSemaphore.Dispose();
-
- }
-
- /// <summary>
- /// Initializes the consumer by logging in (if needed) and setting up consumer groups
- /// </summary>
- /// <param name="ct">Cancellation token</param>
- /// <exception cref="InvalidConsumerGroupNameException">Thrown when consumer group name is invalid</exception>
- /// <exception cref="ConsumerGroupNotFoundException">Thrown when consumer group doesn't exist and auto-creation is disabled</exception>
- public async Task InitAsync(CancellationToken ct = default)
- {
- if (_isInitialized)
- {
- return;
- }
-
- await _connectionStateSemaphore.WaitAsync(ct);
- try
- {
- if (_isInitialized)
- {
- return;
- }
-
- if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null)
- {
- LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup();
- _config.PartitionId = null;
- }
-
- await _client.ConnectAsync(ct);
-
- if (_config.CreateIggyClient)
- {
- await _client.LoginUserAsync(_config.Login, _config.Password, ct);
- }
-
- await InitializeConsumerGroupAsync(ct);
-
- _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
-
- _isInitialized = true;
- }
- finally
- {
- _connectionStateSemaphore.Release();
- }
- }
-
- /// <summary>
- /// Receives messages asynchronously from the consumer as an async stream.
- /// Messages are automatically polled from the server and buffered in a bounded channel.
- /// </summary>
- /// <param name="ct">Cancellation token to stop receiving messages</param>
- /// <returns>An async enumerable of received messages</returns>
- /// <exception cref="ConsumerNotInitializedException">Thrown when InitAsync has not been called</exception>
- public async IAsyncEnumerable<ReceivedMessage> ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default)
- {
- if (!_isInitialized)
- {
- throw new ConsumerNotInitializedException();
- }
-
- do
- {
- if (!_channel.Reader.TryRead(out var message))
- {
- await PollMessagesAsync(ct);
- continue;
- }
-
- yield return message;
-
- if (_config.AutoCommitMode == AutoCommitMode.AfterReceive)
- {
- await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, ct);
- }
- } while (!ct.IsCancellationRequested);
- }
-
- /// <summary>
- /// Manually stores the consumer offset for a specific partition.
- /// Use this when auto-commit is disabled or when you need manual offset control.
- /// </summary>
- /// <param name="offset">The offset to store</param>
- /// <param name="partitionId">The partition ID</param>
- /// <param name="ct">Cancellation token</param>
- public async Task StoreOffsetAsync(ulong offset, uint partitionId, CancellationToken ct = default)
- {
- await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct);
- }
-
- /// <summary>
- /// Deletes the stored consumer offset for a specific partition.
- /// The next poll will start from the beginning or based on the polling strategy.
- /// </summary>
- /// <param name="partitionId">The partition ID</param>
- /// <param name="ct">Cancellation token</param>
- public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default)
- {
- await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct);
- }
-
- /// <summary>
- /// Event raised when an error occurs during polling.
- /// </summary>
- /// <param name="callback">Callback method</param>
- public void SubscribeToErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
- {
- _consumerErrorEvents.Subscribe(callback);
- }
-
- /// <summary>
- /// Unsubscribe from error events
- /// </summary>
- /// <param name="callback"></param>
- public void UnsubscribeFromErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
- {
- _consumerErrorEvents.Unsubscribe(callback);
- }
-
- /// <summary>
- /// Initializes consumer group if configured, creating and joining as needed
- /// </summary>
- private async Task InitializeConsumerGroupAsync(CancellationToken ct = default)
- {
- if (_joinedConsumerGroup)
- {
- return;
- }
-
- if (_config.Consumer.Type == ConsumerType.Consumer)
- {
- _joinedConsumerGroup = true;
- return;
- }
-
- _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String
- ? _config.Consumer.ConsumerId.GetString()
- : _config.ConsumerGroupName;
-
- if (string.IsNullOrEmpty(_consumerGroupName))
- {
- throw new InvalidConsumerGroupNameException("Consumer group name is empty or null.");
- }
-
- try
- {
- var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId,
- Identifier.String(_consumerGroupName), ct);
-
- if (existingGroup == null && _config.CreateConsumerGroupIfNotExists)
- {
- LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
-
- var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct);
-
- if (createdGroup)
- {
- LogConsumerGroupCreated(_consumerGroupName);
- }
- }
- else if (existingGroup == null)
- {
- throw new ConsumerGroupNotFoundException(_consumerGroupName);
- }
-
- if (_config.JoinConsumerGroup)
- {
- LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
-
- await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId,
- Identifier.String(_consumerGroupName), ct);
-
- _joinedConsumerGroup = true;
- LogConsumerGroupJoined(_consumerGroupName);
- }
- }
- catch (Exception ex)
- {
- LogFailedToInitializeConsumerGroup(ex, _consumerGroupName);
- throw;
- }
- }
-
- /// <summary>
- /// Attempts to create a consumer group, handling the case where it already exists
- /// </summary>
- /// <returns>True if the group was created or already exists, false on error</returns>
- private async Task<bool> TryCreateConsumerGroupAsync(string groupName, CancellationToken ct)
- {
- try
- {
- await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId,
- groupName, ct);
- }
- catch (IggyInvalidStatusCodeException ex)
- {
- // 5004 - Consumer group already exists TODO: refactor errors
- if (ex.StatusCode != 5004)
- {
- LogFailedToCreateConsumerGroup(ex, groupName);
- return false;
- }
-
- return true;
- }
-
- return true;
- }
-
- /// <summary>
- /// Polls messages from the server and writes them to the internal channel.
- /// Handles decryption, offset tracking, and auto-commit logic.
- /// Uses semaphore to ensure single concurrent polling operation.
- /// </summary>
- private async Task PollMessagesAsync(CancellationToken ct)
- {
- if (!_joinedConsumerGroup)
- {
- LogConsumerGroupNotJoinedYetSkippingPolling();
- return;
- }
-
- await _pollingSemaphore.WaitAsync(ct);
-
- try
- {
- if (_config.PollingIntervalMs > 0)
- {
- await WaitBeforePollingAsync(ct);
- }
-
- var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId,
- _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize,
- _config.AutoCommit, ct);
-
- var receiveMessages = messages.Messages.Count > 0;
-
- if (_lastPolledOffset.TryGetValue(messages.PartitionId, out var lastPolledPartitionOffset))
- {
- messages.Messages = messages.Messages.Where(x => x.Header.Offset > lastPolledPartitionOffset).ToList();
- }
-
- if (messages.Messages.Count == 0
- && receiveMessages
- && _config.AutoCommitMode != AutoCommitMode.Disabled)
- {
- _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}",
- lastPolledPartitionOffset, messages.PartitionId);
- await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, ct);
- }
-
- if (messages.Messages.Count == 0)
- {
- return;
- }
-
- var currentOffset = 0ul;
- foreach (var message in messages.Messages)
- {
- var processedMessage = message;
- var status = MessageStatus.Success;
- Exception? error = null;
-
- if (_config.MessageEncryptor != null)
- {
- try
- {
- var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload);
-
- Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
- if (message.RawUserHeaders is { Length: > 0 })
- {
- var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders);
- decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
- }
-
- processedMessage = new MessageResponse
- {
- Header = message.Header,
- Payload = decryptedPayload,
- UserHeaders = decryptedHeaders
- };
- }
- catch (Exception ex)
- {
- LogFailedToDecryptMessage(ex, message.Header.Offset);
- status = MessageStatus.DecryptionFailed;
- error = ex;
- }
- }
-
- var receivedMessage = new ReceivedMessage
- {
- Message = processedMessage,
- CurrentOffset = processedMessage.Header.Offset,
- PartitionId = (uint)messages.PartitionId,
- Status = status,
- Error = error
- };
-
- await _channel.Writer.WriteAsync(receivedMessage, ct);
- currentOffset = receivedMessage.CurrentOffset;
- }
-
- _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset,
- (_, _) => currentOffset);
-
- if (_config.PollingStrategy.Kind == MessagePolling.Offset)
- {
- _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
- }
- }
- catch (Exception ex)
- {
- LogFailedToPollMessages(ex);
- _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages"));
- }
- finally
- {
- _pollingSemaphore.Release();
- }
- }
-
- /// <summary>
- /// Implements polling interval throttling to avoid excessive server requests.
- /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments.
- /// </summary>
- private async Task WaitBeforePollingAsync(CancellationToken ct)
- {
- var intervalMs = _config.PollingIntervalMs;
- if (intervalMs <= 0)
- {
- return;
- }
-
- var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
- var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs);
-
- if (nowMs < lastPolledAtMs)
- {
- LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs);
- await Task.Delay(intervalMs, ct);
- Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
- return;
- }
-
- var elapsedMs = nowMs - lastPolledAtMs;
- if (elapsedMs >= intervalMs)
- {
- LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs);
- Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
- return;
- }
-
- var remainingMs = intervalMs - elapsedMs;
- LogWaitingBeforePolling(remainingMs);
-
- if (remainingMs > 0)
- {
- await Task.Delay((int)remainingMs, ct);
- }
-
- Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
- }
-
- /// <summary>
- /// Handles connection state changes from the client.
- /// </summary>
- /// <param name="e">Event object</param>
- private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e)
- {
- LogConnectionStateChanged(e.PreviousState, e.CurrentState);
-
- await _connectionStateSemaphore.WaitAsync();
- try
- {
- if (e.CurrentState == ConnectionState.Disconnected)
- {
- _joinedConsumerGroup = false;
- }
-
- if (e.CurrentState != ConnectionState.Authenticated
- || e.PreviousState == ConnectionState.Authenticated
- || _joinedConsumerGroup)
- {
- return;
- }
-
- await RejoinConsumerGroupOnReconnectionAsync();
- }
- finally
- {
- _connectionStateSemaphore.Release();
- }
- }
-
- /// <summary>
- /// Asynchronously rejoins the consumer group after a client reconnection.
- /// This restores the consumer group membership that was lost during the connection failure.
- /// </summary>
- private async Task RejoinConsumerGroupOnReconnectionAsync()
- {
- if (string.IsNullOrEmpty(_consumerGroupName))
- {
- LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup();
- return;
- }
-
- try
- {
- await InitializeConsumerGroupAsync();
- }
- catch (Exception ex)
- {
- LogFailedToRejoinConsumerGroup(ex, _consumerGroupName);
- _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex,
- "Failed to rejoin consumer group after reconnection"));
- }
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Headers;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Utils;
+using Microsoft.Extensions.Logging;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// High-level consumer for receiving messages from Iggy streams.
+/// Provides automatic polling, offset management, and consumer group support.
+/// </summary>
+public partial class IggyConsumer : IAsyncDisposable
+{
+ private readonly Channel<ReceivedMessage> _channel;
+ private readonly IIggyClient _client;
+ private readonly IggyConsumerConfig _config;
+ private readonly SemaphoreSlim _connectionStateSemaphore = new(1, 1);
+ private readonly EventAggregator<ConsumerErrorEventArgs> _consumerErrorEvents;
+ private readonly ConcurrentDictionary<int, ulong> _lastPolledOffset = new();
+ private readonly ILogger<IggyConsumer> _logger;
+ private readonly SemaphoreSlim _pollingSemaphore = new(1, 1);
+ private string? _consumerGroupName;
+ private int _disposeState;
+ private volatile bool _isInitialized;
+ private volatile bool _joinedConsumerGroup;
+ private long _lastPolledAtMs;
+
+ /// <summary>Whether this consumer has been initialized via <see cref="InitAsync" />.</summary>
+ protected bool IsInitialized => _isInitialized;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="IggyConsumer" /> class
+ /// </summary>
+ /// <param name="client">The Iggy client for server communication</param>
+ /// <param name="config">Consumer configuration settings</param>
+ /// <param name="loggerFactory">Logger for creating loggers</param>
+ public IggyConsumer(IIggyClient client, IggyConsumerConfig config, ILoggerFactory loggerFactory)
+ {
+ _client = client;
+ _config = config;
+ _logger = loggerFactory.CreateLogger<IggyConsumer>();
+
+ _channel = Channel.CreateUnbounded<ReceivedMessage>();
+ _consumerErrorEvents = new EventAggregator<ConsumerErrorEventArgs>(loggerFactory);
+ }
+
+ /// <summary>
+ /// Disposes the consumer, leaving consumer groups and logging out if applicable
+ /// </summary>
+ public async ValueTask DisposeAsync()
+ {
+ if (Interlocked.Exchange(ref _disposeState, 1) == 1)
+ {
+ return;
+ }
+
+ if (_isInitialized)
+ {
+ _client.UnsubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
+ }
+
+ if (!string.IsNullOrEmpty(_consumerGroupName) && _isInitialized)
+ {
+ try
+ {
+ await _client.LeaveConsumerGroupAsync(_config.StreamId, _config.TopicId,
+ Identifier.String(_consumerGroupName));
+
+ LogLeftConsumerGroup(_consumerGroupName);
+ }
+ catch (Exception e)
+ {
+ LogFailedToLeaveConsumerGroup(e, _consumerGroupName);
+ }
+ }
+
+ if (_config.CreateIggyClient && _isInitialized)
+ {
+ try
+ {
+ await _client.LogoutUserAsync();
+ _client.Dispose();
+ }
+ catch (Exception e)
+ {
+ LogFailedToLogoutOrDispose(e);
+ }
+ }
+
+ _consumerErrorEvents.Clear();
+ _pollingSemaphore.Dispose();
+ _connectionStateSemaphore.Dispose();
+ }
+
+ /// <summary>
+ /// Initializes the consumer by logging in (if needed) and setting up consumer groups
+ /// </summary>
+ /// <param name="ct">Cancellation token</param>
+ /// <exception cref="InvalidConsumerGroupNameException">Thrown when consumer group name is invalid</exception>
+ /// <exception cref="ConsumerGroupNotFoundException">Thrown when consumer group doesn't exist and auto-creation is disabled</exception>
+ public async Task InitAsync(CancellationToken ct = default)
+ {
+ if (_isInitialized)
+ {
+ return;
+ }
+
+ await _connectionStateSemaphore.WaitAsync(ct);
+ try
+ {
+ if (_isInitialized)
+ {
+ return;
+ }
+
+ if (_config.Consumer.Type == ConsumerType.ConsumerGroup && _config.PartitionId != null)
+ {
+ LogPartitionIdIsIgnoredWhenConsumerTypeIsConsumerGroup();
+ _config.PartitionId = null;
+ }
+
+ await _client.ConnectAsync(ct);
+
+ if (_config.CreateIggyClient)
+ {
+ await _client.LoginUserAsync(_config.Login, _config.Password, ct);
+ }
+
+ await InitializeConsumerGroupAsync(ct);
+
+ _client.SubscribeConnectionEvents(OnClientConnectionStateChangedAsync);
+
+ _isInitialized = true;
+ }
+ finally
+ {
+ _connectionStateSemaphore.Release();
+ }
+ }
+
+ /// <summary>
+ /// Receives messages asynchronously from the consumer as an async stream.
+ /// Messages are automatically polled from the server and buffered in a bounded channel.
+ /// </summary>
+ /// <param name="ct">Cancellation token to stop receiving messages</param>
+ /// <returns>An async enumerable of received messages</returns>
+ /// <exception cref="ConsumerNotInitializedException">Thrown when InitAsync has not been called</exception>
+ public async IAsyncEnumerable<ReceivedMessage> ReceiveAsync([EnumeratorCancellation] CancellationToken ct = default)
+ {
+ if (!_isInitialized)
+ {
+ throw new ConsumerNotInitializedException();
+ }
+
+ do
+ {
+ if (!_channel.Reader.TryRead(out var message))
+ {
+ await PollMessagesAsync(ct);
+ continue;
+ }
+
+ yield return message;
+
+ if (_config.AutoCommitMode == AutoCommitMode.AfterReceive)
+ {
+ await StoreOffsetAsync(message.CurrentOffset, message.PartitionId, false, ct);
+ }
+ } while (!ct.IsCancellationRequested);
+ }
+
+ /// <summary>
+ /// Manually stores the consumer offset for a specific partition.
+ /// Use this when auto-commit is disabled or when you need manual offset control.
+ /// </summary>
+ /// <param name="offset">The offset to store</param>
+ /// <param name="partitionId">The partition ID</param>
+ /// <param name="resetLastPooled"></param>
+ /// <param name="ct">Cancellation token</param>
+ public async Task StoreOffsetAsync(ulong offset, uint partitionId, bool resetLastPooled = false,
+ CancellationToken ct = default)
+ {
+ await _client.StoreOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, offset, partitionId, ct);
+
+ if (resetLastPooled)
+ {
+ _lastPolledOffset[(int)partitionId] = offset;
+ }
+ }
+
+ /// <summary>
+ /// Deletes the stored consumer offset for a specific partition.
+ /// The next poll will start from the beginning or based on the polling strategy.
+ /// </summary>
+ /// <param name="partitionId">The partition ID</param>
+ /// <param name="ct">Cancellation token</param>
+ public async Task DeleteOffsetAsync(uint partitionId, CancellationToken ct = default)
+ {
+ await _client.DeleteOffsetAsync(_config.Consumer, _config.StreamId, _config.TopicId, partitionId, ct);
+ }
+
+ /// <summary>
+ /// Event raised when an error occurs during polling.
+ /// </summary>
+ /// <param name="callback">Callback method</param>
+ public void SubscribeToErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
+ {
+ _consumerErrorEvents.Subscribe(callback);
+ }
+
+ /// <summary>
+ /// Unsubscribe from error events
+ /// </summary>
+ /// <param name="callback"></param>
+ public void UnsubscribeFromErrorEvents(Func<ConsumerErrorEventArgs, Task> callback)
+ {
+ _consumerErrorEvents.Unsubscribe(callback);
+ }
+
+ /// <summary>
+ /// Initializes consumer group if configured, creating and joining as needed
+ /// </summary>
+ private async Task InitializeConsumerGroupAsync(CancellationToken ct = default)
+ {
+ if (_joinedConsumerGroup)
+ {
+ return;
+ }
+
+ if (_config.Consumer.Type == ConsumerType.Consumer)
+ {
+ _joinedConsumerGroup = true;
+ return;
+ }
+
+ _consumerGroupName = _config.Consumer.ConsumerId.Kind == IdKind.String
+ ? _config.Consumer.ConsumerId.GetString()
+ : _config.ConsumerGroupName;
+
+ if (string.IsNullOrEmpty(_consumerGroupName))
+ {
+ throw new InvalidConsumerGroupNameException("Consumer group name is empty or null.");
+ }
+
+ try
+ {
+ var existingGroup = await _client.GetConsumerGroupByIdAsync(_config.StreamId, _config.TopicId,
+ Identifier.String(_consumerGroupName), ct);
+
+ if (existingGroup == null && _config.CreateConsumerGroupIfNotExists)
+ {
+ LogCreatingConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
+
+ var createdGroup = await TryCreateConsumerGroupAsync(_consumerGroupName, ct);
+
+ if (createdGroup)
+ {
+ LogConsumerGroupCreated(_consumerGroupName);
+ }
+ }
+ else if (existingGroup == null)
+ {
+ throw new ConsumerGroupNotFoundException(_consumerGroupName);
+ }
+
+ if (_config.JoinConsumerGroup)
+ {
+ LogJoiningConsumerGroup(_consumerGroupName, _config.StreamId, _config.TopicId);
+
+ await _client.JoinConsumerGroupAsync(_config.StreamId, _config.TopicId,
+ Identifier.String(_consumerGroupName), ct);
+
+ _joinedConsumerGroup = true;
+ LogConsumerGroupJoined(_consumerGroupName);
+ }
+ }
+ catch (Exception ex)
+ {
+ LogFailedToInitializeConsumerGroup(ex, _consumerGroupName);
+ throw;
+ }
+ }
+
+ /// <summary>
+ /// Attempts to create a consumer group, handling the case where it already exists
+ /// </summary>
+ /// <returns>True if the group was created or already exists, false on error</returns>
+ private async Task<bool> TryCreateConsumerGroupAsync(string groupName, CancellationToken ct)
+ {
+ try
+ {
+ await _client.CreateConsumerGroupAsync(_config.StreamId, _config.TopicId,
+ groupName, ct);
+ }
+ catch (IggyInvalidStatusCodeException ex)
+ {
+ // 5004 - Consumer group already exists TODO: refactor errors
+ if (ex.StatusCode != 5004)
+ {
+ LogFailedToCreateConsumerGroup(ex, groupName);
+ return false;
+ }
+
+ return true;
+ }
+
+ return true;
+ }
+
+ /// <summary>
+ /// Polls messages from the server and writes them to the internal channel.
+ /// Handles decryption, offset tracking, and auto-commit logic.
+ /// Uses semaphore to ensure single concurrent polling operation.
+ /// </summary>
+ private async Task PollMessagesAsync(CancellationToken ct)
+ {
+ if (!_joinedConsumerGroup)
+ {
+ LogConsumerGroupNotJoinedYetSkippingPolling();
+ return;
+ }
+
+ await _pollingSemaphore.WaitAsync(ct);
+
+ try
+ {
+ if (_config.PollingIntervalMs > 0)
+ {
+ await WaitBeforePollingAsync(ct);
+ }
+
+ var messages = await _client.PollMessagesAsync(_config.StreamId, _config.TopicId,
+ _config.PartitionId, _config.Consumer, _config.PollingStrategy, _config.BatchSize,
+ _config.AutoCommit, ct);
+
+ var receiveMessages = messages.Messages.Count > 0;
+
+ if (_lastPolledOffset.TryGetValue(messages.PartitionId, out var lastPolledPartitionOffset))
+ {
+ messages.Messages = messages.Messages.Where(x => x.Header.Offset > lastPolledPartitionOffset).ToList();
+ }
+
+ if (messages.Messages.Count == 0
+ && receiveMessages
+ && _config.AutoCommitMode != AutoCommitMode.Disabled)
+ {
+ _logger.LogDebug("No new messages found, committing offset {Offset} for partition {PartitionId}",
+ lastPolledPartitionOffset, messages.PartitionId);
+ await StoreOffsetAsync(lastPolledPartitionOffset, (uint)messages.PartitionId, false, ct);
+ }
+
+ if (messages.Messages.Count == 0)
+ {
+ return;
+ }
+
+ var currentOffset = 0ul;
+ foreach (var message in messages.Messages)
+ {
+ var processedMessage = message;
+ var status = MessageStatus.Success;
+ Exception? error = null;
+
+ if (_config.MessageEncryptor != null)
+ {
+ try
+ {
+ var decryptedPayload = _config.MessageEncryptor.Decrypt(message.Payload);
+
+ Dictionary<HeaderKey, HeaderValue>? decryptedHeaders = null;
+ if (message.RawUserHeaders is { Length: > 0 })
+ {
+ var decryptedHeaderBytes = _config.MessageEncryptor.Decrypt(message.RawUserHeaders);
+ decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytes);
+ }
+
+ processedMessage = new MessageResponse
+ {
+ Header = message.Header,
+ Payload = decryptedPayload,
+ UserHeaders = decryptedHeaders
+ };
+ }
+ catch (Exception ex)
+ {
+ LogFailedToDecryptMessage(ex, message.Header.Offset);
+ status = MessageStatus.DecryptionFailed;
+ error = ex;
+ }
+ }
+
+ var receivedMessage = new ReceivedMessage
+ {
+ Message = processedMessage,
+ CurrentOffset = processedMessage.Header.Offset,
+ PartitionId = (uint)messages.PartitionId,
+ Status = status,
+ Error = error
+ };
+
+ await _channel.Writer.WriteAsync(receivedMessage, ct);
+ currentOffset = receivedMessage.CurrentOffset;
+ }
+
+ _lastPolledOffset.AddOrUpdate(messages.PartitionId, currentOffset,
+ (_, _) => currentOffset);
+
+ if (_config.PollingStrategy.Kind == MessagePolling.Offset)
+ {
+ _config.PollingStrategy = PollingStrategy.Offset(currentOffset + 1);
+ }
+ }
+ catch (Exception ex)
+ {
+ LogFailedToPollMessages(ex);
+ _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages"));
+ }
+ finally
+ {
+ _pollingSemaphore.Release();
+ }
+ }
+
+ /// <summary>
+ /// Implements polling interval throttling to avoid excessive server requests.
+ /// Uses monotonic time tracking to ensure proper intervals even with clock adjustments.
+ /// </summary>
+ private async Task WaitBeforePollingAsync(CancellationToken ct)
+ {
+ var intervalMs = _config.PollingIntervalMs;
+ if (intervalMs <= 0)
+ {
+ return;
+ }
+
+ var nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+ var lastPolledAtMs = Interlocked.Read(ref _lastPolledAtMs);
+
+ if (nowMs < lastPolledAtMs)
+ {
+ LogMonotonicTimeWentBackwards(nowMs, lastPolledAtMs);
+ await Task.Delay(intervalMs, ct);
+ Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
+ return;
+ }
+
+ var elapsedMs = nowMs - lastPolledAtMs;
+ if (elapsedMs >= intervalMs)
+ {
+ LogNoNeedToWaitBeforePolling(nowMs, lastPolledAtMs, elapsedMs);
+ Interlocked.Exchange(ref _lastPolledAtMs, nowMs);
+ return;
+ }
+
+ var remainingMs = intervalMs - elapsedMs;
+ LogWaitingBeforePolling(remainingMs);
+
+ if (remainingMs > 0)
+ {
+ await Task.Delay((int)remainingMs, ct);
+ }
+
+ Interlocked.Exchange(ref _lastPolledAtMs, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
+ }
+
+ /// <summary>
+ /// Handles connection state changes from the client.
+ /// </summary>
+ /// <param name="e">Event object</param>
+ private async Task OnClientConnectionStateChangedAsync(ConnectionStateChangedEventArgs e)
+ {
+ LogConnectionStateChanged(e.PreviousState, e.CurrentState);
+
+ await _connectionStateSemaphore.WaitAsync();
+ try
+ {
+ if (e.CurrentState == ConnectionState.Disconnected)
+ {
+ _joinedConsumerGroup = false;
+ }
+
+ if (e.CurrentState != ConnectionState.Authenticated
+ || e.PreviousState == ConnectionState.Authenticated
+ || _joinedConsumerGroup)
+ {
+ return;
+ }
+
+ await RejoinConsumerGroupOnReconnectionAsync();
+ }
+ finally
+ {
+ _connectionStateSemaphore.Release();
+ }
+ }
+
+ /// <summary>
+ /// Asynchronously rejoins the consumer group after a client reconnection.
+ /// This restores the consumer group membership that was lost during the connection failure.
+ /// </summary>
+ private async Task RejoinConsumerGroupOnReconnectionAsync()
+ {
+ if (string.IsNullOrEmpty(_consumerGroupName))
+ {
+ LogConsumerGroupNameIsEmptySkippingRejoiningConsumerGroup();
+ return;
+ }
+
+ try
+ {
+ await InitializeConsumerGroupAsync();
+ }
+ catch (Exception ex)
+ {
+ LogFailedToRejoinConsumerGroup(ex, _consumerGroupName);
+ _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex,
+ "Failed to rejoin consumer group after reconnection"));
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
index afaa8cd94..b8ea65b62 100644
--- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
+++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerOfT.cs
@@ -1,107 +1,193 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Runtime.CompilerServices;
-using Apache.Iggy.IggyClient;
-using Microsoft.Extensions.Logging;
-
-namespace Apache.Iggy.Consumers;
-
-/// <summary>
-/// Typed consumer that automatically deserializes message payloads to type T.
-/// Extends <see cref="IggyConsumer" /> with deserialization capabilities.
-/// </summary>
-/// <typeparam name="T">The type to deserialize message payloads to</typeparam>
-public class IggyConsumer<T> : IggyConsumer
-{
- private readonly IggyConsumerConfig<T> _typedConfig;
- private readonly ILogger<IggyConsumer<T>> _typedLogger;
-
- /// <summary>
- /// Initializes a new instance of the typed <see cref="IggyConsumer{T}" /> class
- /// </summary>
- /// <param name="client">The Iggy client for server communication</param>
- /// <param name="config">Typed consumer configuration including deserializer</param>
- /// <param name="logger">Logger instance for diagnostic output</param>
- public IggyConsumer(IIggyClient client, IggyConsumerConfig<T> config, ILoggerFactory logger) : base(
- client, config, logger)
- {
- _typedConfig = config;
- _typedLogger = logger.CreateLogger<IggyConsumer<T>>();
- }
-
- /// <summary>
- /// Receives and deserializes messages from the consumer
- /// </summary>
- /// <param name="ct">Cancellation token</param>
- /// <returns>Async enumerable of deserialized messages with status</returns>
- public async IAsyncEnumerable<ReceivedMessage<T>> ReceiveDeserializedAsync(
- [EnumeratorCancellation] CancellationToken ct = default)
- {
- await foreach (var message in ReceiveAsync(ct))
- {
- if (message.Status != MessageStatus.Success)
- {
- yield return new ReceivedMessage<T>
- {
- Data = default,
- Message = message.Message,
- CurrentOffset = message.CurrentOffset,
- PartitionId = message.PartitionId,
- Status = message.Status,
- Error = message.Error
- };
- continue;
- }
-
- T? deserializedPayload = default;
- Exception? deserializationError = null;
- var status = MessageStatus.Success;
-
- try
- {
- deserializedPayload = Deserialize(message.Message.Payload);
- }
- catch (Exception ex)
- {
- _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", message.CurrentOffset);
- status = MessageStatus.DeserializationFailed;
- deserializationError = ex;
- }
-
- yield return new ReceivedMessage<T>
- {
- Data = deserializedPayload,
- Message = message.Message,
- CurrentOffset = message.CurrentOffset,
- PartitionId = message.PartitionId,
- Status = status,
- Error = deserializationError
- };
- }
- }
-
- /// <summary>
- /// Deserializes a message payload using the configured deserializer
- /// </summary>
- /// <param name="payload">The raw byte array payload to deserialize</param>
- /// <returns>The deserialized object of type T</returns>
- public T Deserialize(byte[] payload)
- {
- return _typedConfig.Deserializer.Deserialize(payload);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Runtime.CompilerServices;
+using System.Threading.Channels;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.IggyClient;
+using Microsoft.Extensions.Logging;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Typed consumer that automatically deserializes message payloads to type T.
+/// Extends <see cref="IggyConsumer" /> with deserialization capabilities.
+/// </summary>
+/// <typeparam name="T">The type to deserialize message payloads to</typeparam>
+public class IggyConsumer<T> : IggyConsumer
+{
+ private readonly Channel<DeserializedMessage<T>> _deserializedChannel =
+ Channel.CreateUnbounded<DeserializedMessage<T>>();
+
+ private readonly IggyConsumerConfig<T> _typedConfig;
+ private readonly ILogger<IggyConsumer<T>> _typedLogger;
+
+ /// <summary>
+ /// Initializes a new instance of the typed <see cref="IggyConsumer{T}" /> class
+ /// </summary>
+ /// <param name="client">The Iggy client for server communication</param>
+ /// <param name="config">Typed consumer configuration including deserializer</param>
+ /// <param name="logger">Logger instance for diagnostic output</param>
+ public IggyConsumer(IIggyClient client, IggyConsumerConfig<T> config, ILoggerFactory logger) : base(client, config,
+ logger)
+ {
+ _typedConfig = config;
+ _typedLogger = logger.CreateLogger<IggyConsumer<T>>();
+ }
+
+ /// <summary>
+ /// Receives and deserializes messages from the consumer
+ /// </summary>
+ /// <param name="ct">Cancellation token</param>
+ /// <returns>Async enumerable of deserialized messages with status</returns>
+ public async IAsyncEnumerable<ReceivedMessage<T>> ReceiveDeserializedAsync(
+ [EnumeratorCancellation] CancellationToken ct = default)
+ {
+ await foreach (var message in ReceiveAsync(ct))
+ {
+ if (message.Status != MessageStatus.Success)
+ {
+ yield return new ReceivedMessage<T>
+ {
+ Data = default,
+ Message = message.Message,
+ CurrentOffset = message.CurrentOffset,
+ PartitionId = message.PartitionId,
+ Status = message.Status,
+ Error = message.Error
+ };
+ continue;
+ }
+
+ T? deserializedPayload = default;
+ Exception? deserializationError = null;
+ var status = MessageStatus.Success;
+
+ try
+ {
+ deserializedPayload = Deserialize(message.Message.Payload);
+ }
+ catch (Exception ex)
+ {
+ _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}", message.CurrentOffset);
+ status = MessageStatus.DeserializationFailed;
+ deserializationError = ex;
+ }
+
+ yield return new ReceivedMessage<T>
+ {
+ Data = deserializedPayload,
+ Message = message.Message,
+ CurrentOffset = message.CurrentOffset,
+ PartitionId = message.PartitionId,
+ Status = status,
+ Error = deserializationError
+ };
+ }
+ }
+
+ /// <summary>
+ /// Receives and deserializes messages via the rented poll path. Each polled batch is deserialized
+ /// in full before any message is yielded — the rented buffer is returned immediately after
+ /// deserialization, independently of how fast the caller iterates. The caller does not need to
+ /// dispose anything.
+ /// </summary>
+ /// <param name="ct">Cancellation token.</param>
+ /// <returns>Async enumerable of deserialized messages with status.</returns>
+ public async IAsyncEnumerable<DeserializedMessage<T>> ReceiveRentedDeserializedAsync(
+ [EnumeratorCancellation] CancellationToken ct = default)
+ {
+ if (!IsInitialized)
+ {
+ throw new ConsumerNotInitializedException();
+ }
+
+ do
+ {
+ if (!_deserializedChannel.Reader.TryRead(out DeserializedMessage<T>? message))
+ {
+ await PollRentedMessagesAsync(ct);
+ continue;
+ }
+
+ yield return message;
+
+ if (_typedConfig.AutoCommitMode == AutoCommitMode.AfterReceive)
+ {
+ await StoreOffsetAsync(message.Header.Offset, message.PartitionId, false, ct);
+ }
+ } while (!ct.IsCancellationRequested);
+ }
+
+ /// <summary>
+ /// Overrides the base batch-publishing step: instead of routing rented messages through the base
+ /// class channel, deserializes the entire batch immediately (releasing all rented buffer refs via
+ /// <see cref="IDisposable.Dispose" />) and writes the deserialized results to
+ /// <see cref="_deserializedChannel" />. Auto-commit is also handled here since the base-class
+ /// yield path is bypassed.
+ /// </summary>
+ protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message,
+ uint partitionId, MessageStatus status,
+ Exception? error, CancellationToken ct)
+ {
+ T? data = default;
+ var deserError = status != MessageStatus.Success ? error : null;
+ var msgStatus = status;
+
+ if (status == MessageStatus.Success)
+ {
+ try
+ {
+ data = Deserialize(message.Payload);
+ }
+ catch (Exception ex)
+ {
+ _typedLogger.LogError(ex, "Failed to deserialize message at offset {Offset}",
+ message.Header.Offset);
+ msgStatus = MessageStatus.DeserializationFailed;
+ deserError = ex;
+ }
+ }
+
+ var deserialized = new DeserializedMessage<T>
+ {
+ Data = data,
+ Header = message.Header,
+ UserHeaders = message.UserHeaders,
+ CurrentOffset = message.Header.Offset,
+ PartitionId = partitionId,
+ Status = msgStatus,
+ Error = deserError
+ };
+
+ await _deserializedChannel.Writer.WriteAsync(deserialized, ct);
+
+ rental.Release();
+ }
+
+ /// <summary>
+ /// Deserializes a message payload from a span using the configured deserializer. Zero-copy when the
+ /// deserializer overrides the span overload; otherwise falls back to a one-time array copy.
+ /// </summary>
+ /// <param name="payload">The payload memory to deserialize.</param>
+ /// <returns>The deserialized object of type T.</returns>
+ public T Deserialize(ReadOnlyMemory<byte> payload)
+ {
+ return _typedConfig.Deserializer.Deserialize(payload);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
new file mode 100644
index 000000000..ac0f92e6c
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK/Consumers/ReceivedRentedMessage.cs
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Contracts;
+
+namespace Apache.Iggy.Consumers;
+
+/// <summary>
+/// Represents a message received from the Iggy consumer whose payload and raw headers are backed by rented memory
+/// shared across a polled batch. The caller SHOULD dispose the message once processing is complete for deterministic
+/// release of the underlying pool buffer. If the caller forgets, the buffer is still returned to the pool when the
+/// owning <see cref="PolledMessagesRental" /> becomes unreachable and its finalizer runs - non-deterministic but safe.
+/// </summary>
+public sealed class ReceivedRentedMessage : IDisposable
+{
+ private int _disposed;
+ internal RentedBatchHandle? Handle { get; init; }
+
+ /// <summary>
+ /// The underlying rented message response containing headers and rented payload memory.
+ /// </summary>
+ public required RentedMessageResponse Message { get; init; }
+
+ /// <summary>
+ /// The current offset of this message in the partition.
+ /// </summary>
+ public required ulong CurrentOffset { get; init; }
+
+ /// <summary>
+ /// The partition ID from which this message was consumed.
+ /// </summary>
+ public uint PartitionId { get; init; }
+
+ /// <summary>
+ /// The status of the message (Success, DecryptionFailed).
+ /// </summary>
+ public MessageStatus Status { get; init; } = MessageStatus.Success;
+
+ /// <summary>
+ /// The exception that occurred during processing, if any.
+ /// </summary>
+ public Exception? Error { get; init; }
+
+ /// <summary>
+ /// Releases this message's reference on the underlying rental. When the final message of a batch is disposed,
+ /// the rented buffer is returned to the pool and any payload/raw header slices are invalidated.
+ /// </summary>
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) == 1)
+ {
+ return;
+ }
+
+ Handle?.Release();
+ }
+}
+
+/// <summary>
+/// Reference-counted handle around a <see cref="PolledMessagesRental" />. Shared by all
+/// <see cref="ReceivedRentedMessage" /> instances produced from one poll; the rental is disposed
+/// when the reference count drops to zero.
+/// </summary>
+public sealed class RentedBatchHandle : IDisposable
+{
+ private readonly PolledMessagesRental _rental;
+ private int _refCount = 1;
+
+ /// <summary>
+ /// Creates a new handle with a single self-reference held by the constructing producer. The producer
+ /// must call <see cref="Acquire" /> before each publish and release the self-reference (via
+ /// <see cref="Dispose" /> or <see cref="Release" />) once it has finished producing.
+ /// </summary>
+ /// <param name="rental">The polled messages rental whose lifetime this handle manages.</param>
+ public RentedBatchHandle(PolledMessagesRental rental)
+ {
+ _rental = rental;
+ }
+
+ /// <summary>
+ /// Releases one reference on the underlying rental. Equivalent to <see cref="Release" />.
+ /// </summary>
+ public void Dispose()
+ {
+ Release();
+ }
+
+ /// <summary>
+ /// Acquires an additional reference. Must be balanced by a matching <see cref="Release" />.
+ /// </summary>
+ public void Acquire()
+ {
+ Interlocked.Increment(ref _refCount);
+ }
+
+ /// <summary>
+ /// Decrements the reference count. When the count reaches zero, the underlying rental is disposed
+ /// and its pool buffer returned.
+ /// </summary>
+ public void Release()
+ {
+ var remaining = Interlocked.Decrement(ref _refCount);
+ if (remaining == 0)
+ {
+ _rental.Dispose();
+ }
+ else if (remaining < 0)
+ {
+ throw new InvalidOperationException("RentedBatchHandle released more times than acquired.");
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
index 9a5eb55af..7895e7921 100644
--- a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
+++ b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs
@@ -1,52 +1,96 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Text.Json.Serialization;
-using Apache.Iggy.Headers;
-using Apache.Iggy.JsonConverters;
-using Apache.Iggy.Messages;
-
-namespace Apache.Iggy.Contracts;
-
-/// <summary>
-/// Response from the server containing a message payload.
-/// </summary>
-[JsonConverter(typeof(MessageResponseConverter))]
-public sealed class MessageResponse
-{
- /// <summary>
- /// Message header.
- /// </summary>
- public required MessageHeader Header { get; set; }
-
- /// <summary>
- /// Message payload.
- /// </summary>
- public required byte[] Payload { get; set; } = [];
-
- /// <summary>
- /// Headers defined by the user.
- /// </summary>
- public Dictionary<HeaderKey, HeaderValue>? UserHeaders { get; set; }
-
- /// <summary>
- /// Raw user header bytes before deserialization.
- /// Used internally for decrypting encrypted headers.
- /// </summary>
- [JsonIgnore]
- internal byte[]? RawUserHeaders { get; set; }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text.Json.Serialization;
+using Apache.Iggy.Headers;
+using Apache.Iggy.JsonConverters;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.Contracts;
+
+/// <summary>
+/// Response from the server containing a message payload.
+/// </summary>
+[JsonConverter(typeof(MessageResponseConverter))]
+public sealed class MessageResponse
+{
+ private byte[]? _rawUserHeaders;
+ private Dictionary<HeaderKey, HeaderValue>? _userHeaders;
+ private bool _userHeadersInitialized;
+
+ /// <summary>
+ /// Message header.
+ /// </summary>
+ public required MessageHeader Header { get; set; }
+
+ /// <summary>
+ /// Message payload.
+ /// </summary>
+ public required byte[] Payload { get; set; } = [];
+
+ /// <summary>
+ /// Headers defined by the user.
+ /// </summary>
+ public Dictionary<HeaderKey, HeaderValue>? UserHeaders
+ {
+ get
+ {
+ if (!_userHeadersInitialized)
+ {
+ _userHeaders = _rawUserHeaders is { Length: > 0 }
+ ? BinaryMapper.TryMapHeaders(_rawUserHeaders)
+ : null;
+ _userHeadersInitialized = true;
+ }
+
+ return _userHeaders;
+ }
+ set
+ {
+ _userHeaders = value;
+ _userHeadersInitialized = true;
+ }
+ }
+
+ /// <summary>
+ /// Raw user header bytes before deserialization.
+ /// Used internally for decrypting encrypted headers.
+ /// </summary>
+ [JsonIgnore]
+ internal byte[]? RawUserHeaders
+ {
+ get => _rawUserHeaders;
+ set
+ {
+ _rawUserHeaders = value;
+ _userHeaders = null;
+ _userHeadersInitialized = false;
+ }
+ }
+
+ internal void ParseUserHeaders()
+ {
+ if (!_userHeadersInitialized)
+ {
+ _userHeaders = _rawUserHeaders is { Length: > 0 }
+ ? BinaryMapper.TryMapHeaders(_rawUserHeaders)
+ : null;
+ _userHeadersInitialized = true;
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
new file mode 100644
index 000000000..04a82834b
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK/Contracts/PolledMessagesRental.cs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Buffers;
+
+namespace Apache.Iggy.Contracts;
+
+/// <summary>
+/// Represents a rented poll result whose payload and raw header memory remain valid until disposed.
+/// </summary>
+public sealed class PolledMessagesRental : IDisposable
+{
+ private readonly IMemoryOwner<byte> _owner;
+ private int _disposed;
+
+ /// <summary>
+ /// Partition identifier for the messages.
+ /// </summary>
+ public required int PartitionId { get; init; }
+
+ /// <summary>
+ /// Current offset for the partition.
+ /// </summary>
+ public required ulong CurrentOffset { get; init; }
+
+ /// <summary>
+ /// Rented messages.
+ /// </summary>
+ public required IReadOnlyList<RentedMessageResponse> Messages { get; init; }
+
+ internal PolledMessagesRental(IMemoryOwner<byte> owner)
+ {
+ _owner = owner;
+ }
+
+ /// <summary>
+ /// Disposes the rental and returns the underlying buffer to the pool.
+ /// </summary>
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) != 0)
+ {
+ return;
+ }
+
+ _owner.Dispose();
+ GC.SuppressFinalize(this);
+ }
+
+ /// <summary>
+ /// Finalizer fallback that returns the buffer to the pool if the caller forgot to dispose.
+ /// </summary>
+ ~PolledMessagesRental()
+ {
+ Dispose();
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs
new file mode 100644
index 000000000..43c629380
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK/Contracts/RentedMessageResponse.cs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Headers;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.Contracts;
+
+/// <summary>
+/// Response containing rented payload and raw header memory.
+/// The payload and raw headers are only valid while the owning <see cref="PolledMessagesRental" /> is alive.
+/// </summary>
+public sealed class RentedMessageResponse
+{
+ private Dictionary<HeaderKey, HeaderValue>? _userHeaders;
+ private bool _userHeadersInitialized;
+
+ /// <summary>
+ /// Message header.
+ /// </summary>
+ public required MessageHeader Header { get; init; }
+
+ /// <summary>
+ /// Message payload backed by rented memory.
+ /// </summary>
+ public required ReadOnlyMemory<byte> Payload { get; init; }
+
+ /// <summary>
+ /// Raw user header bytes backed by rented memory.
+ /// </summary>
+ public ReadOnlyMemory<byte> RawUserHeaders { get; init; }
+
+ /// <summary>
+ /// Parsed user headers. Parsed lazily and cached on first access.
+ /// </summary>
+ public Dictionary<HeaderKey, HeaderValue>? UserHeaders
+ {
+ get
+ {
+ if (!_userHeadersInitialized)
+ {
+ _userHeaders = RawUserHeaders.IsEmpty
+ ? null
+ : BinaryMapper.TryMapHeaders(RawUserHeaders.Span);
+ _userHeadersInitialized = true;
+ }
+
+ return _userHeaders;
+ }
+ init
+ {
+ _userHeaders = value;
+ _userHeadersInitialized = true;
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
index 8bd8ec2a6..6aaf4d822 100644
--- a/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
+++ b/foreign/csharp/Iggy_SDK/Encryption/AesMessageEncryptor.cs
@@ -1,128 +1,129 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Security.Cryptography;
-
-namespace Apache.Iggy.Encryption;
-
-/// <summary>
-/// AES-256-GCM based message encryptor for secure message encryption/decryption.
-/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity.
-/// </summary>
-public sealed class AesMessageEncryptor : IMessageEncryptor
-{
- private readonly byte[] _key;
- private const int NonceSize = 12; // 96 bits - recommended for GCM
- private const int TagSize = 16; // 128 bits authentication tag
-
- /// <summary>
- /// Creates a new AES message encryptor with the specified key.
- /// </summary>
- /// <param name="key">The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively.</param>
- /// <exception cref="ArgumentException">Thrown when key length is invalid</exception>
- public AesMessageEncryptor(byte[] key)
- {
- if (key.Length != 16 && key.Length != 24 && key.Length != 32)
- {
- throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256", nameof(key));
- }
-
- _key = key;
- }
-
- /// <summary>
- /// Creates a new AES-256 message encryptor with the specified base64-encoded key.
- /// </summary>
- /// <param name="base64Key">The base64-encoded encryption key</param>
- /// <returns>A new AesMessageEncryptor instance</returns>
- public static AesMessageEncryptor FromBase64Key(string base64Key)
- {
- return new AesMessageEncryptor(Convert.FromBase64String(base64Key));
- }
-
- /// <summary>
- /// Generates a new random AES-256 key.
- /// </summary>
- /// <returns>A 32-byte random key suitable for AES-256</returns>
- public static byte[] GenerateKey()
- {
- var key = new byte[32];
- using var rng = RandomNumberGenerator.Create();
- rng.GetBytes(key);
- return key;
- }
-
- /// <summary>
- /// Encrypts the provided plain data using AES-GCM.
- /// Format: [12-byte nonce][encrypted data][16-byte authentication tag]
- /// </summary>
- /// <param name="plainData">The plain data to encrypt</param>
- /// <returns>The encrypted data with nonce and tag</returns>
- public byte[] Encrypt(byte[] plainData)
- {
- using var aesGcm = new AesGcm(_key, TagSize);
-
- var nonce = new byte[NonceSize];
- RandomNumberGenerator.Fill(nonce);
-
- var ciphertext = new byte[plainData.Length];
- var tag = new byte[TagSize];
-
- aesGcm.Encrypt(nonce, plainData, ciphertext, tag);
-
- // Combine: nonce + ciphertext + tag
- var result = new byte[NonceSize + ciphertext.Length + TagSize];
- Buffer.BlockCopy(nonce, 0, result, 0, NonceSize);
- Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length);
- Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize);
-
- return result;
- }
-
- /// <summary>
- /// Decrypts the provided encrypted data using AES-GCM.
- /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag]
- /// </summary>
- /// <param name="encryptedData">The encrypted data with nonce and tag</param>
- /// <returns>The decrypted plain data</returns>
- /// <exception cref="ArgumentException">Thrown when encrypted data format is invalid</exception>
- /// <exception cref="CryptographicException">Thrown when decryption or authentication fails</exception>
- public byte[] Decrypt(byte[] encryptedData)
- {
- if (encryptedData.Length < NonceSize + TagSize)
- {
- throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData));
- }
-
- using var aesGcm = new AesGcm(_key, TagSize);
-
- // Extract nonce, ciphertext, and tag
- var nonce = new byte[NonceSize];
- var tag = new byte[TagSize];
- var ciphertextLength = encryptedData.Length - NonceSize - TagSize;
- var ciphertext = new byte[ciphertextLength];
-
- Buffer.BlockCopy(encryptedData, 0, nonce, 0, NonceSize);
- Buffer.BlockCopy(encryptedData, NonceSize, ciphertext, 0, ciphertextLength);
- Buffer.BlockCopy(encryptedData, NonceSize + ciphertextLength, tag, 0, TagSize);
-
- var plaintext = new byte[ciphertextLength];
- aesGcm.Decrypt(nonce, ciphertext, tag, plaintext);
-
- return plaintext;
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Security.Cryptography;
+
+namespace Apache.Iggy.Encryption;
+
+/// <summary>
+/// AES-256-GCM based message encryptor for secure message encryption/decryption.
+/// Uses AES-GCM (Galois/Counter Mode) which provides both confidentiality and authenticity.
+/// </summary>
+public sealed class AesMessageEncryptor : IMessageEncryptor
+{
+ private const int NonceSize = 12; // 96 bits - recommended for GCM
+ private const int TagSize = 16; // 128 bits authentication tag
+ private readonly byte[] _key;
+
+ /// <summary>
+ /// Creates a new AES message encryptor with the specified key.
+ /// </summary>
+ /// <param name="key">The encryption key. Must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively.</param>
+ /// <exception cref="ArgumentException">Thrown when key length is invalid</exception>
+ public AesMessageEncryptor(byte[] key)
+ {
+ if (key.Length != 16 && key.Length != 24 && key.Length != 32)
+ {
+ throw new ArgumentException("Key must be 16, 24, or 32 bytes for AES-128, AES-192, or AES-256",
+ nameof(key));
+ }
+
+ _key = key;
+ }
+
+ /// <summary>
+ /// Encrypts the provided plain data using AES-GCM.
+ /// Format: [12-byte nonce][encrypted data][16-byte authentication tag]
+ /// </summary>
+ /// <param name="plainData">The plain data to encrypt</param>
+ /// <returns>The encrypted data with nonce and tag</returns>
+ public byte[] Encrypt(Span<byte> plainData)
+ {
+ using var aesGcm = new AesGcm(_key, TagSize);
+
+ var nonce = new byte[NonceSize];
+ RandomNumberGenerator.Fill(nonce);
+
+ var ciphertext = new byte[plainData.Length];
+ var tag = new byte[TagSize];
+
+ aesGcm.Encrypt(nonce, plainData, ciphertext, tag);
+
+ // Combine: nonce + ciphertext + tag
+ var result = new byte[NonceSize + ciphertext.Length + TagSize];
+ Buffer.BlockCopy(nonce, 0, result, 0, NonceSize);
+ Buffer.BlockCopy(ciphertext, 0, result, NonceSize, ciphertext.Length);
+ Buffer.BlockCopy(tag, 0, result, NonceSize + ciphertext.Length, TagSize);
+
+ return result;
+ }
+
+ /// <summary>
+ /// Decrypts the provided encrypted data using AES-GCM.
+ /// Expected format: [12-byte nonce][encrypted data][16-byte authentication tag]
+ /// </summary>
+ /// <param name="encryptedData">The encrypted data with nonce and tag</param>
+ /// <returns>The decrypted plain data</returns>
+ /// <exception cref="ArgumentException">Thrown when encrypted data format is invalid</exception>
+ /// <exception cref="CryptographicException">Thrown when decryption or authentication fails</exception>
+ public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
+ {
+ if (encryptedData.Length < NonceSize + TagSize)
+ {
+ throw new ArgumentException("Encrypted data is too short to contain nonce and tag", nameof(encryptedData));
+ }
+
+ using var aesGcm = new AesGcm(_key, TagSize);
+
+ // Extract nonce, ciphertext, and tag
+ Span<byte> nonce = stackalloc byte[NonceSize];
+ Span<byte> tag = stackalloc byte[TagSize];
+ var ciphertextLength = encryptedData.Length - NonceSize - TagSize;
+ Span<byte> ciphertext = stackalloc byte[ciphertextLength];
+
+ encryptedData[..NonceSize].CopyTo(nonce);
+ encryptedData.Slice(NonceSize, ciphertextLength).CopyTo(ciphertext);
+ encryptedData.Slice(NonceSize + ciphertextLength, TagSize).CopyTo(tag);
+
+ var plaintext = new byte[ciphertextLength];
+ aesGcm.Decrypt(nonce, ciphertext, tag, plaintext);
+
+ return plaintext;
+ }
+
+ /// <summary>
+ /// Creates a new AES-256 message encryptor with the specified base64-encoded key.
+ /// </summary>
+ /// <param name="base64Key">The base64-encoded encryption key</param>
+ /// <returns>A new AesMessageEncryptor instance</returns>
+ public static AesMessageEncryptor FromBase64Key(string base64Key)
+ {
+ return new AesMessageEncryptor(Convert.FromBase64String(base64Key));
+ }
+
+ /// <summary>
+ /// Generates a new random AES-256 key.
+ /// </summary>
+ /// <returns>A 32-byte random key suitable for AES-256</returns>
+ public static byte[] GenerateKey()
+ {
+ var key = new byte[32];
+ using var rng = RandomNumberGenerator.Create();
+ rng.GetBytes(key);
+ return key;
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
index fbb99c626..f6111f89d 100644
--- a/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
+++ b/foreign/csharp/Iggy_SDK/Encryption/IMessageEncryptor.cs
@@ -1,40 +1,40 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-namespace Apache.Iggy.Encryption;
-
-/// <summary>
-/// Interface for encrypting and decrypting message payloads in Iggy messaging system.
-/// Implementations of this interface can be used with IggyPublisher and IggyConsumer
-/// to provide end-to-end encryption of message data.
-/// </summary>
-public interface IMessageEncryptor
-{
- /// <summary>
- /// Encrypts the provided plain data.
- /// </summary>
- /// <param name="plainData">The plain data to encrypt</param>
- /// <returns>The encrypted data</returns>
- byte[] Encrypt(byte[] plainData);
-
- /// <summary>
- /// Decrypts the provided encrypted data.
- /// </summary>
- /// <param name="encryptedData">The encrypted data to decrypt</param>
- /// <returns>The decrypted plain data</returns>
- byte[] Decrypt(byte[] encryptedData);
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.Iggy.Encryption;
+
+/// <summary>
+/// Interface for encrypting and decrypting message payloads in Iggy messaging system.
+/// Implementations of this interface can be used with IggyPublisher and IggyConsumer
+/// to provide end-to-end encryption of message data.
+/// </summary>
+public interface IMessageEncryptor
+{
+ /// <summary>
+ /// Encrypts the provided plain data.
+ /// </summary>
+ /// <param name="plainData">The plain data to encrypt</param>
+ /// <returns>The encrypted data</returns>
+ byte[] Encrypt(Span<byte> plainData);
+
+ /// <summary>
+ /// Decrypts the provided encrypted data.
+ /// </summary>
+ /// <param name="encryptedData">The encrypted data to decrypt</param>
+ /// <returns>The decrypted plain data</returns>
+ byte[] Decrypt(ReadOnlySpan<byte> encryptedData);
+}
diff --git a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs
index 82d1a8139..c585e1fb5 100644
--- a/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs
+++ b/foreign/csharp/Iggy_SDK/Extensions/IggyClientExtension.cs
@@ -1,74 +1,74 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Consumers;
-using Apache.Iggy.IggyClient;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Publishers;
-
-namespace Apache.Iggy.Extensions;
-
-/// <summary>
-/// Extension methods for <see cref="IIggyClient" />
-/// </summary>
-public static class IggyClientExtension
-{
- /// <summary>
- /// Creates a new <see cref="IggyConsumerBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
- /// topic.
- /// </summary>
- /// <param name="client">Existing iggy client</param>
- /// <param name="streamId">Stream identifier from which to consume</param>
- /// <param name="topicId">Topic identifier from which to consume</param>
- /// <param name="consumer">Consumer</param>
- /// <returns>Iggy consumer builder</returns>
- public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId,
- Identifier topicId, Consumer consumer)
- {
- return IggyConsumerBuilder.Create(client, streamId, topicId, consumer);
- }
-
- /// <summary>
- /// Creates a new <see cref="IggyConsumerBuilder{T}" /> from <see cref="IIggyClient" /> for the specified stream and
- /// topic.
- /// </summary>
- /// <param name="client">Existing iggy client</param>
- /// <param name="streamId">Stream identifier from which to consume</param>
- /// <param name="topicId">Topic identifier from which to consume</param>
- /// <param name="consumer">Consumer</param>
- /// <param name="deserializer">Optional deserializer</param>
- /// <returns>Iggy consumer builder</returns>
- public static IggyConsumerBuilder CreateConsumerBuilder<T>(this IIggyClient client, Identifier streamId,
- Identifier topicId, Consumer consumer, IDeserializer<T> deserializer) where T : IDeserializer<T>
- {
- return IggyConsumerBuilder.Create(client, streamId, topicId, consumer);
- }
-
- /// <summary>
- /// Creates a new <see cref="IggyPublisherBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
- /// topic.
- /// </summary>
- /// <param name="client">Existing iggy client</param>
- /// <param name="streamId">Stream identifier to publish to</param>
- /// <param name="topicId">>Topic identifier to publish to</param>
- /// <returns>Iggy publisher builder</returns>
- public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId,
- Identifier topicId)
- {
- return IggyPublisherBuilder.Create(client, streamId, topicId);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Consumers;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Publishers;
+
+namespace Apache.Iggy.Extensions;
+
+/// <summary>
+/// Extension methods for <see cref="IIggyClient" />
+/// </summary>
+public static class IggyClientExtension
+{
+ /// <summary>
+ /// Creates a new <see cref="IggyConsumerBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
+ /// topic.
+ /// </summary>
+ /// <param name="client">Existing iggy client</param>
+ /// <param name="streamId">Stream identifier from which to consume</param>
+ /// <param name="topicId">Topic identifier from which to consume</param>
+ /// <param name="consumer">Consumer</param>
+ /// <returns>Iggy consumer builder</returns>
+ public static IggyConsumerBuilder CreateConsumerBuilder(this IIggyClient client, Identifier streamId,
+ Identifier topicId, Consumer consumer)
+ {
+ return IggyConsumerBuilder.Create(client, streamId, topicId, consumer);
+ }
+
+ /// <summary>
+ /// Creates a new <see cref="IggyConsumerBuilder{T}" /> from <see cref="IIggyClient" /> for the specified stream and
+ /// topic.
+ /// </summary>
+ /// <param name="client">Existing iggy client</param>
+ /// <param name="streamId">Stream identifier from which to consume</param>
+ /// <param name="topicId">Topic identifier from which to consume</param>
+ /// <param name="consumer">Consumer</param>
+ /// <param name="deserializer">Optional deserializer</param>
+ /// <returns>Iggy consumer builder</returns>
+ public static IggyConsumerBuilder<T> CreateConsumerBuilder<T>(this IIggyClient client, Identifier streamId,
+ Identifier topicId, Consumer consumer, IDeserializer<T> deserializer)
+ {
+ return IggyConsumerBuilder<T>.Create(client, streamId, topicId, consumer, deserializer);
+ }
+
+ /// <summary>
+ /// Creates a new <see cref="IggyPublisherBuilder" /> from <see cref="IIggyClient" /> for the specified stream and
+ /// topic.
+ /// </summary>
+ /// <param name="client">Existing iggy client</param>
+ /// <param name="streamId">Stream identifier to publish to</param>
+ /// <param name="topicId">>Topic identifier to publish to</param>
+ /// <returns>Iggy publisher builder</returns>
+ public static IggyPublisherBuilder CreatePublisherBuilder(this IIggyClient client, Identifier streamId,
+ Identifier topicId)
+ {
+ return IggyPublisherBuilder.Create(client, streamId, topicId);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs
index 476b09b1b..1898bdbd3 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyConsumer.cs
@@ -1,63 +1,85 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Kinds;
-
-namespace Apache.Iggy.IggyClient;
-
-/// <summary>
-/// Defines methods for consuming messages from topics in an Iggy client.
-/// </summary>
-public interface IIggyConsumer
-{
- /// <summary>
- /// Polls messages from a specified topic and partition with a given polling strategy.
- /// </summary>
- /// <remarks>
- /// This method retrieves messages from a topic based on the specified consumer and polling strategy.
- /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest).
- /// If a partition ID is not specified, messages can be consumed from any partition.
- /// </remarks>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic to consume from (numeric ID or name).</param>
- /// <param name="partitionId">The specific partition to consume from, or null to consume from any partition.</param>
- /// <param name="consumer">The consumer identifier (group ID or member ID).</param>
- /// <param name="pollingStrategy">The strategy for determining where to start reading messages.</param>
- /// <param name="count">The maximum number of messages to retrieve.</param>
- /// <param name="autoCommit">If true, automatically commit the offset after polling.</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
- Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
- Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit,
- CancellationToken token = default);
-
- /// <summary>
- /// Polls messages from a specified topic using a pre-constructed request.
- /// </summary>
- /// <remarks>
- /// This is a convenience method that wraps the full PollMessagesAsync method using a request object.
- /// </remarks>
- /// <param name="request">The message fetch request containing all polling parameters.</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
- Task<PolledMessages> PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default)
- {
- return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer,
- request.PollingStrategy, request.Count, request.AutoCommit, token);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Kinds;
+
+namespace Apache.Iggy.IggyClient;
+
+/// <summary>
+/// Defines methods for consuming messages from topics in an Iggy client.
+/// </summary>
+public interface IIggyConsumer
+{
+ /// <summary>
+ /// Polls messages from a specified topic and partition with a given polling strategy.
+ /// </summary>
+ /// <remarks>
+ /// This method retrieves messages from a topic based on the specified consumer and polling strategy.
+ /// The polling strategy determines where to start reading messages (e.g., from a specific offset, latest, earliest).
+ /// If a partition ID is not specified, messages can be consumed from any partition.
+ /// </remarks>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic to consume from (numeric ID or name).</param>
+ /// <param name="partitionId">The specific partition to consume from, or null to consume from any partition.</param>
+ /// <param name="consumer">The consumer identifier (group ID or member ID).</param>
+ /// <param name="pollingStrategy">The strategy for determining where to start reading messages.</param>
+ /// <param name="count">The maximum number of messages to retrieve.</param>
+ /// <param name="autoCommit">If true, automatically commit the offset after polling.</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
+ Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit,
+ CancellationToken token = default);
+
+ /// <summary>
+ /// Polls messages from a specified topic and partition while renting the payload buffers from a shared pool
+ /// instead of copying them into byte arrays.
+ /// </summary>
+ /// <remarks>
+ /// The returned rental must be disposed when the caller is done reading the payload and raw header memory.
+ /// Payload and raw header slices are invalidated once the rental is disposed.
+ /// </remarks>
+ Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit,
+ CancellationToken token = default);
+
+ /// <summary>
+ /// Polls messages from a specified topic using a pre-constructed request.
+ /// </summary>
+ /// <remarks>
+ /// This is a convenience method that wraps the full PollMessagesAsync method using a request object.
+ /// </remarks>
+ /// <param name="request">The message fetch request containing all polling parameters.</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task that represents the asynchronous operation and returns the polled messages.</returns>
+ Task<PolledMessages> PollMessagesAsync(MessageFetchRequest request, CancellationToken token = default)
+ {
+ return PollMessagesAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer,
+ request.PollingStrategy, request.Count, request.AutoCommit, token);
+ }
+
+ /// <summary>
+ /// Polls messages from a specified topic using a pre-constructed request while renting the payload buffers
+ /// from a shared pool.
+ /// </summary>
+ Task<PolledMessagesRental> PollMessagesRentedAsync(MessageFetchRequest request, CancellationToken token = default)
+ {
+ return PollMessagesRentedAsync(request.StreamId, request.TopicId, request.PartitionId, request.Consumer,
+ request.PollingStrategy, request.Count, request.AutoCommit, token);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
index fac18e1de..e7e7e77a7 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs
@@ -1,850 +1,862 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Net;
-using System.Net.Http.Headers;
-using System.Net.Http.Json;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Contracts.Auth;
-using Apache.Iggy.Contracts.Http;
-using Apache.Iggy.Contracts.Http.Auth;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Messages;
-using Apache.Iggy.StringHandlers;
-using Apache.Iggy.Utils;
-using Partitioning = Apache.Iggy.Kinds.Partitioning;
-
-namespace Apache.Iggy.IggyClient.Implementations;
-
-/// <summary>
-/// Implementation of <see cref="IIggyClient" /> that uses <see cref="HttpClient" /> to communicate with the server.
-/// </summary>
-public class HttpMessageStream : IIggyClient
-{
- private const string Context = "csharp-sdk";
-
- private readonly HttpClient _httpClient;
-
- //TODO - create mechanism for refreshing jwt token
- //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI
- //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method
- private readonly JsonSerializerOptions _jsonSerializerOptions;
-
- internal HttpMessageStream(HttpClient httpClient)
- {
- _httpClient = httpClient;
-
- _jsonSerializerOptions = new JsonSerializerOptions
- {
- PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
- Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) }
- };
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync("/streams", data, token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/streams", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<StreamResponse>>(_jsonSerializerOptions,
- token)
- ?? Array.Empty<StreamResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<StreamResponse>();
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
- TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateTopicRequest
- {
- Name = name,
- CompressionAlgorithm = compressionAlgorithm,
- MaxTopicSize = maxTopicSize,
- MessageExpiry = DurationHelpers.ToDuration(messageExpiry),
- PartitionsCount = partitionsCount,
- ReplicationFactor = replicationFactor
- }, _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
- ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize,
- DurationHelpers.ToDuration(messageExpiry),
- replicationFactor),
- _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token)
- .ContinueWith(async response =>
- {
- if (!response.Result.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response.Result);
- }
- }, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
- CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<TopicResponse>>(_jsonSerializerOptions, token)
- ?? Array.Empty<TopicResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<TopicResponse>();
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
- CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
- IList<Message> messages,
- CancellationToken token = default)
- {
- var request = new MessageSendRequest
- {
- StreamId = streamId,
- TopicId = topicId,
- Partitioning = partitioning,
- Messages = messages
- };
- var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages",
- data,
- token);
-
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
- CancellationToken token = default)
- {
- var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}");
-
- var response = await _httpClient.GetAsync(url, token);
-
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response, true);
- }
- }
-
- /// <inheritdoc />
- public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
- Consumer consumer,
- PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
- {
- var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
- var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" +
- $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}");
-
- var response = await _httpClient.GetAsync(url, token);
- if (response.IsSuccessStatusCode)
- {
- var pollMessages = await response.Content.ReadFromJsonAsync<PolledMessages>(_jsonSerializerOptions, token)
- ?? PolledMessages.Empty;
-
- return pollMessages;
- }
-
- await HandleResponseAsync(response, true);
- return PolledMessages.Empty;
- }
-
- /// <inheritdoc />
- public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
- uint? partitionId, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset),
- _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response
- = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
- uint? partitionId, CancellationToken token = default)
- {
- var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" +
- $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}",
- token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<OffsetResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
- CancellationToken token = default)
- {
- var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty;
- var response = await _httpClient.DeleteAsync(
- $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token);
- await HandleResponseAsync(response);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
- Identifier topicId, CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<ConsumerGroupResponse>>(
- _jsonSerializerOptions, token)
- ?? Array.Empty<ConsumerGroupResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<ConsumerGroupResponse>();
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
- Identifier groupId, CancellationToken token = default)
- {
- var response
- = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
- string name, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response
- = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var response
- = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
- await HandleResponseAsync(response);
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="token"></param>
- /// <returns></returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- public Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <inheritdoc />
- public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/stats", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<StatsResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/cluster/metadata", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ClusterMetadata>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <inheritdoc />
- public async Task PingAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/ping", token);
-
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
- IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
- {
- // Rust serde uses default derive (PascalCase) for these enums, not snake_case.
- // We use .ToString() to produce PascalCase names matching Rust's serde expectations.
- var request = new
- {
- compression = compression.ToString(),
- snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList()
- };
- var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync("/snapshot", data, token);
-
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadAsByteArrayAsync(token);
- }
-
- await HandleResponseAsync(response);
- return [];
- }
-
- /// <inheritdoc />
- public Task ConnectAsync(CancellationToken token = default)
- {
- return Task.CompletedTask;
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/clients", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<ClientResponse>>(_jsonSerializerOptions,
- token)
- ?? Array.Empty<ClientResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<ClientResponse>();
- }
-
- /// <inheritdoc />
- public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync($"/clients/{clientId}", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<ClientResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
- /// <param name="groupId">The identifier of the consumer group to join (numeric ID or name).</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task representing the asynchronous operation.</returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- [Obsolete("This method is only supported in TCP protocol", true)]
- public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
- /// <param name="groupId">The identifier of the consumer group to leave (numeric ID or name).</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task representing the asynchronous operation.</returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- [Obsolete("This method is only supported in TCP protocol", true)]
- public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <inheritdoc />
- public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var response
- = await _httpClient.DeleteAsync(
- $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <summary>
- /// This method is only supported in TCP protocol
- /// </summary>
- /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
- /// <param name="topicId">The identifier of the topic containing the partition (numeric ID or name).</param>
- /// <param name="partitionId">The unique partition ID.</param>
- /// <param name="segmentsCount">The number of segments to delete.</param>
- /// <param name="token">The cancellation token to cancel the operation.</param>
- /// <returns>A task representing the asynchronous operation.</returns>
- /// <exception cref="FeatureUnavailableException"></exception>
- public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
- uint segmentsCount, CancellationToken token = default)
- {
- throw new FeatureUnavailableException();
- }
-
- /// <inheritdoc />
- public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
- {
- //TODO - this doesn't work prob needs a custom json serializer
- var response = await _httpClient.GetAsync($"/users/{userId}", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/users", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<UserResponse>>(_jsonSerializerOptions, token)
- ?? Array.Empty<UserResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<UserResponse>();
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
- Permissions? permissions = null, CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions),
- _jsonSerializerOptions);
-
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PostAsync("/users", content, token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/users/{userId}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions);
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/users/{userId}", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions);
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword),
- _jsonSerializerOptions);
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
- {
- // TODO: Add binary protocol version
- var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context),
- _jsonSerializerOptions);
-
- var data = new StringContent(json, Encoding.UTF8, "application/json");
-
- var response = await _httpClient.PostAsync("users/login", data, token);
- if (response.IsSuccessStatusCode)
- {
- var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, token);
- var jwtToken = authResponse!.AccessToken?.Token;
- if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
- {
- _httpClient.DefaultRequestHeaders.Authorization =
- new AuthenticationHeaderValue("Bearer", jwtToken);
- }
- else
- {
- throw new Exception("The JWT token is missing.");
- }
-
- return authResponse;
- }
-
- await HandleResponseAsync(response);
- return null;
- }
-
- /// <inheritdoc />
- public async Task LogoutUserAsync(CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync("users/logout", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
-
- _httpClient.DefaultRequestHeaders.Authorization = null;
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
- CancellationToken token = default)
- {
- var response = await _httpClient.GetAsync("/personal-access-tokens", token);
- if (response.IsSuccessStatusCode)
- {
- return await response.Content.ReadFromJsonAsync<IReadOnlyList<PersonalAccessTokenResponse>>(
- _jsonSerializerOptions, token)
- ?? Array.Empty<PersonalAccessTokenResponse>();
- }
-
- await HandleResponseAsync(response);
- return Array.Empty<PersonalAccessTokenResponse>();
- }
-
- /// <inheritdoc />
- public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
- CancellationToken token = default)
- {
- var json = JsonSerializer.Serialize(
- new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions);
-
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PostAsync("/personal-access-tokens", content, token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
-
- return await response.Content.ReadFromJsonAsync<RawPersonalAccessToken>(_jsonSerializerOptions, token);
- }
-
- /// <inheritdoc />
- public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
- {
- var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token);
- if (!response.IsSuccessStatusCode)
- {
- await HandleResponseAsync(response);
- }
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
- {
- var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions);
-
- var content = new StringContent(json, Encoding.UTF8, "application/json");
- var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct);
- if (response.IsSuccessStatusCode)
- {
- var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, ct);
- var jwtToken = authResponse!.AccessToken?.Token;
- if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
- {
- _httpClient.DefaultRequestHeaders.Authorization =
- new AuthenticationHeaderValue("Bearer", jwtToken);
- }
- else
- {
- throw new Exception("The JWT token is missing.");
- }
-
- return authResponse;
- }
-
- await HandleResponseAsync(response);
-
- return null;
- }
-
- /// <summary>
- /// Dispose the client.
- /// </summary>
- public void Dispose()
- {
- }
-
- /// <inheritdoc />
- public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- }
-
- /// <inheritdoc />
- public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- }
-
- /// <inheritdoc />
- public string GetCurrentAddress()
- {
- return _httpClient.BaseAddress?.ToString() ?? string.Empty;
- }
-
- private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false)
- {
- if ((int)response.StatusCode > 300
- && (int)response.StatusCode < 500
- && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound &&
- !shouldThrowOnGetNotFound))
- {
- var err = await response.Content.ReadAsStringAsync();
- var errorModel = JsonSerializer.Deserialize<ErrorResponse>(err);
- throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err);
- }
-
- if (response.StatusCode == HttpStatusCode.InternalServerError)
- {
- throw new Exception("Internal server error");
- }
- }
-
- private static string CreateUrl(ref MessageRequestInterpolationHandler message)
- {
- return message.ToString();
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Net;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Contracts.Auth;
+using Apache.Iggy.Contracts.Http;
+using Apache.Iggy.Contracts.Http.Auth;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+using Apache.Iggy.StringHandlers;
+using Apache.Iggy.Utils;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.IggyClient.Implementations;
+
+/// <summary>
+/// Implementation of <see cref="IIggyClient" /> that uses <see cref="HttpClient" /> to communicate with the server.
+/// </summary>
+public class HttpMessageStream : IIggyClient
+{
+ private const string Context = "csharp-sdk";
+
+ private readonly HttpClient _httpClient;
+
+ //TODO - create mechanism for refreshing jwt token
+ //TODO - replace the HttpClient with IHttpClientFactory, when implementing support for ASP.NET Core DI
+ //TODO - the error handling pattern is pretty ugly, look into moving it into an extension method
+ private readonly JsonSerializerOptions _jsonSerializerOptions;
+
+ internal HttpMessageStream(HttpClient httpClient)
+ {
+ _httpClient = httpClient;
+
+ _jsonSerializerOptions = new JsonSerializerOptions
+ {
+ PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+ Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) }
+ };
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateStreamRequest(name), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync("/streams", data, token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/streams/{streamId}/purge", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/streams/{streamId}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<StreamResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateStreamRequest(name), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/streams/{streamId}", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/streams", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<StreamResponse>>(_jsonSerializerOptions,
+ token)
+ ?? Array.Empty<StreamResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<StreamResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
+ TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateTopicRequest
+ {
+ Name = name,
+ CompressionAlgorithm = compressionAlgorithm,
+ MaxTopicSize = maxTopicSize,
+ MessageExpiry = DurationHelpers.ToDuration(messageExpiry),
+ PartitionsCount = partitionsCount,
+ ReplicationFactor = replicationFactor
+ }, _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync($"/streams/{streamId}/topics", data, token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
+ ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize,
+ DurationHelpers.ToDuration(messageExpiry),
+ replicationFactor),
+ _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ return _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/purge", token)
+ .ContinueWith(async response =>
+ {
+ if (!response.Result.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response.Result);
+ }
+ }, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
+ CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<TopicResponse>>(_jsonSerializerOptions, token)
+ ?? Array.Empty<TopicResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<TopicResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
+ CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<TopicResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
+ IList<Message> messages,
+ CancellationToken token = default)
+ {
+ var request = new MessageSendRequest
+ {
+ StreamId = streamId,
+ TopicId = topicId,
+ Partitioning = partitioning,
+ Messages = messages
+ };
+ var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync($"/streams/{request.StreamId}/topics/{request.TopicId}/messages",
+ data,
+ token);
+
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
+ CancellationToken token = default)
+ {
+ var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}");
+
+ var response = await _httpClient.GetAsync(url, token);
+
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response, true);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
+ var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages?consumer_id={consumer.ConsumerId}" +
+ $"{partitionIdParam}&kind={pollingStrategy.Kind}&value={pollingStrategy.Value}&count={count}&auto_commit={autoCommit}");
+
+ var response = await _httpClient.GetAsync(url, token);
+ if (response.IsSuccessStatusCode)
+ {
+ var pollMessages = await response.Content.ReadFromJsonAsync<PolledMessages>(_jsonSerializerOptions, token)
+ ?? PolledMessages.Empty;
+
+ return pollMessages;
+ }
+
+ await HandleResponseAsync(response, true);
+ return PolledMessages.Empty;
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId,
+ uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ var messages = await PollMessagesAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count,
+ autoCommit, token);
+ return BinaryMapper.ToRentedMessages(messages);
+ }
+
+ /// <inheritdoc />
+ public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new StoreOffsetRequest(consumer, partitionId, offset),
+ _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response
+ = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}/consumer-offsets", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var partitionIdParam = partitionId.HasValue ? $"&partition_id={partitionId.Value}" : string.Empty;
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/" +
+ $"consumer-offsets?consumer_id={consumer.ConsumerId}{partitionIdParam}",
+ token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<OffsetResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
+ CancellationToken token = default)
+ {
+ var partitionIdParam = partitionId.HasValue ? $"?partition_id={partitionId.Value}" : string.Empty;
+ var response = await _httpClient.DeleteAsync(
+ $"/streams/{streamId}/topics/{topicId}/consumer-offsets/{consumer}{partitionIdParam}", token);
+ await HandleResponseAsync(response);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
+ Identifier topicId, CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<ConsumerGroupResponse>>(
+ _jsonSerializerOptions, token)
+ ?? Array.Empty<ConsumerGroupResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<ConsumerGroupResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
+ Identifier groupId, CancellationToken token = default)
+ {
+ var response
+ = await _httpClient.GetAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
+ string name, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateConsumerGroupRequest(name), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response
+ = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups", data, token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ConsumerGroupResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var response
+ = await _httpClient.DeleteAsync($"/streams/{streamId}/topics/{topicId}/consumer-groups/{groupId}", token);
+ await HandleResponseAsync(response);
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="token"></param>
+ /// <returns></returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ public Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <inheritdoc />
+ public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/stats", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<StatsResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/cluster/metadata", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ClusterMetadata>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task PingAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/ping", token);
+
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
+ IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
+ {
+ // Rust serde uses default derive (PascalCase) for these enums, not snake_case.
+ // We use .ToString() to produce PascalCase names matching Rust's serde expectations.
+ var request = new
+ {
+ compression = compression.ToString(),
+ snapshot_types = snapshotTypes.Select(t => t.ToString()).ToList()
+ };
+ var json = JsonSerializer.Serialize(request, _jsonSerializerOptions);
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync("/snapshot", data, token);
+
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadAsByteArrayAsync(token);
+ }
+
+ await HandleResponseAsync(response);
+ return [];
+ }
+
+ /// <inheritdoc />
+ public Task ConnectAsync(CancellationToken token = default)
+ {
+ return Task.CompletedTask;
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/clients", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<ClientResponse>>(_jsonSerializerOptions,
+ token)
+ ?? Array.Empty<ClientResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<ClientResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync($"/clients/{clientId}", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<ClientResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
+ /// <param name="groupId">The identifier of the consumer group to join (numeric ID or name).</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task representing the asynchronous operation.</returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ [Obsolete("This method is only supported in TCP protocol", true)]
+ public Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic (numeric ID or name).</param>
+ /// <param name="groupId">The identifier of the consumer group to leave (numeric ID or name).</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task representing the asynchronous operation.</returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ [Obsolete("This method is only supported in TCP protocol", true)]
+ public Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var response
+ = await _httpClient.DeleteAsync(
+ $"/streams/{streamId}/topics/{topicId}/partitions?partitions_count={partitionsCount}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <summary>
+ /// This method is only supported in TCP protocol
+ /// </summary>
+ /// <param name="streamId">The identifier of the stream containing the topic (numeric ID or name).</param>
+ /// <param name="topicId">The identifier of the topic containing the partition (numeric ID or name).</param>
+ /// <param name="partitionId">The unique partition ID.</param>
+ /// <param name="segmentsCount">The number of segments to delete.</param>
+ /// <param name="token">The cancellation token to cancel the operation.</param>
+ /// <returns>A task representing the asynchronous operation.</returns>
+ /// <exception cref="FeatureUnavailableException"></exception>
+ public Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
+ uint segmentsCount, CancellationToken token = default)
+ {
+ throw new FeatureUnavailableException();
+ }
+
+ /// <inheritdoc />
+ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreatePartitionsRequest(partitionsCount), _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync($"/streams/{streamId}/topics/{topicId}/partitions", data, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ //TODO - this doesn't work prob needs a custom json serializer
+ var response = await _httpClient.GetAsync($"/users/{userId}", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/users", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<UserResponse>>(_jsonSerializerOptions, token)
+ ?? Array.Empty<UserResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<UserResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
+ Permissions? permissions = null, CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new CreateUserRequest(userName, password, status, permissions),
+ _jsonSerializerOptions);
+
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PostAsync("/users", content, token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<UserResponse>(_jsonSerializerOptions, token);
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/users/{userId}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateUserRequest(userName, status), _jsonSerializerOptions);
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/users/{userId}", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new UpdateUserPermissionsRequest(permissions), _jsonSerializerOptions);
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/users/{userId}/permissions", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(new ChangePasswordRequest(currentPassword, newPassword),
+ _jsonSerializerOptions);
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PutAsync($"/users/{userId}/password", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
+ {
+ // TODO: Add binary protocol version
+ var json = JsonSerializer.Serialize(new LoginUserRequest(userName, password, SdkVersion.Value, Context),
+ _jsonSerializerOptions);
+
+ var data = new StringContent(json, Encoding.UTF8, "application/json");
+
+ var response = await _httpClient.PostAsync("users/login", data, token);
+ if (response.IsSuccessStatusCode)
+ {
+ var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, token);
+ var jwtToken = authResponse!.AccessToken?.Token;
+ if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
+ {
+ _httpClient.DefaultRequestHeaders.Authorization =
+ new AuthenticationHeaderValue("Bearer", jwtToken);
+ }
+ else
+ {
+ throw new Exception("The JWT token is missing.");
+ }
+
+ return authResponse;
+ }
+
+ await HandleResponseAsync(response);
+ return null;
+ }
+
+ /// <inheritdoc />
+ public async Task LogoutUserAsync(CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync("users/logout", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+
+ _httpClient.DefaultRequestHeaders.Authorization = null;
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
+ CancellationToken token = default)
+ {
+ var response = await _httpClient.GetAsync("/personal-access-tokens", token);
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadFromJsonAsync<IReadOnlyList<PersonalAccessTokenResponse>>(
+ _jsonSerializerOptions, token)
+ ?? Array.Empty<PersonalAccessTokenResponse>();
+ }
+
+ await HandleResponseAsync(response);
+ return Array.Empty<PersonalAccessTokenResponse>();
+ }
+
+ /// <inheritdoc />
+ public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
+ CancellationToken token = default)
+ {
+ var json = JsonSerializer.Serialize(
+ new CreatePersonalAccessTokenRequest(name, DurationHelpers.ToDuration(expiry)), _jsonSerializerOptions);
+
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PostAsync("/personal-access-tokens", content, token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+
+ return await response.Content.ReadFromJsonAsync<RawPersonalAccessToken>(_jsonSerializerOptions, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
+ {
+ var response = await _httpClient.DeleteAsync($"/personal-access-tokens/{name}", token);
+ if (!response.IsSuccessStatusCode)
+ {
+ await HandleResponseAsync(response);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
+ {
+ var json = JsonSerializer.Serialize(new LoginWithPersonalAccessTokenRequest(token), _jsonSerializerOptions);
+
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await _httpClient.PostAsync("/personal-access-tokens/login", content, ct);
+ if (response.IsSuccessStatusCode)
+ {
+ var authResponse = await response.Content.ReadFromJsonAsync<AuthResponse>(_jsonSerializerOptions, ct);
+ var jwtToken = authResponse!.AccessToken?.Token;
+ if (!string.IsNullOrEmpty(authResponse!.AccessToken!.Token))
+ {
+ _httpClient.DefaultRequestHeaders.Authorization =
+ new AuthenticationHeaderValue("Bearer", jwtToken);
+ }
+ else
+ {
+ throw new Exception("The JWT token is missing.");
+ }
+
+ return authResponse;
+ }
+
+ await HandleResponseAsync(response);
+
+ return null;
+ }
+
+ /// <summary>
+ /// Dispose the client.
+ /// </summary>
+ public void Dispose()
+ {
+ }
+
+ /// <inheritdoc />
+ public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ }
+
+ /// <inheritdoc />
+ public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ }
+
+ /// <inheritdoc />
+ public string GetCurrentAddress()
+ {
+ return _httpClient.BaseAddress?.ToString() ?? string.Empty;
+ }
+
+ private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false)
+ {
+ if ((int)response.StatusCode > 300
+ && (int)response.StatusCode < 500
+ && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound &&
+ !shouldThrowOnGetNotFound))
+ {
+ var err = await response.Content.ReadAsStringAsync();
+ var errorModel = JsonSerializer.Deserialize<ErrorResponse>(err);
+ throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err);
+ }
+
+ if (response.StatusCode == HttpStatusCode.InternalServerError)
+ {
+ throw new Exception("Internal server error");
+ }
+ }
+
+ private static string CreateUrl(ref MessageRequestInterpolationHandler message)
+ {
+ return message.ToString();
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
index ed8d42445..72d0d9b51 100644
--- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
+++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs
@@ -1,1239 +1,1317 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Buffers;
-using System.Buffers.Binary;
-using System.Net.Security;
-using System.Net.Sockets;
-using System.Security.Cryptography.X509Certificates;
-using System.Text;
-using Apache.Iggy.Configuration;
-using Apache.Iggy.ConnectionStream;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Contracts.Auth;
-using Apache.Iggy.Contracts.Tcp;
-using Apache.Iggy.Enums;
-using Apache.Iggy.Exceptions;
-using Apache.Iggy.Kinds;
-using Apache.Iggy.Mappers;
-using Apache.Iggy.Messages;
-using Apache.Iggy.Utils;
-using Microsoft.Extensions.Logging;
-using Partitioning = Apache.Iggy.Kinds.Partitioning;
-
-namespace Apache.Iggy.IggyClient.Implementations;
-
-/// <summary>
-/// A TCP client for interacting with the Iggy server.
-/// </summary>
-public sealed class TcpMessageStream : IIggyClient
-{
- private readonly IggyClientConfigurator _configuration;
- private readonly EventAggregator<ConnectionStateChangedEventArgs> _connectionEvents;
- private readonly SemaphoreSlim _connectionSemaphore;
- private readonly ILogger<TcpMessageStream> _logger;
- private readonly SemaphoreSlim _sendingSemaphore;
- private string _currentAddress = string.Empty;
- private X509Certificate2Collection _customCaStore = [];
- private bool _isConnecting;
- private DateTimeOffset _lastConnectionTime;
- private ConnectionState _state = ConnectionState.Disconnected;
- private TcpConnectionStream _stream = null!;
-
- internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory)
- {
- _configuration = configuration;
- _logger = loggerFactory.CreateLogger<TcpMessageStream>();
- _sendingSemaphore = new SemaphoreSlim(1, 1);
- _connectionSemaphore = new SemaphoreSlim(1, 1);
- _lastConnectionTime = DateTimeOffset.MinValue;
- _connectionEvents = new EventAggregator<ConnectionStateChangedEventArgs>(loggerFactory);
- }
-
- /// <summary>
- /// Fired whenever the connection state changes.
- /// </summary>
- //public event EventHandler<ConnectionStateChangedEventArgs>? OnConnectionStateChanged;
- public void Dispose()
- {
- _stream?.Close();
- _stream?.Dispose();
- _sendingSemaphore.Dispose();
- _connectionSemaphore.Dispose();
- _connectionEvents.Clear();
- }
-
- /// <inheritdoc />
- public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- _connectionEvents.Subscribe(callback);
- }
-
- /// <inheritdoc />
- public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
- {
- _connectionEvents.Unsubscribe(callback);
- }
-
- /// <inheritdoc />
- public string GetCurrentAddress()
- {
- return _currentAddress;
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
- {
- var message = TcpContracts.CreateStream(name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- throw new InvalidResponseException("Received empty response while trying to create stream.");
- }
-
- return BinaryMapper.MapStream(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapStream(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapStreams(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
- {
- var message = TcpContracts.UpdateStream(streamId, name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
- CancellationToken token = default)
- {
- var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapTopics(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
- CancellationToken token = default)
- {
- var message = TcpContracts.GetTopicById(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapTopic(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
- TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
- {
- var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
- var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm,
- replicationFactor, messageExpiryValue, maxTopicSize);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapTopic(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
- CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
- ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
- CancellationToken token = default)
- {
- var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
- var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize,
- messageExpiryValue, replicationFactor);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- var message = TcpContracts.DeleteTopic(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
- {
- var message = TcpContracts.PurgeTopic(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
-
- /// <inheritdoc />
- public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
- IList<Message> messages, CancellationToken token = default)
- {
- var metadataLength = 2 + streamId.Length + 2 + topicId.Length
- + 2 + partitioning.Length + 4 + 4;
- var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages)
- + metadataLength;
- var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
-
- IMemoryOwner<byte> messageBuffer = MemoryPool<byte>.Shared.Rent(messageBufferSize);
- IMemoryOwner<byte> payloadBuffer = MemoryPool<byte>.Shared.Rent(payloadBufferSize);
- try
- {
- TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId,
- topicId, partitioning, messages);
-
- TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize],
- messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE);
-
- await SendWithResponseAsync(payloadBuffer.Memory[..payloadBufferSize].ToArray(), token);
- }
- finally
- {
- messageBuffer.Dispose();
- payloadBuffer.Dispose();
- }
- }
-
- /// <inheritdoc />
- public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
- CancellationToken token = default)
- {
- var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync);
-
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
- Consumer consumer,
- PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
- {
- var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer);
- var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize);
- var message = new byte[messageBufferSize];
- var payload = new byte[payloadBufferSize];
-
- TcpContracts.GetMessages(message.AsSpan()[..messageBufferSize], consumer, streamId,
- topicId, pollingStrategy, count, autoCommit, partitionId);
- TcpMessageStreamHelpers.CreatePayload(payload, message.AsSpan()[..messageBufferSize],
- CommandCodes.POLL_MESSAGES_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- return BinaryMapper.MapMessages(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
- uint? partitionId, CancellationToken token = default)
- {
- var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
- uint? partitionId, CancellationToken token = default)
- {
- var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapOffsets(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
- CancellationToken token = default)
- {
- var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
- Identifier topicId,
- CancellationToken token = default)
- {
- var message = TcpContracts.GetGroups(streamId, topicId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapConsumerGroups(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
- Identifier groupId, CancellationToken token = default)
- {
- var message = TcpContracts.GetGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapConsumerGroup(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
- string name, CancellationToken token = default)
- {
- var message = TcpContracts.CreateGroup(streamId, topicId, name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapConsumerGroup(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var message = TcpContracts.DeleteGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var message = TcpContracts.JoinGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
- CancellationToken token = default)
- {
- var message = TcpContracts.LeaveGroup(streamId, topicId, groupId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
- CancellationToken token = default)
- {
- var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
- uint segmentsCount, CancellationToken token = default)
- {
- var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapClient(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapStats(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapClusterMetadata(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task PingAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
- IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
- {
- var message = TcpContracts.GetSnapshot(compression, snapshotTypes);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE);
-
- return await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task ConnectAsync(CancellationToken token = default)
- {
- if (_state is ConnectionState.Connected
- or ConnectionState.Authenticating
- or ConnectionState.Authenticated)
- {
- _logger.LogWarning("Connection is already connected");
- return;
- }
-
- if (_lastConnectionTime != DateTimeOffset.MinValue)
- {
- await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token);
- }
-
- SetConnectionStateAsync(ConnectionState.Connecting);
- _isConnecting = true;
- try
- {
- await TryEstablishConnectionAsync(token);
- }
- finally
- {
- _isConnecting = false;
- }
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapClients(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
- {
- var message = TcpContracts.GetClient(clientId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapClient(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
- {
- var message = TcpContracts.GetUser(userId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapUser(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapUsers(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
- Permissions? permissions = null, CancellationToken token = default)
- {
- var message = TcpContracts.CreateUser(userName, password, status, permissions);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapUser(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
- {
- var message = TcpContracts.DeleteUser(userId);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
- CancellationToken token = default)
- {
- var message = TcpContracts.UpdateUser(userId, userName, status);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
- CancellationToken token = default)
- {
- var message = TcpContracts.UpdatePermissions(userId, permissions);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
- CancellationToken token = default)
- {
- var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
- {
- if (_state == ConnectionState.Disconnected)
- {
- throw new NotConnectedException();
- }
-
- // TODO: Add binary protocol version
- var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk");
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE);
-
- SetConnectionStateAsync(ConnectionState.Authenticating);
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length <= 0)
- {
- return null;
- }
-
- var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.AsSpan()[..responseBuffer.Length]);
- SetConnectionStateAsync(ConnectionState.Authenticated);
-
- if (await RedirectAsync(token))
- {
- await ConnectAsync(token);
- return await LoginUserAsync(userName, password, token);
- }
-
- var authResponse = new AuthResponse(userId, null);
- return authResponse;
- }
-
- /// <inheritdoc />
- public async Task LogoutUserAsync(CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
- CancellationToken token = default)
- {
- var message = Array.Empty<byte>();
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return [];
- }
-
- return BinaryMapper.MapPersonalAccessTokens(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
- CancellationToken token = default)
- {
- var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry));
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE);
-
- var responseBuffer = await SendWithResponseAsync(payload, token);
-
- if (responseBuffer.Length == 0)
- {
- return null;
- }
-
- return BinaryMapper.MapRawPersonalAccessToken(responseBuffer);
- }
-
- /// <inheritdoc />
- public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
- {
- var message = TcpContracts.DeletePersonalRequestToken(name);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE);
-
- await SendWithResponseAsync(payload, token);
- }
-
- /// <inheritdoc />
- public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
- {
- var message = TcpContracts.LoginWithPersonalAccessToken(token);
- var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
- TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE);
-
- SetConnectionStateAsync(ConnectionState.Authenticating);
- var responseBuffer = await SendWithResponseAsync(payload, ct);
-
- if (responseBuffer.Length <= 1)
- {
- return null;
- }
-
- var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.AsSpan()[..4]);
-
- SetConnectionStateAsync(ConnectionState.Authenticated);
-
- if (await RedirectAsync(ct))
- {
- await ConnectAsync(ct);
- return await LoginWithPersonalAccessTokenAsync(token, ct);
- }
-
- return new AuthResponse(userId, null);
- }
-
- private async Task TryEstablishConnectionAsync(CancellationToken token)
- {
- var retryCount = 0;
- var delay = _configuration.ReconnectionSettings.InitialDelay;
- do
- {
- // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
- _stream?.Close();
- _stream?.Dispose();
-
- if (string.IsNullOrEmpty(_currentAddress))
- {
- _currentAddress = _configuration.BaseAddress;
- }
-
- var urlPortSplitter = _currentAddress.Split(":");
- if (urlPortSplitter.Length > 2)
- {
- throw new InvalidBaseAddressException();
- }
-
- try
- {
- var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
- socket.SendBufferSize = _configuration.SendBufferSize;
- socket.ReceiveBufferSize = _configuration.ReceiveBufferSize;
-
- await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token);
-
- socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
- socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5);
-
- SetConnectionStateAsync(ConnectionState.Connected);
- _lastConnectionTime = DateTimeOffset.UtcNow;
-
- _stream = _configuration.TlsSettings.Enabled switch
- {
- true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings),
- false => new TcpConnectionStream(new NetworkStream(socket, true))
- };
-
- if (_configuration.AutoLoginSettings.Enabled)
- {
- _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}",
- _configuration.AutoLoginSettings.Username);
- await LoginUserAsync(_configuration.AutoLoginSettings.Username,
- _configuration.AutoLoginSettings.Password, token);
- }
-
- break;
- }
- catch (Exception e)
- {
- _logger.LogError(e, "Failed to connect");
-
- if (!_configuration.ReconnectionSettings.Enabled ||
- (_configuration.ReconnectionSettings.MaxRetries > 0 &&
- retryCount >= _configuration.ReconnectionSettings.MaxRetries))
- {
- SetConnectionStateAsync(ConnectionState.Disconnected);
- throw;
- }
-
- retryCount++;
- if (_configuration.ReconnectionSettings.UseExponentialBackoff)
- {
- delay *= _configuration.ReconnectionSettings.BackoffMultiplier;
-
- if (delay > _configuration.ReconnectionSettings.MaxDelay)
- {
- delay = _configuration.ReconnectionSettings.MaxDelay;
- }
- }
-
- if (_logger.IsEnabled(LogLevel.Information))
- {
- _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount,
- delay);
- }
-
- await Task.Delay(delay, token);
- }
- } while (true);
- }
-
- private async Task<ClusterNode?> GetCurrentLeaderNodeAsync(CancellationToken token)
- {
- try
- {
- var clusterMetadata = await GetClusterMetadataAsync(token);
- if (clusterMetadata == null)
- {
- return null;
- }
-
- // Single-node cluster (clustering disabled) - no redirection needed
- if (clusterMetadata.Nodes.Count() == 1)
- {
- return null;
- }
-
- var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader);
- if (leaderNode == null)
- {
- throw new MissingLeaderException();
- }
-
- return leaderNode;
- }
- // todo: change after error refactoring, error code 5 is for feature not supported
- catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5)
- {
- return null;
- }
- }
-
- private async Task<TcpConnectionStream> CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings)
- {
- ValidateCertificatePath(tlsSettings.CertificatePath);
-
- _customCaStore = new X509Certificate2Collection();
- _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath);
- var stream = new NetworkStream(socket, true);
- var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback);
-
- await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname);
-
- return new TcpConnectionStream(sslStream);
- }
-
- private async Task<byte[]> SendWithResponseAsync(byte[] payload, CancellationToken token = default)
- {
- try
- {
- return await SendRawAsync(payload, token);
- }
- catch (Exception e) when (IsConnectionException(e) && !_isConnecting)
- {
- _logger.LogWarning("Connection lost");
- if (!_configuration.ReconnectionSettings.Enabled)
- {
- _logger.LogWarning("Reconnection is disabled");
- SetConnectionStateAsync(ConnectionState.Disconnected);
- throw;
- }
-
- return await HandleReconnectionAsync(payload, token);
- }
- }
-
- private async Task<byte[]> HandleReconnectionAsync(byte[] payload, CancellationToken token)
- {
- var currentTime = DateTimeOffset.UtcNow;
- await _connectionSemaphore.WaitAsync(token);
-
- try
- {
- if (_state is ConnectionState.Connected or ConnectionState.Authenticated
- && _lastConnectionTime > currentTime)
- {
- _logger.LogInformation("Connection already established, sending payload");
- return await SendRawAsync(payload, token);
- }
-
- SetConnectionStateAsync(ConnectionState.Disconnected);
- _logger.LogInformation("Reconnecting to the server");
- await ConnectAsync(token);
-
- _logger.LogInformation("Reconnected to the server");
-
- await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token);
-
- return await SendRawAsync(payload, token);
- }
- finally
- {
- _connectionSemaphore.Release();
- }
- }
-
- private async Task<byte[]> SendRawAsync(byte[] payload, CancellationToken token)
- {
- if (_state is ConnectionState.Disconnected or ConnectionState.Connecting)
- {
- throw new NotConnectedException();
- }
-
- try
- {
- await _sendingSemaphore.WaitAsync(token);
- await _stream.SendAsync(payload, token);
- await _stream.FlushAsync(token);
-
- // Read the 8-byte header (4 bytes status + 4 bytes length)
- var buffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE];
- var totalRead = 0;
- while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE)
- {
- var readBytes
- = await _stream.ReadAsync(
- buffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead),
- token);
- if (readBytes == 0)
- {
- throw new IggyZeroBytesException();
- }
-
- totalRead += readBytes;
- }
-
- var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(buffer);
-
- if (response.Status != 0)
- {
- if (response.Length == 0)
- {
- throw new IggyInvalidStatusCodeException(response.Status,
- $"Invalid response status code: {response.Status}");
- }
-
- var errorBuffer = new byte[response.Length];
- totalRead = 0;
- while (totalRead < response.Length)
- {
- var readBytes
- = await _stream.ReadAsync(errorBuffer.AsMemory(totalRead, response.Length - totalRead), token);
- if (readBytes == 0)
- {
- throw new IggyZeroBytesException();
- }
-
- totalRead += readBytes;
- }
-
- throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer));
- }
-
- if (response.Length == 0)
- {
- return [];
- }
-
- var responseBuffer = new byte[response.Length];
- totalRead = 0;
- while (totalRead < response.Length)
- {
- var readBytes = await _stream.ReadAsync(responseBuffer.AsMemory(totalRead, response.Length - totalRead),
- token);
- if (readBytes == 0)
- {
- throw new IggyZeroBytesException();
- }
-
- totalRead += readBytes;
- }
-
- return responseBuffer;
- }
- finally
- {
- _sendingSemaphore.Release();
- }
- }
-
- private static bool IsConnectionException(Exception ex)
- {
- return ex is IggyZeroBytesException or
- NotConnectedException or
- SocketException or
- IOException or
- ObjectDisposedException;
- }
-
- private static int CalculatePayloadBufferSize(int messageBufferSize)
- {
- return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
- }
-
- private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer)
- {
- // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length
- // Added 1 byte for partition flag
- return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length;
- }
-
- /// <summary>
- /// Sets the connection state and fires the OnConnectionStateChanged event.
- /// </summary>
- /// <param name="newState">The new connection state</param>
- private void SetConnectionStateAsync(ConnectionState newState)
- {
- if (_state == newState)
- {
- return;
- }
-
- var previousState = _state;
- _state = newState;
-
- _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState);
- _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState));
- }
-
- private void ValidateCertificatePath(string tlsCertificatePath)
- {
- if (string.IsNullOrEmpty(tlsCertificatePath)
- || !File.Exists(tlsCertificatePath))
- {
- throw new InvalidCertificatePathException(tlsCertificatePath);
- }
- }
-
- private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain,
- SslPolicyErrors sslPolicyErrors)
- {
- if (sslPolicyErrors == SslPolicyErrors.None)
- {
- return true;
- }
-
- if (certificate is null)
- {
- return false;
- }
-
- if (certificate is not X509Certificate2 serverCert)
- {
- serverCert = new X509Certificate2(certificate);
- }
-
- if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint))
- {
- if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore)
- {
- return true;
- }
-
- _logger.LogError(
- "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}",
- serverCert.NotBefore, serverCert.NotAfter);
- return false;
- }
-
-
- using var customChain = new X509Chain();
- customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
- customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
- foreach (var ca in _customCaStore)
- {
- customChain.ChainPolicy.CustomTrustStore.Add(ca);
- customChain.ChainPolicy.ExtraStore.Add(ca);
- }
-
- customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
-
- if (customChain.Build(new X509Certificate2(certificate)))
- {
- if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch))
- {
- return true;
- }
-
- _logger.LogError("Custom CA chain is valid, but hostname does not match");
- return false;
- }
-
- foreach (var chainStatus in customChain.ChainStatus)
- {
- _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status,
- chainStatus.StatusInformation);
- }
-
- return false;
- }
-
- private async Task<bool> RedirectAsync(CancellationToken token)
- {
- var currentLeaderNode = await GetCurrentLeaderNodeAsync(token);
- if (currentLeaderNode == null)
- {
- return false;
- }
-
- var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}";
- if (leaderAddress == _currentAddress)
- {
- return false;
- }
-
- _currentAddress = leaderAddress;
-
- _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}",
- leaderAddress);
-
- _stream.Close();
- SetConnectionStateAsync(ConnectionState.Disconnected);
- return true;
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Buffers;
+using System.Buffers.Binary;
+using System.Net.Security;
+using System.Net.Sockets;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using Apache.Iggy.Configuration;
+using Apache.Iggy.ConnectionStream;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Contracts.Auth;
+using Apache.Iggy.Contracts.Tcp;
+using Apache.Iggy.Enums;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Mappers;
+using Apache.Iggy.Messages;
+using Apache.Iggy.Utils;
+using Microsoft.Extensions.Logging;
+using Partitioning = Apache.Iggy.Kinds.Partitioning;
+
+namespace Apache.Iggy.IggyClient.Implementations;
+
+/// <summary>
+/// A TCP client for interacting with the Iggy server.
+/// </summary>
+public sealed class TcpMessageStream : IIggyClient
+{
+ private readonly IggyClientConfigurator _configuration;
+ private readonly EventAggregator<ConnectionStateChangedEventArgs> _connectionEvents;
+ private readonly SemaphoreSlim _connectionSemaphore;
+ private readonly ILogger<TcpMessageStream> _logger;
+ private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE];
+ private readonly SemaphoreSlim _sendingSemaphore;
+ private string _currentAddress = string.Empty;
+ private X509Certificate2Collection _customCaStore = [];
+ private bool _isConnecting;
+ private DateTimeOffset _lastConnectionTime;
+ private ConnectionState _state = ConnectionState.Disconnected;
+ private TcpConnectionStream _stream = null!;
+
+ internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory)
+ {
+ _configuration = configuration;
+ _logger = loggerFactory.CreateLogger<TcpMessageStream>();
+ _sendingSemaphore = new SemaphoreSlim(1, 1);
+ _connectionSemaphore = new SemaphoreSlim(1, 1);
+ _lastConnectionTime = DateTimeOffset.MinValue;
+ _connectionEvents = new EventAggregator<ConnectionStateChangedEventArgs>(loggerFactory);
+ }
+
+ /// <summary>
+ /// Fired whenever the connection state changes.
+ /// </summary>
+ //public event EventHandler<ConnectionStateChangedEventArgs>? OnConnectionStateChanged;
+ public void Dispose()
+ {
+ _stream?.Close();
+ _stream?.Dispose();
+ _sendingSemaphore.Dispose();
+ _connectionSemaphore.Dispose();
+ _connectionEvents.Clear();
+ }
+
+ /// <inheritdoc />
+ public void SubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ _connectionEvents.Subscribe(callback);
+ }
+
+ /// <inheritdoc />
+ public void UnsubscribeConnectionEvents(Func<ConnectionStateChangedEventArgs, Task> callback)
+ {
+ _connectionEvents.Unsubscribe(callback);
+ }
+
+ /// <inheritdoc />
+ public string GetCurrentAddress()
+ {
+ return _currentAddress;
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> CreateStreamAsync(string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.CreateStream(name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_STREAM_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ throw new InvalidResponseException("Received empty response while trying to create stream.");
+ }
+
+ return BinaryMapper.MapStream(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<StreamResponse?> GetStreamByIdAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAM_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapStream(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<StreamResponse>> GetStreamsAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STREAMS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapStreams(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateStreamAsync(Identifier streamId, string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdateStream(streamId, name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_STREAM_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task PurgeStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_STREAM_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteStreamAsync(Identifier streamId, CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_STREAM_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<TopicResponse>> GetTopicsAsync(Identifier streamId,
+ CancellationToken token = default)
+ {
+ var message = TcpMessageStreamHelpers.GetBytesFromIdentifier(streamId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPICS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapTopics(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> GetTopicByIdAsync(Identifier streamId, Identifier topicId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.GetTopicById(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_TOPIC_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapTopic(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<TopicResponse?> CreateTopicAsync(Identifier streamId, string name, uint partitionsCount,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null,
+ TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default)
+ {
+ var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
+ var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm,
+ replicationFactor, messageExpiryValue, maxTopicSize);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapTopic(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name,
+ CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None,
+ ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null,
+ CancellationToken token = default)
+ {
+ var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry);
+ var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize,
+ messageExpiryValue, replicationFactor);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteTopic(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, CancellationToken token = default)
+ {
+ var message = TcpContracts.PurgeTopic(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PURGE_TOPIC_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+
+ /// <inheritdoc />
+ public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning,
+ IList<Message> messages, CancellationToken token = default)
+ {
+ var metadataLength = 2 + streamId.Length + 2 + topicId.Length
+ + 2 + partitioning.Length + 4 + 4;
+ var messageBufferSize = TcpMessageStreamHelpers.CalculateMessageBytesCount(messages)
+ + metadataLength;
+ var payloadBufferSize = messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
+
+ IMemoryOwner<byte> messageBuffer = MemoryPool<byte>.Shared.Rent(messageBufferSize);
+ IMemoryOwner<byte> payloadBuffer = MemoryPool<byte>.Shared.Rent(payloadBufferSize);
+ try
+ {
+ TcpContracts.CreateMessage(messageBuffer.Memory.Span[..messageBufferSize], streamId,
+ topicId, partitioning, messages);
+
+ TcpMessageStreamHelpers.CreatePayload(payloadBuffer.Memory.Span[..payloadBufferSize],
+ messageBuffer.Memory.Span[..messageBufferSize], CommandCodes.SEND_MESSAGES_CODE);
+
+ await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token);
+ }
+ finally
+ {
+ messageBuffer.Dispose();
+ payloadBuffer.Dispose();
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync);
+
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessages> PollMessagesAsync(Identifier streamId, Identifier topicId, uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ using var rental = await PollMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy,
+ count, autoCommit, token);
+ return BinaryMapper.MaterializeMessages(rental);
+ }
+
+ /// <inheritdoc />
+ public async Task<PolledMessagesRental> PollMessagesRentedAsync(Identifier streamId, Identifier topicId,
+ uint? partitionId,
+ Consumer consumer,
+ PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default)
+ {
+ var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer);
+ var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize);
+ var payload = ArrayPool<byte>.Shared.Rent(payloadBufferSize);
+
+ try
+ {
+ TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId,
+ topicId, pollingStrategy, count, autoCommit, partitionId);
+ BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4);
+ BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE);
+
+ IMemoryOwner<byte> responseBuffer
+ = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token);
+ return BinaryMapper.MapRentedMessages(responseBuffer);
+ }
+ finally
+ {
+ ArrayPool<byte>.Shared.Return(payload);
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task StoreOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, ulong offset,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdateOffset(streamId, topicId, consumer, offset, partitionId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.STORE_CONSUMER_OFFSET_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<OffsetResponse?> GetOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId,
+ uint? partitionId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetOffset(streamId, topicId, consumer, partitionId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_OFFSET_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapOffsets(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteOffsetAsync(Consumer consumer, Identifier streamId, Identifier topicId, uint? partitionId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteOffset(streamId, topicId, consumer, partitionId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_OFFSET_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ConsumerGroupResponse>> GetConsumerGroupsAsync(Identifier streamId,
+ Identifier topicId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.GetGroups(streamId, topicId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUPS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapConsumerGroups(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> GetConsumerGroupByIdAsync(Identifier streamId, Identifier topicId,
+ Identifier groupId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CONSUMER_GROUP_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ConsumerGroupResponse?> CreateConsumerGroupAsync(Identifier streamId, Identifier topicId,
+ string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.CreateGroup(streamId, topicId, name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_CONSUMER_GROUP_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapConsumerGroup(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.JoinGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicId, Identifier groupId,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.LeaveGroup(streamId, topicId, groupId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.DeletePartitions(streamId, topicId, partitionsCount);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, uint partitionsCount,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.CreatePartitions(streamId, topicId, partitionsCount);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, uint partitionId,
+ uint segmentsCount, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteSegments(streamId, topicId, partitionId, segmentsCount);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_SEGMENTS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<ClientResponse?> GetMeAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_ME_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapClient(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<StatsResponse?> GetStatsAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_STATS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapStats(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ClusterMetadata?> GetClusterMetadataAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLUSTER_METADATA_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapClusterMetadata(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task PingAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<byte[]> GetSnapshotAsync(SnapshotCompression compression,
+ IList<SystemSnapshotType> snapshotTypes, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetSnapshot(compression, snapshotTypes);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_SNAPSHOT_CODE);
+
+ using IMemoryOwner<byte> result = await SendWithResponseAsync(payload, token);
+
+ return result.Memory.Span.ToArray();
+ }
+
+ /// <inheritdoc />
+ public async Task ConnectAsync(CancellationToken token = default)
+ {
+ if (_state is ConnectionState.Connected
+ or ConnectionState.Authenticating
+ or ConnectionState.Authenticated)
+ {
+ _logger.LogWarning("Connection is already connected");
+ return;
+ }
+
+ if (_lastConnectionTime != DateTimeOffset.MinValue)
+ {
+ await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token);
+ }
+
+ SetConnectionStateAsync(ConnectionState.Connecting);
+ _isConnecting = true;
+ try
+ {
+ await TryEstablishConnectionAsync(token);
+ }
+ finally
+ {
+ _isConnecting = false;
+ }
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<ClientResponse>> GetClientsAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENTS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapClients(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<ClientResponse?> GetClientByIdAsync(uint clientId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetClient(clientId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_CLIENT_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapClient(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> GetUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ var message = TcpContracts.GetUser(userId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USER_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapUser(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<UserResponse>> GetUsersAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_USERS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapUsers(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<UserResponse?> CreateUserAsync(string userName, string password, UserStatus status,
+ Permissions? permissions = null, CancellationToken token = default)
+ {
+ var message = TcpContracts.CreateUser(userName, password, status, permissions);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_USER_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapUser(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeleteUserAsync(Identifier userId, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeleteUser(userId);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_USER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdateUserAsync(Identifier userId, string? userName = null, UserStatus? status = null,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdateUser(userId, userName, status);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_USER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task UpdatePermissionsAsync(Identifier userId, Permissions? permissions = null,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.UpdatePermissions(userId, permissions);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_PERMISSIONS_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, string newPassword,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.ChangePassword(userId, currentPassword, newPassword);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CHANGE_PASSWORD_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginUserAsync(string userName, string password, CancellationToken token = default)
+ {
+ if (_state == ConnectionState.Disconnected)
+ {
+ throw new NotConnectedException();
+ }
+
+ // TODO: Add binary protocol version
+ var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk");
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE);
+
+ SetConnectionStateAsync(ConnectionState.Authenticating);
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]);
+ SetConnectionStateAsync(ConnectionState.Authenticated);
+
+ if (await RedirectAsync(token))
+ {
+ await ConnectAsync(token);
+ return await LoginUserAsync(userName, password, token);
+ }
+
+ var authResponse = new AuthResponse(userId, null);
+ return authResponse;
+ }
+
+ /// <inheritdoc />
+ public async Task LogoutUserAsync(CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<IReadOnlyList<PersonalAccessTokenResponse>> GetPersonalAccessTokensAsync(
+ CancellationToken token = default)
+ {
+ var message = Array.Empty<byte>();
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return [];
+ }
+
+ return BinaryMapper.MapPersonalAccessTokens(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task<RawPersonalAccessToken?> CreatePersonalAccessTokenAsync(string name, TimeSpan? expiry = null,
+ CancellationToken token = default)
+ {
+ var message = TcpContracts.CreatePersonalAccessToken(name, DurationHelpers.ToDuration(expiry));
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE);
+
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, token);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ return BinaryMapper.MapRawPersonalAccessToken(responseBuffer.Memory.Span);
+ }
+
+ /// <inheritdoc />
+ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken token = default)
+ {
+ var message = TcpContracts.DeletePersonalRequestToken(name);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE);
+
+ await SendAckAsync(payload, token);
+ }
+
+ /// <inheritdoc />
+ public async Task<AuthResponse?> LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default)
+ {
+ var message = TcpContracts.LoginWithPersonalAccessToken(token);
+ var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length];
+ TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE);
+
+ SetConnectionStateAsync(ConnectionState.Authenticating);
+ using IMemoryOwner<byte> responseBuffer = await SendWithResponseAsync(payload, ct);
+
+ if (responseBuffer.Memory.Length == 0)
+ {
+ return null;
+ }
+
+ var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]);
+
+ SetConnectionStateAsync(ConnectionState.Authenticated);
+
+ if (await RedirectAsync(ct))
+ {
+ await ConnectAsync(ct);
+ return await LoginWithPersonalAccessTokenAsync(token, ct);
+ }
+
+ return new AuthResponse(userId, null);
+ }
+
+ private async Task TryEstablishConnectionAsync(CancellationToken token)
+ {
+ var retryCount = 0;
+ var delay = _configuration.ReconnectionSettings.InitialDelay;
+ do
+ {
+ // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
+ _stream?.Close();
+ _stream?.Dispose();
+
+ if (string.IsNullOrEmpty(_currentAddress))
+ {
+ _currentAddress = _configuration.BaseAddress;
+ }
+
+ var urlPortSplitter = _currentAddress.Split(":");
+ if (urlPortSplitter.Length > 2)
+ {
+ throw new InvalidBaseAddressException();
+ }
+
+ try
+ {
+ var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+ socket.SendBufferSize = _configuration.SendBufferSize;
+ socket.ReceiveBufferSize = _configuration.ReceiveBufferSize;
+
+ await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token);
+
+ socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true);
+ socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5);
+
+ SetConnectionStateAsync(ConnectionState.Connected);
+ _lastConnectionTime = DateTimeOffset.UtcNow;
+
+ _stream = _configuration.TlsSettings.Enabled switch
+ {
+ true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings),
+ false => new TcpConnectionStream(new NetworkStream(socket, true))
+ };
+
+ if (_configuration.AutoLoginSettings.Enabled)
+ {
+ _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}",
+ _configuration.AutoLoginSettings.Username);
+ await LoginUserAsync(_configuration.AutoLoginSettings.Username,
+ _configuration.AutoLoginSettings.Password, token);
+ }
+
+ break;
+ }
+ catch (Exception e)
+ {
+ _logger.LogError(e, "Failed to connect");
+
+ if (!_configuration.ReconnectionSettings.Enabled ||
+ (_configuration.ReconnectionSettings.MaxRetries > 0 &&
+ retryCount >= _configuration.ReconnectionSettings.MaxRetries))
+ {
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ throw;
+ }
+
+ retryCount++;
+ if (_configuration.ReconnectionSettings.UseExponentialBackoff)
+ {
+ delay *= _configuration.ReconnectionSettings.BackoffMultiplier;
+
+ if (delay > _configuration.ReconnectionSettings.MaxDelay)
+ {
+ delay = _configuration.ReconnectionSettings.MaxDelay;
+ }
+ }
+
+ if (_logger.IsEnabled(LogLevel.Information))
+ {
+ _logger.LogInformation("Retrying connection attempt {RetryCount} with delay {Delay}", retryCount,
+ delay);
+ }
+
+ await Task.Delay(delay, token);
+ }
+ } while (true);
+ }
+
+ private async Task<ClusterNode?> GetCurrentLeaderNodeAsync(CancellationToken token)
+ {
+ try
+ {
+ var clusterMetadata = await GetClusterMetadataAsync(token);
+ if (clusterMetadata == null)
+ {
+ return null;
+ }
+
+ // Single-node cluster (clustering disabled) - no redirection needed
+ if (clusterMetadata.Nodes.Count() == 1)
+ {
+ return null;
+ }
+
+ var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader);
+ if (leaderNode == null)
+ {
+ throw new MissingLeaderException();
+ }
+
+ return leaderNode;
+ }
+ // todo: change after error refactoring, error code 5 is for feature not supported
+ catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5)
+ {
+ return null;
+ }
+ }
+
+ private async Task<TcpConnectionStream> CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings)
+ {
+ ValidateCertificatePath(tlsSettings.CertificatePath);
+
+ _customCaStore = new X509Certificate2Collection();
+ _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath);
+ var stream = new NetworkStream(socket, true);
+ var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback);
+
+ await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname);
+
+ return new TcpConnectionStream(sslStream);
+ }
+
+ private async Task SendAckAsync(ReadOnlyMemory<byte> payload, CancellationToken token = default)
+ {
+ using IMemoryOwner<byte> _ = await SendWithResponseAsync(payload, token);
+ }
+
+ private async Task<IMemoryOwner<byte>> SendWithResponseAsync(ReadOnlyMemory<byte> payload,
+ CancellationToken token = default)
+ {
+ try
+ {
+ return await SendRawAsync(payload, token);
+ }
+ catch (Exception e) when (IsConnectionException(e) && !_isConnecting)
+ {
+ _logger.LogWarning("Connection lost");
+ if (!_configuration.ReconnectionSettings.Enabled)
+ {
+ _logger.LogWarning("Reconnection is disabled");
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ throw;
+ }
+
+ return await HandleReconnectionAsync(payload, token);
+ }
+ }
+
+ private async Task<IMemoryOwner<byte>> HandleReconnectionAsync(ReadOnlyMemory<byte> payload,
+ CancellationToken token)
+ {
+ var currentTime = DateTimeOffset.UtcNow;
+ await _connectionSemaphore.WaitAsync(token);
+
+ try
+ {
+ if (_state is ConnectionState.Connected or ConnectionState.Authenticated
+ && _lastConnectionTime > currentTime)
+ {
+ _logger.LogInformation("Connection already established, sending payload");
+ return await SendRawAsync(payload, token);
+ }
+
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ _logger.LogInformation("Reconnecting to the server");
+ await ConnectAsync(token);
+
+ _logger.LogInformation("Reconnected to the server");
+
+ await Task.Delay(_configuration.ReconnectionSettings.WaitAfterReconnect, token);
+
+ return await SendRawAsync(payload, token);
+ }
+ finally
+ {
+ _connectionSemaphore.Release();
+ }
+ }
+
+ private async Task<IMemoryOwner<byte>> SendRawAsync(ReadOnlyMemory<byte> payload, CancellationToken token)
+ {
+ if (_state is ConnectionState.Disconnected or ConnectionState.Connecting)
+ {
+ throw new NotConnectedException();
+ }
+
+ await _sendingSemaphore.WaitAsync(token);
+
+ try
+ {
+ await _stream.SendAsync(payload, token);
+ await _stream.FlushAsync(token);
+
+ // Read the 8-byte header (4 bytes status + 4 bytes length)
+ var totalRead = 0;
+ while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE)
+ {
+ var readBytes
+ = await _stream.ReadAsync(
+ _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead),
+ token);
+ if (readBytes == 0)
+ {
+ throw new IggyZeroBytesException();
+ }
+
+ totalRead += readBytes;
+ }
+
+ var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer);
+
+ if (response.Status != 0)
+ {
+ if (response.Length == 0)
+ {
+ throw new IggyInvalidStatusCodeException(response.Status,
+ $"Invalid response status code: {response.Status}");
+ }
+
+ var errorBuffer = new byte[response.Length];
+ totalRead = 0;
+ while (totalRead < response.Length)
+ {
+ var readBytes
+ = await _stream.ReadAsync(errorBuffer.AsMemory(totalRead, response.Length - totalRead), token);
+ if (readBytes == 0)
+ {
+ throw new IggyZeroBytesException();
+ }
+
+ totalRead += readBytes;
+ }
+
+ throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer));
+ }
+
+ if (response.Length == 0)
+ {
+ return EmptyMemoryOwner.Instance;
+ }
+
+ var responseBuffer = ArrayPoolHelper.Rent(response.Length);
+ try
+ {
+ totalRead = 0;
+ while (totalRead < response.Length)
+ {
+ var readBytes
+ = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead),
+ token);
+
+ if (readBytes == 0)
+ {
+ throw new IggyZeroBytesException();
+ }
+
+ totalRead += readBytes;
+ }
+ }
+ catch
+ {
+ responseBuffer.Dispose();
+ throw;
+ }
+
+ return responseBuffer;
+ }
+ finally
+ {
+ _sendingSemaphore.Release();
+ }
+ }
+
+ private static bool IsConnectionException(Exception ex)
+ {
+ return ex is IggyZeroBytesException or
+ NotConnectedException or
+ SocketException or
+ IOException or
+ ObjectDisposedException;
+ }
+
+ private static int CalculatePayloadBufferSize(int messageBufferSize)
+ {
+ return messageBufferSize + 4 + BufferSizes.INITIAL_BYTES_LENGTH;
+ }
+
+ private static int CalculateMessageBufferSize(Identifier streamId, Identifier topicId, Consumer consumer)
+ {
+ // Original: 14 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.Id.Length
+ // Added 1 byte for partition flag
+ return 15 + 5 + 2 + streamId.Length + 2 + topicId.Length + 2 + consumer.ConsumerId.Length;
+ }
+
+ /// <summary>
+ /// Sets the connection state and fires the OnConnectionStateChanged event.
+ /// </summary>
+ /// <param name="newState">The new connection state</param>
+ private void SetConnectionStateAsync(ConnectionState newState)
+ {
+ if (_state == newState)
+ {
+ return;
+ }
+
+ var previousState = _state;
+ _state = newState;
+
+ _logger.LogInformation("Connection state changed: {PreviousState} -> {CurrentState}", previousState, newState);
+ _connectionEvents.Publish(new ConnectionStateChangedEventArgs(previousState, newState));
+ }
+
+ private void ValidateCertificatePath(string tlsCertificatePath)
+ {
+ if (string.IsNullOrEmpty(tlsCertificatePath)
+ || !File.Exists(tlsCertificatePath))
+ {
+ throw new InvalidCertificatePathException(tlsCertificatePath);
+ }
+ }
+
+ private bool RemoteCertificateValidationCallback(object sender, X509Certificate? certificate, X509Chain? chain,
+ SslPolicyErrors sslPolicyErrors)
+ {
+ if (sslPolicyErrors == SslPolicyErrors.None)
+ {
+ return true;
+ }
+
+ if (certificate is null)
+ {
+ return false;
+ }
+
+ if (certificate is not X509Certificate2 serverCert)
+ {
+ serverCert = new X509Certificate2(certificate);
+ }
+
+ if (_customCaStore.Any(ca => ca.Thumbprint == serverCert.Thumbprint))
+ {
+ if (DateTime.UtcNow <= serverCert.NotAfter && DateTime.UtcNow >= serverCert.NotBefore)
+ {
+ return true;
+ }
+
+ _logger.LogError(
+ "Server certificate matches trusted key but is expired. Valid from {NotBefore} to {NotAfter}",
+ serverCert.NotBefore, serverCert.NotAfter);
+ return false;
+ }
+
+
+ using var customChain = new X509Chain();
+ customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
+ customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
+ foreach (var ca in _customCaStore)
+ {
+ customChain.ChainPolicy.CustomTrustStore.Add(ca);
+ customChain.ChainPolicy.ExtraStore.Add(ca);
+ }
+
+ customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
+
+ if (customChain.Build(new X509Certificate2(certificate)))
+ {
+ if (!sslPolicyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch))
+ {
+ return true;
+ }
+
+ _logger.LogError("Custom CA chain is valid, but hostname does not match");
+ return false;
+ }
+
+ foreach (var chainStatus in customChain.ChainStatus)
+ {
+ _logger.LogWarning("Certificate validation failed: {ChainStatus} - {StatusInformation}", chainStatus.Status,
+ chainStatus.StatusInformation);
+ }
+
+ return false;
+ }
+
+ private async Task<bool> RedirectAsync(CancellationToken token)
+ {
+ var currentLeaderNode = await GetCurrentLeaderNodeAsync(token);
+ if (currentLeaderNode == null)
+ {
+ return false;
+ }
+
+ var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}";
+ if (leaderAddress == _currentAddress)
+ {
+ return false;
+ }
+
+ _currentAddress = leaderAddress;
+
+ _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}",
+ leaderAddress);
+
+ _stream.Close();
+ SetConnectionStateAsync(ConnectionState.Disconnected);
+ return true;
+ }
+
+ internal sealed class EmptyMemoryOwner : IMemoryOwner<byte>
+ {
+ public static readonly EmptyMemoryOwner Instance = new();
+
+ private EmptyMemoryOwner()
+ {
+ }
+
+ public Memory<byte> Memory => Memory<byte>.Empty;
+
+ public void Dispose()
+ {
+ }
+ }
+}
+
+internal static class ArrayPoolHelper
+{
+ public static SlicedMemoryOwner Rent(int minimumLength)
+ {
+ return new SlicedMemoryOwner(minimumLength);
+ }
+
+ internal sealed class SlicedMemoryOwner(int minimumLength) : IMemoryOwner<byte>
+ {
+ private readonly byte[] _value = ArrayPool<byte>.Shared.Rent(minimumLength);
+ private int _disposed;
+
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) != 0)
+ {
+ return;
+ }
+
+ ArrayPool<byte>.Shared.Return(_value);
+ }
+
+ public Memory<byte> Memory => _value.AsMemory()[..minimumLength];
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
index 651406eda..146a42ae8 100644
--- a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
+++ b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
@@ -7,7 +7,7 @@
<TargetFrameworks>net8.0;net10.0</TargetFrameworks>
<AssemblyName>Apache.Iggy</AssemblyName>
<RootNamespace>Apache.Iggy</RootNamespace>
- <Version>0.8.0</Version>
+ <Version>0.8.1-edge.1</Version>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs
index 9cb9821d8..ffdbf3260 100644
--- a/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs
+++ b/foreign/csharp/Iggy_SDK/JsonConverters/MessageResponseConverter.cs
@@ -1,98 +1,106 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using Apache.Iggy.Contracts;
-using Apache.Iggy.Headers;
-using Apache.Iggy.Messages;
-
-namespace Apache.Iggy.JsonConverters;
-
-internal sealed class MessageResponseConverter : JsonConverter<MessageResponse>
-{
- private static readonly UserHeadersConverter HeadersConverter = new();
-
- public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
- {
- if (reader.TokenType != JsonTokenType.StartObject)
- {
- throw new JsonException("Expected start of object for MessageResponse.");
- }
-
- MessageHeader? header = null;
- byte[]? payload = null;
- Dictionary<HeaderKey, HeaderValue>? userHeaders = null;
- byte[]? rawUserHeaders = null;
-
- while (reader.Read())
- {
- if (reader.TokenType == JsonTokenType.EndObject)
- {
- break;
- }
-
- if (reader.TokenType != JsonTokenType.PropertyName)
- {
- throw new JsonException("Expected property name.");
- }
-
- var propertyName = reader.GetString();
- reader.Read();
-
- switch (propertyName)
- {
- case "header":
- header = JsonSerializer.Deserialize<MessageHeader>(ref reader, options);
- break;
- case "payload":
- payload = reader.GetBytesFromBase64();
- break;
- case "user_headers":
- if (reader.TokenType == JsonTokenType.String)
- {
- rawUserHeaders = reader.GetBytesFromBase64();
- }
- else if (reader.TokenType == JsonTokenType.Null)
- {
- userHeaders = null;
- }
- else
- {
- userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary<HeaderKey, HeaderValue>), options);
- }
- break;
- default:
- reader.Skip();
- break;
- }
- }
-
- return new MessageResponse
- {
- Header = header ?? throw new JsonException("Missing 'header' field."),
- Payload = payload ?? throw new JsonException("Missing 'payload' field."),
- UserHeaders = userHeaders,
- RawUserHeaders = rawUserHeaders
- };
- }
-
- public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options)
- {
- JsonSerializer.Serialize(writer, value, options);
- }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Headers;
+using Apache.Iggy.Messages;
+
+namespace Apache.Iggy.JsonConverters;
+
+internal sealed class MessageResponseConverter : JsonConverter<MessageResponse>
+{
+ private static readonly UserHeadersConverter HeadersConverter = new();
+
+ public override MessageResponse Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+ {
+ if (reader.TokenType != JsonTokenType.StartObject)
+ {
+ throw new JsonException("Expected start of object for MessageResponse.");
+ }
+
+ MessageHeader? header = null;
+ byte[]? payload = null;
+ Dictionary<HeaderKey, HeaderValue>? userHeaders = null;
+ byte[]? rawUserHeaders = null;
+
+ while (reader.Read())
+ {
+ if (reader.TokenType == JsonTokenType.EndObject)
+ {
+ break;
+ }
+
+ if (reader.TokenType != JsonTokenType.PropertyName)
+ {
+ throw new JsonException("Expected property name.");
+ }
+
+ var propertyName = reader.GetString();
+ reader.Read();
+
+ switch (propertyName)
+ {
+ case "header":
+ header = JsonSerializer.Deserialize<MessageHeader>(ref reader, options);
+ break;
+ case "payload":
+ payload = reader.GetBytesFromBase64();
+ break;
+ case "user_headers":
+ if (reader.TokenType == JsonTokenType.String)
+ {
+ rawUserHeaders = reader.GetBytesFromBase64();
+ }
+ else if (reader.TokenType == JsonTokenType.Null)
+ {
+ userHeaders = null;
+ }
+ else
+ {
+ userHeaders = HeadersConverter.Read(ref reader, typeof(Dictionary<HeaderKey, HeaderValue>),
+ options);
+ }
+
+ break;
+ default:
+ reader.Skip();
+ break;
+ }
+ }
+
+ var response = new MessageResponse
+ {
+ Header = header ?? throw new JsonException("Missing 'header' field."),
+ Payload = payload ?? throw new JsonException("Missing 'payload' field."),
+ RawUserHeaders = rawUserHeaders
+ };
+
+ if (rawUserHeaders is null)
+ {
+ response.UserHeaders = userHeaders;
+ }
+
+ return response;
+ }
+
+ public override void Write(Utf8JsonWriter writer, MessageResponse value, JsonSerializerOptions options)
+ {
+ JsonSerializer.Serialize(writer, value, options);
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
index 890b66802..ad47f0399 100644
--- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
+++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
@@ -23,6 +23,7 @@ using Apache.Iggy.Contracts.Auth;
using Apache.Iggy.Enums;
using Apache.Iggy.Extensions;
using Apache.Iggy.Headers;
+using Apache.Iggy.IggyClient.Implementations;
using Apache.Iggy.Messages;
using Apache.Iggy.Utils;
@@ -227,34 +228,38 @@ internal static class BinaryMapper
var readBytes = 4 + 8 + 1 + 1 + usernameLength;
return (new UserResponse
- {
- Id = id,
- CreatedAt = createdAt,
- Status = userStatus,
- Username = username
- },
+ {
+ Id = id,
+ CreatedAt = createdAt,
+ Status = userStatus,
+ Username = username
+ },
readBytes);
}
internal static ClientResponse MapClient(ReadOnlySpan<byte> payload)
{
var (response, position) = MapClientInfo(payload, 0);
- var consumerGroups = new List<ConsumerGroupInfo>(response.ConsumerGroupsCount);
+ var consumerGroups = new List<ConsumerGroupInfo>();
+ var length = payload.Length;
- for (var i = 0; i < response.ConsumerGroupsCount; i++)
+ while (position < length)
{
- var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]);
- var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]);
- var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]);
- var consumerGroup
- = new ConsumerGroupInfo
- {
- StreamId = streamId,
- TopicId = topicId,
- GroupId = consumerGroupId
- };
- consumerGroups.Add(consumerGroup);
- position += 12;
+ for (var i = 0; i < response.ConsumerGroupsCount; i++)
+ {
+ var streamId = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]);
+ var topicId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 4)..(position + 8)]);
+ var consumerGroupId = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 8)..(position + 12)]);
+ var consumerGroup
+ = new ConsumerGroupInfo
+ {
+ StreamId = streamId,
+ TopicId = topicId,
+ GroupId = consumerGroupId
+ };
+ consumerGroups.Add(consumerGroup);
+ position += 12;
+ }
}
return new ClientResponse
@@ -332,85 +337,88 @@ internal static class BinaryMapper
};
}
- internal static PolledMessages MapMessages(ReadOnlySpan<byte> payload)
+ internal static PolledMessagesRental MapRentedMessages(IMemoryOwner<byte> payloadOwner)
{
+ try
+ {
+ return MapRentedMessages(payloadOwner.Memory, payloadOwner);
+ }
+ catch
+ {
+ payloadOwner.Dispose();
+ throw;
+ }
+ }
+
+ internal static PolledMessagesRental MapRentedMessages(ReadOnlyMemory<byte> payload,
+ IMemoryOwner<byte> payloadOwner)
+ {
+ ReadOnlySpan<byte> span = payload.Span;
var length = payload.Length;
- var partitionId = BinaryPrimitives.ReadInt32LittleEndian(payload[..4]);
- var currentOffset = BinaryPrimitives.ReadUInt64LittleEndian(payload[4..12]);
- var messagesCount = BinaryPrimitives.ReadUInt32LittleEndian(payload[12..16]);
+ var partitionId = BinaryPrimitives.ReadInt32LittleEndian(span[..4]);
+ var currentOffset = BinaryPrimitives.ReadUInt64LittleEndian(span[4..12]);
+ var messagesCount = BinaryPrimitives.ReadUInt32LittleEndian(span[12..16]);
var position = 16;
if (position >= length)
{
- return PolledMessages.Empty;
+ return new PolledMessagesRental(payloadOwner)
+ {
+ PartitionId = partitionId,
+ CurrentOffset = currentOffset,
+ Messages = []
+ };
}
- List<MessageResponse> messages = new();
+ List<RentedMessageResponse> messages = new((int)messagesCount);
while (position < length)
{
- var checksum = BinaryPrimitives.ReadUInt64LittleEndian(payload[position..(position + 8)]);
- var id = BinaryPrimitives.ReadUInt128LittleEndian(payload[(position + 8)..(position + 24)]);
- var offset = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 24)..(position + 32)]);
- var timestamp = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 32)..(position + 40)]);
- var originTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 40)..(position + 48)]);
- var headersLength = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 48)..(position + 52)]);
- var payloadLength = BinaryPrimitives.ReadInt32LittleEndian(payload[(position + 52)..(position + 56)]);
- var reserved = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 56)..(position + 64)]);
+ var checksum = BinaryPrimitives.ReadUInt64LittleEndian(span[position..(position + 8)]);
+ var id = BinaryPrimitives.ReadUInt128LittleEndian(span[(position + 8)..(position + 24)]);
+ var offset = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 24)..(position + 32)]);
+ var timestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 32)..(position + 40)]);
+ var originTimestamp = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 40)..(position + 48)]);
+ var headersLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 48)..(position + 52)]);
+ var payloadLength = BinaryPrimitives.ReadInt32LittleEndian(span[(position + 52)..(position + 56)]);
+ var reserved = BinaryPrimitives.ReadUInt64LittleEndian(span[(position + 56)..(position + 64)]);
var wireHeadersLength = headersLength;
- byte[]? rawUserHeaders = null;
- Dictionary<HeaderKey, HeaderValue>? headers;
- if (headersLength == 0)
- {
- headers = null;
- }
- else if (headersLength < 0)
+ if (headersLength < 0)
{
throw new ArgumentOutOfRangeException();
}
- else
- {
- rawUserHeaders = payload[(position + 64 + payloadLength)..(position + 64 + payloadLength + headersLength)].ToArray();
- headers = TryMapHeaders(rawUserHeaders);
- }
var payloadRangeStart = position + 64;
var payloadRangeEnd = position + 64 + payloadLength;
- if (payloadRangeStart > length || payloadRangeEnd > length)
+ var headersRangeStart = payloadRangeEnd;
+ var headersRangeEnd = headersRangeStart + headersLength;
+ if (payloadRangeStart > length || payloadRangeEnd > length || headersRangeStart > length ||
+ headersRangeEnd > length)
{
break;
}
- ReadOnlySpan<byte> payloadSlice = payload[payloadRangeStart..payloadRangeEnd];
- var messagePayload = ArrayPool<byte>.Shared.Rent(payloadSlice.Length);
- var payloadSliceLen = payloadSlice.Length;
+ ReadOnlyMemory<byte> payloadSlice = payload.Slice(payloadRangeStart, payloadLength);
+ ReadOnlyMemory<byte> rawHeaders = headersLength > 0
+ ? payload.Slice(headersRangeStart, headersLength)
+ : ReadOnlyMemory<byte>.Empty;
- try
+ messages.Add(new RentedMessageResponse
{
- payloadSlice.CopyTo(messagePayload.AsSpan()[..payloadSliceLen]);
-
- messages.Add(new MessageResponse
+ Header = new MessageHeader
{
- Header = new MessageHeader
- {
- Checksum = checksum,
- Id = id,
- Offset = offset,
- OriginTimestamp = originTimestamp,
- PayloadLength = payloadLength,
- Timestamp = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(timestamp),
- UserHeadersLength = headersLength,
- Reserved = reserved
- },
- UserHeaders = headers,
- RawUserHeaders = rawUserHeaders,
- Payload = messagePayload[..payloadSliceLen]
- });
- }
- finally
- {
- ArrayPool<byte>.Shared.Return(messagePayload);
- }
+ Checksum = checksum,
+ Id = id,
+ Offset = offset,
+ OriginTimestamp = originTimestamp,
+ PayloadLength = payloadLength,
+ Timestamp = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(timestamp),
+ UserHeadersLength = headersLength,
+ Reserved = reserved
+ },
+ RawUserHeaders = rawHeaders,
+ Payload = payloadSlice
+ });
position += 64 + payloadLength + wireHeadersLength;
if (position + PropertiesSize >= length)
@@ -419,11 +427,59 @@ internal static class BinaryMapper
}
}
- return new PolledMessages
+ return new PolledMessagesRental(payloadOwner)
{
PartitionId = partitionId,
CurrentOffset = currentOffset,
- Messages = messages.AsReadOnly()
+ Messages = messages
+ };
+ }
+
+ internal static PolledMessages MaterializeMessages(PolledMessagesRental rental)
+ {
+ if (rental.Messages.Count == 0 && rental.PartitionId == 0 && rental.CurrentOffset == 0)
+ {
+ return PolledMessages.Empty;
+ }
+
+ var messages = new List<MessageResponse>(rental.Messages.Count);
+ foreach (var message in rental.Messages)
+ {
+ messages.Add(new MessageResponse
+ {
+ Header = message.Header,
+ RawUserHeaders = message.RawUserHeaders.IsEmpty ? null : message.RawUserHeaders.ToArray(),
+ Payload = message.Payload.ToArray()
+ });
+ }
+
+ return new PolledMessages
+ {
+ PartitionId = rental.PartitionId,
+ CurrentOffset = rental.CurrentOffset,
+ Messages = messages
+ };
+ }
+
+ internal static PolledMessagesRental ToRentedMessages(PolledMessages messages)
+ {
+ var rentedMessages = new List<RentedMessageResponse>(messages.Messages.Count);
+ foreach (var message in messages.Messages)
+ {
+ rentedMessages.Add(new RentedMessageResponse
+ {
+ Header = message.Header,
+ Payload = message.Payload,
+ RawUserHeaders = message.RawUserHeaders ?? ReadOnlyMemory<byte>.Empty,
+ UserHeaders = message.UserHeaders
+ });
+ }
+
+ return new PolledMessagesRental(TcpMessageStream.EmptyMemoryOwner.Instance)
+ {
+ PartitionId = messages.PartitionId,
+ CurrentOffset = messages.CurrentOffset,
+ Messages = rentedMessages
};
}
@@ -480,36 +536,61 @@ internal static class BinaryMapper
while (position < payload.Length)
{
if (!TryMapHeaderKind(payload[position], out var keyKind))
+ {
return null;
+ }
+
position++;
if (position + 4 > payload.Length)
+ {
return null;
+ }
+
var keyLength = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]);
if (keyLength is <= 0 or > 255)
+ {
return null;
+ }
position += 4;
if (position + keyLength > payload.Length)
+ {
return null;
+ }
+
var keyValue = payload[position..(position + keyLength)].ToArray();
position += keyLength;
if (position >= payload.Length)
+ {
return null;
+ }
+
if (!TryMapHeaderKind(payload[position], out var valueKind))
+ {
return null;
+ }
+
position++;
if (position + 4 > payload.Length)
+ {
return null;
+ }
+
var valueLength = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]);
if (valueLength is <= 0 or > 255)
+ {
return null;
+ }
position += 4;
if (position + valueLength > payload.Length)
+ {
return null;
+ }
+
ReadOnlySpan<byte> value = payload[position..(position + valueLength)];
position += valueLength;
@@ -550,6 +631,7 @@ internal static class BinaryMapper
kind = MapHeaderKind(value);
return true;
}
+
kind = default;
return false;
}
@@ -871,11 +953,11 @@ internal static class BinaryMapper
}
return (new ConsumerGroupMember
- {
- Id = id,
- PartitionsCount = partitionsCount,
- Partitions = partitions
- },
+ {
+ Id = id,
+ PartitionsCount = partitionsCount,
+ Partitions = partitions
+ },
8 + partitionsCount * 4);
}
diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
new file mode 100644
index 000000000..e9a8084a8
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedConsumerTests.cs
@@ -0,0 +1,483 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Buffers;
+using System.Runtime.CompilerServices;
+using System.Text;
+using Apache.Iggy.Consumers;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.Encryption;
+using Apache.Iggy.Exceptions;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.IggyClient.Implementations;
+using Apache.Iggy.Kinds;
+using Apache.Iggy.Messages;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+
+namespace Apache.Iggy.Tests.ConsumerTests;
+
+public class RentedConsumerTests
+{
+ [Fact]
+ public async Task ReceiveRentedAsync_Should_YieldMessages_WithExpectedPayloads()
+ {
+ var owner = new TrackingMemoryOwner(1024);
+ IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 3);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 2,
+ Messages = messages
+ };
+ Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
+ var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance);
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ var got = new List<ReceivedRentedMessage>();
+ await using (IAsyncEnumerator<ReceivedRentedMessage> e = consumer.ReceiveRentedAsync(cts.Token)
+ .GetAsyncEnumerator(TestContext.Current.CancellationToken))
+ {
+ for (var i = 0; i < 3; i++)
+ {
+ Assert.True(await e.MoveNextAsync());
+ got.Add(e.Current);
+ }
+ }
+
+ Assert.Equal(3, got.Count);
+ for (var i = 0; i < 3; i++)
+ {
+ Assert.Equal(MessageStatus.Success, got[i].Status);
+ Assert.Equal($"msg-{i}", Encoding.UTF8.GetString(got[i].Message.Payload.Span));
+ Assert.Equal((ulong)i, got[i].CurrentOffset);
+ Assert.Equal(1u, got[i].PartitionId);
+ Assert.Null(got[i].Error);
+ }
+
+ // Buffer still rented — none of the messages have been disposed.
+ Assert.Equal(0, owner.DisposeCount);
+
+ foreach (var m in got)
+ {
+ m.Dispose();
+ }
+
+ // After disposing every message of the batch, rental returned to pool exactly once.
+ Assert.Equal(1, owner.DisposeCount);
+ await consumer.DisposeAsync();
+ }
+
+ [Fact]
+ public async Task ReceiveRentedAsync_Should_NotReturnBuffer_UntilAllMessagesDisposed()
+ {
+ var owner = new TrackingMemoryOwner(1024);
+ IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 3);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 2,
+ Messages = messages
+ };
+ Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
+ var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance);
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ var got = new List<ReceivedRentedMessage>();
+ await using (IAsyncEnumerator<ReceivedRentedMessage> e = consumer.ReceiveRentedAsync(cts.Token)
+ .GetAsyncEnumerator(TestContext.Current.CancellationToken))
+ {
+ for (var i = 0; i < 3; i++)
+ {
+ Assert.True(await e.MoveNextAsync());
+ got.Add(e.Current);
+ }
+ }
+
+ // Dispose only the first two: buffer must still be alive.
+ got[0].Dispose();
+ Assert.Equal(0, owner.DisposeCount);
+ got[1].Dispose();
+ Assert.Equal(0, owner.DisposeCount);
+
+ // Final dispose returns the buffer.
+ got[2].Dispose();
+ Assert.Equal(1, owner.DisposeCount);
+
+ // Calling Dispose again is a no-op — refcount must not drop below zero.
+ got[0].Dispose();
+ got[1].Dispose();
+ got[2].Dispose();
+ Assert.Equal(1, owner.DisposeCount);
+
+ await consumer.DisposeAsync();
+ }
+
+ [Fact]
+ public async Task ReceiveRentedAsync_Should_YieldDecryptionFailed_AndStillReleaseBuffer_WhenEncryptorThrows()
+ {
+ var owner = new TrackingMemoryOwner(1024);
+ IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 1);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = messages
+ };
+ Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
+ var encryptor = new ThrowingEncryptor();
+ var config = BuildConfig();
+ config.MessageEncryptor = encryptor;
+ var consumer = new IggyConsumer(client.Object, config, NullLoggerFactory.Instance);
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ ReceivedRentedMessage? got = null;
+ await using (IAsyncEnumerator<ReceivedRentedMessage> e = consumer.ReceiveRentedAsync(cts.Token)
+ .GetAsyncEnumerator(TestContext.Current.CancellationToken))
+ {
+ Assert.True(await e.MoveNextAsync());
+ got = e.Current;
+ }
+
+ Assert.NotNull(got);
+ Assert.Equal(MessageStatus.DecryptionFailed, got!.Status);
+ Assert.IsType<InvalidOperationException>(got.Error);
+
+ Assert.Equal(0, owner.DisposeCount);
+ got.Dispose();
+ Assert.Equal(1, owner.DisposeCount);
+
+ await consumer.DisposeAsync();
+ }
+
+ [Fact]
+ public async Task ReceiveRentedAsync_Should_Throw_WhenConsumerNotInitialized()
+ {
+ Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>());
+ var consumer = new IggyConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance);
+
+ await Assert.ThrowsAsync<ConsumerNotInitializedException>(async () =>
+ {
+ await foreach (var _ in consumer.ReceiveRentedAsync(TestContext.Current.CancellationToken))
+ {
+ }
+ });
+
+ await consumer.DisposeAsync();
+ }
+
+ [Fact]
+ public void RentedBatchHandle_Release_BeyondAcquired_Throws()
+ {
+ var owner = new TrackingMemoryOwner(16);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = Array.Empty<RentedMessageResponse>()
+ };
+ var handle = new RentedBatchHandle(rental);
+
+ handle.Release(); // releases self-ref -> 0 -> disposes rental
+ Assert.Equal(1, owner.DisposeCount);
+
+ Assert.Throws<InvalidOperationException>(() => handle.Release());
+ }
+
+ [Fact]
+ public void RentedBatchHandle_AcquireRelease_Balanced_DisposesOnce()
+ {
+ var owner = new TrackingMemoryOwner(16);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = Array.Empty<RentedMessageResponse>()
+ };
+ var handle = new RentedBatchHandle(rental); // refCount=1 (self-ref)
+
+ handle.Acquire(); // 2
+ handle.Acquire(); // 3
+ Assert.Equal(0, owner.DisposeCount);
+
+ handle.Release(); // 2
+ handle.Release(); // 1 (self-ref still alive)
+ Assert.Equal(0, owner.DisposeCount);
+
+ handle.Release(); // 0 -> disposed
+ Assert.Equal(1, owner.DisposeCount);
+ }
+
+ [Fact]
+ public void PolledMessagesRental_Dispose_Idempotent()
+ {
+ var owner = new TrackingMemoryOwner(16);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = Array.Empty<RentedMessageResponse>()
+ };
+
+ rental.Dispose();
+ rental.Dispose();
+ rental.Dispose();
+
+ Assert.Equal(1, owner.DisposeCount);
+ }
+
+ [Fact]
+ public async Task PolledMessagesRental_ConcurrentDispose_ReturnsBufferOnce()
+ {
+ var owner = new TrackingMemoryOwner(16);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = Array.Empty<RentedMessageResponse>()
+ };
+
+ using var start = new ManualResetEventSlim(false);
+ Task[] tasks = Enumerable.Range(0, 64).Select(_ => Task.Run(() =>
+ {
+ start.Wait();
+ rental.Dispose();
+ })).ToArray();
+
+ start.Set();
+ await Task.WhenAll(tasks);
+
+ Assert.Equal(1, owner.DisposeCount);
+ }
+
+ [Fact]
+ public void PolledMessagesRental_ForgotDispose_FinalizerReturnsBuffer()
+ {
+ var owner = new TrackingMemoryOwner(16);
+ MakeAndDrop(owner);
+
+ GC.Collect();
+ GC.WaitForPendingFinalizers();
+ GC.Collect();
+
+ Assert.Equal(1, owner.DisposeCount);
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ static void MakeAndDrop(IMemoryOwner<byte> o)
+ {
+ _ = new PolledMessagesRental(o)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = Array.Empty<RentedMessageResponse>()
+ };
+ }
+ }
+
+ [Fact]
+ public async Task PollRented_MidLoopPublishFailure_DoesNotLeakBuffer()
+ {
+ var owner = new TrackingMemoryOwner(1024);
+ IReadOnlyList<RentedMessageResponse> messages = BuildMessages(owner, 5);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 4,
+ Messages = messages
+ };
+ Mock<IIggyClient> client = BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
+ var consumer
+ = new FailingPublishConsumer(client.Object, BuildConfig(), NullLoggerFactory.Instance) { FailAfter = 2 };
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ var got = new List<ReceivedRentedMessage>();
+ IAsyncEnumerator<ReceivedRentedMessage> enumerator = consumer.ReceiveRentedAsync(cts.Token)
+ .GetAsyncEnumerator(TestContext.Current.CancellationToken);
+
+ for (var i = 0; i < 2; i++)
+ {
+ Assert.True(await enumerator.MoveNextAsync());
+ got.Add(enumerator.Current);
+ }
+
+ // First 2 publishes succeeded, 3rd injected failure aborted the loop.
+ // Producer self-ref was released in finally; consumer refs (2) still hold the buffer.
+ Assert.Equal(2, got.Count);
+ Assert.Equal(0, owner.DisposeCount);
+
+ foreach (var m in got)
+ {
+ m.Dispose();
+ }
+
+ // All refs released -> rental disposed exactly once.
+ Assert.Equal(1, owner.DisposeCount);
+
+ cts.Cancel();
+ try
+ {
+ await enumerator.DisposeAsync();
+ }
+ catch (OperationCanceledException)
+ {
+ }
+
+ await consumer.DisposeAsync();
+ }
+
+ internal static IggyConsumerConfig BuildConfig()
+ {
+ return new IggyConsumerConfig
+ {
+ StreamId = Identifier.Numeric(1),
+ TopicId = Identifier.Numeric(1),
+ Consumer = Consumer.New(1),
+ PollingStrategy = PollingStrategy.Next(),
+ BatchSize = 10,
+ PartitionId = 1,
+ AutoCommitMode = AutoCommitMode.Disabled,
+ AutoCommit = false,
+ PollingIntervalMs = 0
+ };
+ }
+
+ /// <summary>
+ /// Slices payload bytes into the supplied owner's memory and returns a list of
+ /// <see cref="RentedMessageResponse" /> instances backed by that single rented buffer.
+ /// </summary>
+ internal static IReadOnlyList<RentedMessageResponse> BuildMessages(TrackingMemoryOwner owner, int count)
+ {
+ var list = new List<RentedMessageResponse>(count);
+ Memory<byte> buffer = owner.Memory;
+ var written = 0;
+ for (var i = 0; i < count; i++)
+ {
+ var bytes = Encoding.UTF8.GetBytes($"msg-{i}");
+ bytes.CopyTo(buffer.Slice(written, bytes.Length));
+ Memory<byte> slice = buffer.Slice(written, bytes.Length);
+ written += bytes.Length;
+
+ list.Add(new RentedMessageResponse
+ {
+ Header = new MessageHeader
+ {
+ Offset = (ulong)i,
+ PayloadLength = bytes.Length
+ },
+ Payload = slice,
+ RawUserHeaders = ReadOnlyMemory<byte>.Empty
+ });
+ }
+
+ return list;
+ }
+
+ /// <summary>
+ /// Builds a <see cref="Mock{IIggyClient}" /> that dequeues rentals on each
+ /// <c>PollMessagesRentedAsync</c> call. When the queue is empty, returns an
+ /// empty rental so the consumer can spin without dereferencing null.
+ /// </summary>
+ internal static Mock<IIggyClient> BuildClientMock(Queue<PolledMessagesRental> rentals)
+ {
+ var mock = new Mock<IIggyClient>(MockBehavior.Loose);
+ mock.Setup(c => c.ConnectAsync(It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
+ mock.Setup(c => c.PollMessagesRentedAsync(It.IsAny<Identifier>(), It.IsAny<Identifier>(), It.IsAny<uint?>(),
+ It.IsAny<Consumer>(), It.IsAny<PollingStrategy>(), It.IsAny<uint>(), It.IsAny<bool>(),
+ It.IsAny<CancellationToken>()))
+ .ReturnsAsync(() =>
+ {
+ if (rentals.Count > 0)
+ {
+ return rentals.Dequeue();
+ }
+
+ return new PolledMessagesRental(TcpMessageStream.EmptyMemoryOwner.Instance)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = Array.Empty<RentedMessageResponse>()
+ };
+ });
+ return mock;
+ }
+
+ /// <summary>
+ /// <see cref="IMemoryOwner{T}" /> wrapper that counts how many times
+ /// <see cref="IDisposable.Dispose" /> has been invoked, so tests can assert that
+ /// the rental returns to the pool exactly once.
+ /// </summary>
+ internal sealed class TrackingMemoryOwner : IMemoryOwner<byte>
+ {
+ private readonly byte[] _buffer;
+
+ public int DisposeCount { get; private set; }
+
+ public TrackingMemoryOwner(int size)
+ {
+ _buffer = new byte[size];
+ }
+
+ public Memory<byte> Memory => _buffer;
+
+ public void Dispose()
+ {
+ DisposeCount++;
+ }
+ }
+
+ private sealed class ThrowingEncryptor : IMessageEncryptor
+ {
+ public byte[] Encrypt(Span<byte> plainData)
+ {
+ throw new NotSupportedException();
+ }
+
+ public byte[] Decrypt(ReadOnlySpan<byte> encryptedData)
+ {
+ throw new InvalidOperationException("decrypt fail");
+ }
+ }
+
+ private sealed class FailingPublishConsumer : IggyConsumer
+ {
+ private int _calls;
+
+ public int FailAfter { get; set; }
+
+ public FailingPublishConsumer(IIggyClient client, IggyConsumerConfig config,
+ ILoggerFactory loggerFactory)
+ : base(client, config, loggerFactory)
+ {
+ }
+
+ protected override async Task PublishRentedAsync(RentedBatchHandle rental, RentedMessageResponse message,
+ uint partitionId, MessageStatus status, Exception? error, CancellationToken ct)
+ {
+ if (Interlocked.Increment(ref _calls) > FailAfter)
+ {
+ throw new InvalidOperationException("inject publish failure");
+ }
+
+ await base.PublishRentedAsync(rental, message, partitionId, status, error, ct);
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs
new file mode 100644
index 000000000..26dfd89e5
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/RentedTypedConsumerTests.cs
@@ -0,0 +1,183 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text;
+using Apache.Iggy.Consumers;
+using Apache.Iggy.Contracts;
+using Apache.Iggy.IggyClient;
+using Apache.Iggy.Kinds;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+
+namespace Apache.Iggy.Tests.ConsumerTests;
+
+public class RentedTypedConsumerTests
+{
+ [Fact]
+ public async Task ReceiveRentedDeserializedAsync_Should_YieldDeserialized_AndReturnBuffer()
+ {
+ var owner = new RentedConsumerTests.TrackingMemoryOwner(1024);
+ IReadOnlyList<RentedMessageResponse> messages = RentedConsumerTests.BuildMessages(owner, 3);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 2,
+ Messages = messages
+ };
+ Mock<IIggyClient> client
+ = RentedConsumerTests.BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
+ var deserializer = new StringDeserializer();
+ var consumer
+ = new IggyConsumer<string>(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance);
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ var got = new List<DeserializedMessage<string>>();
+ await using (IAsyncEnumerator<DeserializedMessage<string>> e = consumer
+ .ReceiveRentedDeserializedAsync(cts.Token)
+ .GetAsyncEnumerator(TestContext.Current.CancellationToken))
+ {
+ for (var i = 0; i < 3; i++)
+ {
+ Assert.True(await e.MoveNextAsync());
+ got.Add(e.Current);
+ }
+ }
+
+ Assert.Equal(3, got.Count);
+ for (var i = 0; i < 3; i++)
+ {
+ Assert.Equal(MessageStatus.Success, got[i].Status);
+ Assert.Equal($"msg-{i}", got[i].Data);
+ Assert.Equal((ulong)i, got[i].CurrentOffset);
+ Assert.Equal(1u, got[i].PartitionId);
+ Assert.Null(got[i].Error);
+ }
+
+ // Entire batch deserialized before first yield; rental already returned to pool.
+ Assert.Equal(1, owner.DisposeCount);
+ await consumer.DisposeAsync();
+ }
+
+ [Fact]
+ public async Task ReceiveRentedDeserializedAsync_Should_ReleaseEntireBatch_BeforeFirstYield()
+ {
+ var owner = new RentedConsumerTests.TrackingMemoryOwner(1024);
+ IReadOnlyList<RentedMessageResponse> messages = RentedConsumerTests.BuildMessages(owner, 3);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 2,
+ Messages = messages
+ };
+ Mock<IIggyClient> client
+ = RentedConsumerTests.BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
+ var deserializer = new StringDeserializer();
+ var consumer
+ = new IggyConsumer<string>(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance);
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ await using IAsyncEnumerator<DeserializedMessage<string>> e = consumer.ReceiveRentedDeserializedAsync(cts.Token)
+ .GetAsyncEnumerator(TestContext.Current.CancellationToken);
+
+ Assert.True(await e.MoveNextAsync());
+ Assert.Equal(1, owner.DisposeCount);
+ Assert.Equal("msg-0", e.Current.Data);
+
+ // Remaining messages come from the pre-deserialized list — no further disposal needed.
+ Assert.True(await e.MoveNextAsync());
+ Assert.Equal(1, owner.DisposeCount);
+ Assert.Equal("msg-1", e.Current.Data);
+
+ Assert.True(await e.MoveNextAsync());
+ Assert.Equal(1, owner.DisposeCount);
+ Assert.Equal("msg-2", e.Current.Data);
+
+ await consumer.DisposeAsync();
+ }
+
+ [Fact]
+ public async Task DeserializerThrows_Should_YieldDeserializationFailed_AndStillReturnBuffer()
+ {
+ var owner = new RentedConsumerTests.TrackingMemoryOwner(1024);
+ IReadOnlyList<RentedMessageResponse> messages = RentedConsumerTests.BuildMessages(owner, 1);
+ var rental = new PolledMessagesRental(owner)
+ {
+ PartitionId = 1,
+ CurrentOffset = 0,
+ Messages = messages
+ };
+ Mock<IIggyClient> client
+ = RentedConsumerTests.BuildClientMock(new Queue<PolledMessagesRental>(new[] { rental }));
+ var deserializer = new StringDeserializer { ThrowOnNext = true };
+ var consumer
+ = new IggyConsumer<string>(client.Object, BuildTypedConfig(deserializer), NullLoggerFactory.Instance);
+ await consumer.InitAsync(TestContext.Current.CancellationToken);
+
+ var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+ DeserializedMessage<string>? got = null;
+ await using (IAsyncEnumerator<DeserializedMessage<string>> e = consumer
+ .ReceiveRentedDeserializedAsync(cts.Token)
+ .GetAsyncEnumerator(TestContext.Current.CancellationToken))
+ {
+ Assert.True(await e.MoveNextAsync());
+ got = e.Current;
+ }
+
+ Assert.NotNull(got);
+ Assert.Equal(MessageStatus.DeserializationFailed, got!.Status);
+ Assert.Null(got.Data);
+ Assert.IsType<InvalidOperationException>(got.Error);
+
+ // Even on deserialization failure, the using-block inside the typed consumer releases the rental.
+ Assert.Equal(1, owner.DisposeCount);
+ await consumer.DisposeAsync();
+ }
+
+ private static IggyConsumerConfig<string> BuildTypedConfig(IDeserializer<string> deserializer)
+ {
+ return new IggyConsumerConfig<string>
+ {
+ StreamId = Identifier.Numeric(1),
+ TopicId = Identifier.Numeric(1),
+ Consumer = Consumer.New(1),
+ PollingStrategy = PollingStrategy.Next(),
+ BatchSize = 10,
+ PartitionId = 1,
+ AutoCommitMode = AutoCommitMode.Disabled,
+ AutoCommit = false,
+ PollingIntervalMs = 0,
+ Deserializer = deserializer
+ };
+ }
+
+ private sealed class StringDeserializer : IDeserializer<string>
+ {
+ public bool ThrowOnNext { get; set; }
+
+ public string Deserialize(ReadOnlyMemory<byte> data)
+ {
+ if (ThrowOnNext)
+ {
+ throw new InvalidOperationException("deserialize fail");
+ }
+
+ return Encoding.UTF8.GetString(data.Span);
+ }
+ }
+}
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
index 234f5239b..580d905f8 100644
--- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
+++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
@@ -19,6 +19,7 @@ using Apache.Iggy.Contracts;
using Apache.Iggy.Contracts.Auth;
using Apache.Iggy.Enums;
using Apache.Iggy.Extensions;
+using Apache.Iggy.IggyClient.Implementations;
using Apache.Iggy.Tests.Utils;
using Apache.Iggy.Tests.Utils.Groups;
using Apache.Iggy.Tests.Utils.Messages;
@@ -84,7 +85,8 @@ public sealed class BinaryMapper
msgTwoPayload.CopyTo(combinedPayload.AsSpan(16 + msgOnePayload.Length));
// Act
- var responses = Mappers.BinaryMapper.MapMessages(combinedPayload);
+ var responses
+ = Mappers.BinaryMapper.MapRentedMessages(combinedPayload, TcpMessageStream.EmptyMemoryOwner.Instance);
// Assert
Assert.NotNull(responses);
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/MessageResponseConverterTests.cs
@@ -0,0 +1 @@
+